func NewCmdVolume(f *cmdutil.Factory) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "volume",
		Short: "Creates a persisent volume for fabric8 apps needing persistent disk",
		Long:  `Creates a persisent volume so that the PersistentVolumeClaims in fabric8 apps can be satisfied when creating fabric8 apps`,
		PreRun: func(cmd *cobra.Command, args []string) {
			showBanner()
		},
		Run: func(cmd *cobra.Command, args []string) {
			c, cfg := client.NewClient(f)
			ns, _, err := f.DefaultNamespace()
			if err != nil {
				util.Fatal("No default namespace")
				printResult("Get default namespace", Failure, err)
			} else {
				util.Info("Creating a persistent volume for your ")
				util.Success(string(util.TypeOfMaster(c)))
				util.Info(" installation at ")
				util.Success(cfg.Host)
				util.Info(" in namespace ")
				util.Successf("%s\n\n", ns)

				r, err := createPersistentVolume(cmd, ns, c, f)
				printResult("Create PersistentVolume", r, err)
			}
		},
	}
	cmd.PersistentFlags().StringP(hostPathFlag, "", "", "Defines the host folder on which to define a persisent volume for single node setups")
	cmd.PersistentFlags().StringP(nameFlag, "", "fabric8", "The name of the PersistentVolume to create")
	return cmd
}

func logPackageVersion(packageName string, version string) {
	util.Info("Deploying package: ")
	util.Success(packageName)
	util.Info(" version: ")
	util.Success(version)
	util.Info("\n\n")
}

func NewCmdPull(f *cmdutil.Factory) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "pull [templateNames]",
		Short: "Pulls the docker images for the given templates",
		Long:  `Performs a docker pull on all the docker images referenced in the given templates to preload the local docker registry with images`,
		PreRun: func(cmd *cobra.Command, args []string) {
			showBanner()
		},
		Run: func(cmd *cobra.Command, args []string) {
			if len(args) < 1 {
				util.Error("No template names specified!")
				cmd.Usage()
			} else {
				_, cfg := client.NewClient(f)
				oc, _ := client.NewOpenShiftClient(cfg)
				ns, _, err := f.DefaultNamespace()
				if err != nil {
					util.Fatal("No default namespace")
				} else {
					for _, template := range args {
						util.Info("Downloading docker images for template ")
						util.Success(template)
						util.Info("\n\n")

						r, err := downloadTemplateDockerImages(cmd, ns, oc, f, template)
						printResult("Download Docker images", r, err)
					}
				}
			}
		},
	}
	return cmd
}

func NewCmdIngress(f *cmdutil.Factory) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "ingress",
		Short: "Creates any missing Ingress resources for services",
		Long:  `Creates any missing Ingress resources for Services which are of type LoadBalancer`,
		PreRun: func(cmd *cobra.Command, args []string) {
			showBanner()
		},
		Run: func(cmd *cobra.Command, args []string) {
			c, cfg := client.NewClient(f)
			ns, _, err := f.DefaultNamespace()
			if err != nil {
				util.Fatal("No default namespace")
				printResult("Get default namespace", Failure, err)
			} else {
				domain := cmd.Flags().Lookup(domainFlag).Value.String()

				util.Info("Setting up ingress on your ")
				util.Success(string(util.TypeOfMaster(c)))
				util.Info(" installation at ")
				util.Success(cfg.Host)
				util.Info(" in namespace ")
				util.Successf("%s at domain %s\n\n", ns, domain)
				err := createIngressForDomain(ns, domain, c, f)
				printError("Create Ingress", err)
			}
		},
	}
	cmd.PersistentFlags().StringP(domainFlag, "", defaultDomain(), "The domain to put the created routes inside")
	return cmd
}

// NewCmdCleanUp delete all fabric8 apps, environments and configurations
func NewCmdCleanUp(f *cmdutil.Factory) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "cleanup",
		Short: "Hard delete all fabric8 apps, environments and configurations",
		Long:  `Hard delete all fabric8 apps, environments and configurations`,

		Run: func(cmd *cobra.Command, args []string) {

			currentContext, err := util.GetCurrentContext()
			if err != nil {
				util.Fatalf("%s", err)
			}
			fmt.Fprintf(os.Stdout, `WARNING this is destructive and will remove ALL fabric8 apps, environments and configuration from cluster %s.  Continue? [y/N] `, currentContext)

			var confirm string
			fmt.Scanln(&confirm)

			if confirm == "y" {
				util.Info("Removing...\n")
				cleanUp(f)
				return
			}
			util.Info("Cancelled")
		},
	}

	return cmd
}

func NewCmdRoutes(f *cmdutil.Factory) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "routes",
		Short: "Creates any missing Routes for services",
		Long:  `Creates any missing Route resources for Services which need to be exposed remotely`,
		PreRun: func(cmd *cobra.Command, args []string) {
			showBanner()
		},
		Run: func(cmd *cobra.Command, args []string) {
			c, cfg := client.NewClient(f)
			oc, _ := client.NewOpenShiftClient(cfg)
			ns, _, err := f.DefaultNamespace()
			if err != nil {
				util.Fatal("No default namespace")
				printResult("Get default namespace", Failure, err)
			} else {
				util.Info("Creating a persistent volume for your ")
				util.Success(string(util.TypeOfMaster(c)))
				util.Info(" installation at ")
				util.Success(cfg.Host)
				util.Info(" in namespace ")
				util.Successf("%s\n\n", ns)

				domain := cmd.Flags().Lookup(domainFlag).Value.String()

				err := createRoutesForDomain(ns, domain, c, oc, f)
				printError("Create Routes", err)
			}
		},
	}
	cmd.PersistentFlags().StringP(domainFlag, "", defaultDomain(), "The domain to put the created routes inside")
	return cmd
}

func NewCmdSecrets(f *cmdutil.Factory) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "secrets",
		Short: "Set up Secrets on your Kubernetes or OpenShift environment",
		Long:  `set up Secrets on your Kubernetes or OpenShift environment`,
		PreRun: func(cmd *cobra.Command, args []string) {
			showBanner()
		},
		Run: func(cmd *cobra.Command, args []string) {
			c, cfg := client.NewClient(f)
			ns, _, _ := f.DefaultNamespace()
			util.Info("Setting up secrets on your ")
			util.Success(string(util.TypeOfMaster(c)))
			util.Info(" installation at ")
			util.Success(cfg.Host)
			util.Info(" in namespace ")
			util.Successf("%s\n\n", ns)

			if confirmAction(cmd.Flags()) {
				typeOfMaster := util.TypeOfMaster(c)

				if typeOfMaster == util.Kubernetes {
					util.Fatal("Support for Kubernetes not yet available...\n")
				} else {
					oc, _ := client.NewOpenShiftClient(cfg)
					t := getTemplates(oc, ns)

					count := 0
					// get all the Templates and find the annotations on any Pods
					for _, i := range t.Items {
						// convert TemplateList.Objects to Kubernetes resources
						_ = runtime.DecodeList(i.Objects, api.Scheme, runtime.UnstructuredJSONScheme)
						for _, rc := range i.Objects {
							switch rc := rc.(type) {
							case *api.ReplicationController:
								for secretType, secretDataIdentifiers := range rc.Spec.Template.Annotations {
									count += createAndPrintSecrets(secretDataIdentifiers, secretType, c, f, cmd.Flags())
								}
							}
						}
					}

					if count == 0 {
						util.Info("No secrets created as no fabric8 secrets annotations found in the templates\n")
						util.Info("For more details see: https://github.com/fabric8io/fabric8/blob/master/docs/secretAnnotations.md\n")
					}
				}
			}
		},
	}
	cmd.PersistentFlags().BoolP("print-import-folder-structure", "", true, "Prints the folder structures that are being used by the template annotations to import secrets")
	cmd.PersistentFlags().BoolP("write-generated-keys", "", false, "Write generated secrets to the local filesystem")
	cmd.PersistentFlags().BoolP("generate-secrets-data", "g", true, "Generate secrets data if secrets cannot be found to import from the local filesystem")
	return cmd
}

func NewCmdRun(f *cmdutil.Factory) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "run",
		Short: "Runs a fabric8 microservice from one of the installed templates",
		Long:  `runs a fabric8 microservice from one of the installed templates`,
		PreRun: func(cmd *cobra.Command, args []string) {
			showBanner()
		},
		Run: func(cmd *cobra.Command, args []string) {
			c, cfg := client.NewClient(f)
			ns, _, _ := f.DefaultNamespace()

			if len(args) == 0 {
				util.Info("Please specify a template name to run\n")
				return
			}
			domain := cmd.Flags().Lookup(domainFlag).Value.String()
			apiserver := cmd.Flags().Lookup(apiServerFlag).Value.String()
			pv := cmd.Flags().Lookup(pvFlag).Value.String() == "true"

			typeOfMaster := util.TypeOfMaster(c)

			util.Info("Running an app template to your ")
			util.Success(string(typeOfMaster))
			util.Info(" installation at ")
			util.Success(cfg.Host)
			util.Info(" for domain ")
			util.Success(domain)
			util.Info(" in namespace ")
			util.Successf("%s\n\n", ns)

			if len(apiserver) == 0 {
				apiserver = domain
			}

			yes := cmd.Flags().Lookup(yesFlag).Value.String() == "false"
			if strings.Contains(domain, "=") {
				util.Warnf("\nInvalid domain: %s\n\n", domain)

			} else if confirmAction(yes) {
				oc, _ := client.NewOpenShiftClient(cfg)
				initSchema()

				for _, app := range args {
					runTemplate(c, oc, app, ns, domain, apiserver, pv)
				}
			}
		},
	}
	cmd.PersistentFlags().StringP(domainFlag, "d", defaultDomain(), "The domain name to append to the service name to access web applications")
	cmd.PersistentFlags().String(apiServerFlag, "", "overrides the api server url")
	cmd.PersistentFlags().Bool(pvFlag, true, "Enable the use of persistence (enabling the PersistentVolumeClaims)?")
	return cmd
}

func runTemplate(c *k8sclient.Client, oc *oclient.Client, appToRun string, ns string, domain string, apiserver string, pv bool) {
	util.Info("\n\nInstalling: ")
	util.Successf("%s\n\n", appToRun)
	typeOfMaster := util.TypeOfMaster(c)
	if typeOfMaster == util.Kubernetes {
		jsonData, format, err := loadTemplateData(ns, appToRun, c, oc)
		if err != nil {
			printError("Failed to load app "+appToRun, err)
		}
		createTemplate(jsonData, format, appToRun, ns, domain, apiserver, c, oc, pv)
	} else {
		tmpl, err := oc.Templates(ns).Get(appToRun)
		if err != nil {
			printError("Failed to load template "+appToRun, err)
		}
		util.Infof("Loaded template with %d objects", len(tmpl.Objects))
		processTemplate(tmpl, ns, domain, apiserver)

		objectCount := len(tmpl.Objects)

		util.Infof("Creating "+appToRun+" template resources from %d objects\n", objectCount)
		for _, o := range tmpl.Objects {
			err = processItem(c, oc, &o, ns, pv)
		}
	}
}

func runTemplate(c *k8sclient.Client, oc *oclient.Client, appToRun string, ns string, domain string, apiserver string) {
	util.Info("\n\nInstalling: ")
	util.Successf("%s\n\n", appToRun)
	jsonData, err := loadTemplateData(ns, appToRun, c, oc)
	if err != nil {
		printError("Failed to load app "+appToRun, err)
	}
	createTemplate(jsonData, appToRun, ns, domain, apiserver, c)
}

func confirmAction(flags *pflag.FlagSet) bool {
	if flags.Lookup(yesFlag).Value.String() == "false" {
		util.Info("Continue? [Y/n] ")
		cont := util.AskForConfirmation(true)
		if !cont {
			util.Fatal("Cancelled...\n")
			return false
		}
	}
	return true
}

func NewCmdValidate(f *cmdutil.Factory) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "validate",
		Short: "Validate your Kubernetes or OpenShift environment",
		Long:  `validate your Kubernetes or OpenShift environment`,
		PreRun: func(cmd *cobra.Command, args []string) {
			showBanner()
		},
		Run: func(cmd *cobra.Command, args []string) {
			c, cfg := client.NewClient(f)
			ns, _, _ := f.DefaultNamespace()
			util.Info("Validating your ")
			util.Success(string(util.TypeOfMaster(c)))
			util.Info(" installation at ")
			util.Success(cfg.Host)
			util.Info(" in namespace ")
			util.Successf("%s\n\n", ns)
			printValidationResult("Service account", validateServiceAccount, c, f)
			printValidationResult("Console", validateConsoleDeployment, c, f)

			r, err := validateProxyServiceRestAPI(c, f, cfg.Host)
			printResult("REST Proxy Service API", r, err)

			if util.TypeOfMaster(c) == util.Kubernetes {
				printValidationResult("Jenkinshift Service", validateJenkinshiftService, c, f)
			}

			if util.TypeOfMaster(c) == util.OpenShift {
				printValidationResult("Router", validateRouter, c, f)
				oc, _ := client.NewOpenShiftClient(cfg)
				printOValidationResult("Templates", validateTemplates, oc, f)
				printValidationResult("SecurityContextConstraints", validateSecurityContextConstraints, c, f)
			}

			printValidationResult("PersistentVolumeClaims", validatePersistenceVolumeClaims, c, f)
			printValidationResult("ConfigMaps", validateConfigMaps, c, f)
		},
	}

	return cmd
}

func validatePersistenceVolumeClaims(c *k8sclient.Client, f *cmdutil.Factory) (Result, error) {
	ns, _, err := f.DefaultNamespace()
	if err != nil {
		return Failure, err
	}
	rc, err := c.PersistentVolumeClaims(ns).List(api.ListOptions{})
	if err != nil {
		util.Fatalf("Failed to get PersistentVolumeClaims, %s in namespace %s\n", err, ns)
	}
	if rc != nil {
		items := rc.Items
		pendingClaimNames := make([]string, 0, len(items))
		for _, item := range items {
			status := item.Status.Phase
			if status != "Bound" {
				pendingClaimNames = append(pendingClaimNames, item.ObjectMeta.Name)
			}
		}
		if len(pendingClaimNames) > 0 {
			util.Failuref("PersistentVolumeClaim not Bound for: %s. You need to create a PersistentVolume!\n", strings.Join(pendingClaimNames, ", "))
			util.Info(`
You can enable dynamic PersistentVolume creation with Kubernetes 1.4 or later.

Or to get gofabric8 to create HostPath based PersistentVolume resources for you on minikube and minishift type:

  gofabric8 volumes

For other clusters you could do something like this - though ideally with a persistent volume implementation other than hostPath:

cat <<EOF | oc create -f -
---
kind: PersistentVolume
apiVersion: v1
metadata:
  name: fabric8
spec:
  accessModes:
    - ReadWrite
  capacity:
    storage: 1000
  hostPath:
    path: /opt/fabric8-data
EOF


`)
			return Failure, err
		}
		return Success, err
	}
	return Failure, err
}

// simulates: oadm policy add-cluster-role-to-user roleName userName
func addClusterRoleToUser(c *oclient.Client, f *cmdutil.Factory, roleName string, userName string) error {
	namespace, _, err := f.DefaultNamespace()
	if err != nil {
		util.Info("No namespace!'\n")
		return err
	}
	options := policy.RoleModificationOptions{
		RoleName:            roleName,
		RoleBindingAccessor: policy.NewLocalRoleBindingAccessor(namespace, c),
		Users:               []string{userName},
	}
	return options.AddRole()
}

func printSummary(typeOfMaster util.MasterType, externalNodeName string, mini bool, ns string, domain string) {
	util.Info("\n")
	util.Info("-------------------------\n")
	util.Info("\n")
	clientType := getClientTypeName(typeOfMaster)

	if externalNodeName != "" {
		util.Info("Deploying ingress controller on node ")
		util.Successf("%s", externalNodeName)
		util.Info(" use its external ip when configuring your wildcard DNS.\n")
		util.Infof("To change node move the label: `%s label node %s %s- && %s label node $YOUR_NEW_NODE %s=true`\n", clientType, externalNodeName, externalIPLabel, clientType, externalIPLabel)
		util.Info("\n")
	}

	util.Info("Default GOGS admin username/password = ")
	util.Successf("%s/%s\n", gogsDefaultUsername, gogsDefaultPassword)
	util.Info("\n")

	util.Infof("Downloading images and waiting to open the fabric8 console...\n")
	util.Info("\n")
	util.Info("-------------------------\n")
}

func checkIfPVCsPending(c *k8sclient.Client, ns string) (bool, error) {
	timeout := time.After(20 * time.Second)
	tick := time.Tick(2 * time.Second)
	util.Info("Checking if PersistentVolumeClaims bind to a PersistentVolume ")
	// Keep trying until we're timed out or got a result or got an error
	for {
		select {
		// Got a timeout! fail with a timeout error
		case <-timeout:
			return true, errors.New("timed out")
		// Got a tick, check if PVc have bound
		case <-tick:
			found, _, _ := findPendingPVs(c, ns)
			if !found {
				util.Info("\n")
				return false, nil
			}
			util.Info(".")
			// retry
		}
	}
}

func downloadDockerImage(imageName string) error {
	util.Info("Downloading image ")
	util.Success(imageName)
	util.Info("\n")

	cmd := exec.Command("docker", "pull", imageName)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	var waitStatus syscall.WaitStatus
	if err := cmd.Run(); err != nil {
		printErr(err)
		if exitError, ok := err.(*exec.ExitError); ok {
			waitStatus = exitError.Sys().(syscall.WaitStatus)
			printStatus(waitStatus.ExitStatus())
		}
		return err
	} else {
		waitStatus = cmd.ProcessState.Sys().(syscall.WaitStatus)
		printStatus(waitStatus.ExitStatus())
		return nil
	}
}

func NewCmdCopyEndpoints(f *cmdutil.Factory) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "copy-endpoints",
		Short: "Copies endpoints from the current namespace to a target namespace",
		Long:  `Copies endpoints from the current namespace to a target namespace`,
		PreRun: func(cmd *cobra.Command, args []string) {
			showBanner()
		},
		Run: func(cmd *cobra.Command, args []string) {

			if len(args) == 0 {
				util.Info("Please specify one or more endpoint names to copy as arguments!\n")
				return
			}
			c, cfg := client.NewClient(f)
			oc, _ := client.NewOpenShiftClient(cfg)

			initSchema()

			toNamespace := cmd.Flags().Lookup(toNamespaceFlag).Value.String()

			fromNamespace := cmd.Flags().Lookup(fromNamespaceFlag).Value.String()
			if len(fromNamespace) == 0 {
				ns, _, err := f.DefaultNamespace()
				if err != nil {
					util.Fatal("No default namespace")
				}
				fromNamespace = ns
			}
			if len(toNamespace) == 0 {
				util.Fatal("No target namespace specified!")
			}

			util.Infof("Copying endpoints from namespace: %s to namespace: %s\n", fromNamespace, toNamespace)
			err := ensureNamespaceExists(c, oc, toNamespace)
			if err != nil {
				util.Fatalf("Failed to copy endpoints %v", err)
			}

			err = copyEndpoints(c, fromNamespace, toNamespace, args)
			if err != nil {
				util.Fatalf("Failed to copy endpoints %v", err)
			}
		},
	}
	cmd.PersistentFlags().StringP(fromNamespaceFlag, "f", "", "the source namespace or uses the default namespace")
	cmd.PersistentFlags().StringP(toNamespaceFlag, "t", "", "the destination namespace")
	return cmd
}

func validatePersistenceVolumeClaims(c *k8sclient.Client, f *cmdutil.Factory) (Result, error) {
	ns, _, err := f.DefaultNamespace()
	if err != nil {
		return Failure, err
	}
	rc, err := c.PersistentVolumeClaims(ns).List(labels.Everything(), fields.Everything())
	if err != nil {
		util.Fatalf("Failed to get PersistentVolumeClaims, %s in namespace %s\n", err, ns)
	}
	if rc != nil {
		items := rc.Items
		pendingClaimNames := make([]string, 0, len(items))
		for _, item := range items {
			status := item.Status.Phase
			if status != "Bound" {
				pendingClaimNames = append(pendingClaimNames, item.ObjectMeta.Name)
			}
		}
		if len(pendingClaimNames) > 0 {
			util.Failuref("PersistentVolumeClaim not Bound for: %s. You need to create a PersistentVolume!\n", strings.Join(pendingClaimNames, ", "))
			util.Info(`
to generate a single node PersistentVolume then type something like this:


cat <<EOF | oc create -f -
---
kind: PersistentVolume
apiVersion: v1
metadata:
  name: fabric8
spec:
  accessModes:
    - ReadWrite
  capacity:
    storage: 1000
  hostPath:
    path: /opt/fabric8-data
EOF


`)
			return Failure, err
		}
		return Success, err
	}
	return Failure, err
}

func keepTryingToGetClient(f *cmdutil.Factory) (*k8client.Client, *restclient.Config, error) {
	timeout := time.After(2 * time.Minute)
	tick := time.Tick(1 * time.Second)
	// Keep trying until we're timed out or got a result or got an error
	for {
		select {
		// Got a timeout! fail with a timeout error
		case <-timeout:
			return nil, nil, errors.New("timed out")
		// Got a tick, try and get teh client
		case <-tick:
			c, cfg, _ := getClient(f)
			// return if we have a client
			if c != nil {
				return c, cfg, nil
			}
			util.Info("Cannot connect to api server, retrying...\n")
			// retry
		}
	}
}