func (s *DockerSuite) TestHelpExitCodesHelpOutput(c *check.C) {
	// Test to make sure the exit code and output (stdout vs stderr) of
	// various good and bad cases are what we expect

	// docker : stdout=all, stderr=empty, rc=0
	out, _ := dockerCmd(c)
	// Be really pick
	c.Assert(out, checker.Not(checker.HasSuffix), "\n\n", check.Commentf("Should not have a blank line at the end of 'docker'\n"))

	// docker help: stdout=all, stderr=empty, rc=0
	out, _ = dockerCmd(c, "help")
	// Be really pick
	c.Assert(out, checker.Not(checker.HasSuffix), "\n\n", check.Commentf("Should not have a blank line at the end of 'docker help'\n"))

	// docker --help: stdout=all, stderr=empty, rc=0
	out, _ = dockerCmd(c, "--help")
	// Be really pick
	c.Assert(out, checker.Not(checker.HasSuffix), "\n\n", check.Commentf("Should not have a blank line at the end of 'docker --help'\n"))

	// docker inspect busybox: stdout=all, stderr=empty, rc=0
	// Just making sure stderr is empty on valid cmd
	out, _ = dockerCmd(c, "inspect", "busybox")
	// Be really pick
	c.Assert(out, checker.Not(checker.HasSuffix), "\n\n", check.Commentf("Should not have a blank line at the end of 'docker inspect busyBox'\n"))

	// docker rm: stdout=empty, stderr=all, rc!=0
	// testing the min arg error msg
	icmd.RunCommand(dockerBinary, "rm").Assert(c, icmd.Expected{
		ExitCode: 1,
		Error:    "exit status 1",
		Out:      "",
		// Should not contain full help text but should contain info about
		// # of args and Usage line
		Err: "requires at least 1 argument",
	})

	// docker rm NoSuchContainer: stdout=empty, stderr=all, rc=0
	// testing to make sure no blank line on error
	result := icmd.RunCommand(dockerBinary, "rm", "NoSuchContainer")
	result.Assert(c, icmd.Expected{
		ExitCode: 1,
		Error:    "exit status 1",
		Out:      "",
	})
	// Be really picky
	c.Assert(len(result.Stderr()), checker.Not(checker.Equals), 0)
	c.Assert(result.Stderr(), checker.Not(checker.HasSuffix), "\n\n", check.Commentf("Should not have a blank line at the end of 'docker rm'\n"))

	// docker BadCmd: stdout=empty, stderr=all, rc=0
	icmd.RunCommand(dockerBinary, "BadCmd").Assert(c, icmd.Expected{
		ExitCode: 1,
		Error:    "exit status 1",
		Out:      "",
		Err:      "docker: 'BadCmd' is not a docker command.\nSee 'docker --help'\n",
	})
}

// RunAtDifferentDate runs the specified function with the given time.
// It changes the date of the system, which can led to weird behaviors.
func RunAtDifferentDate(date time.Time, block func()) {
	// Layout for date. MMDDhhmmYYYY
	const timeLayout = "010203042006"
	// Ensure we bring time back to now
	now := time.Now().Format(timeLayout)
	defer icmd.RunCommand("date", now)

	icmd.RunCommand("date", date.Format(timeLayout))
	block()
	return
}

func (s *DockerSuite) TestLogsSince(c *check.C) {
	name := "testlogssince"
	dockerCmd(c, "run", "--name="+name, "busybox", "/bin/sh", "-c", "for i in $(seq 1 3); do sleep 2; echo log$i; done")
	out, _ := dockerCmd(c, "logs", "-t", name)

	log2Line := strings.Split(strings.Split(out, "\n")[1], " ")
	t, err := time.Parse(time.RFC3339Nano, log2Line[0]) // the timestamp log2 is written
	c.Assert(err, checker.IsNil)
	since := t.Unix() + 1 // add 1s so log1 & log2 doesn't show up
	out, _ = dockerCmd(c, "logs", "-t", fmt.Sprintf("--since=%v", since), name)

	// Skip 2 seconds
	unexpected := []string{"log1", "log2"}
	for _, v := range unexpected {
		c.Assert(out, checker.Not(checker.Contains), v, check.Commentf("unexpected log message returned, since=%v", since))
	}

	// Test to make sure a bad since format is caught by the client
	out, _, _ = dockerCmdWithError("logs", "-t", "--since=2006-01-02T15:04:0Z", name)
	c.Assert(out, checker.Contains, "cannot parse \"0Z\" as \"05\"", check.Commentf("bad since format passed to server"))

	// Test with default value specified and parameter omitted
	expected := []string{"log1", "log2", "log3"}
	for _, cmd := range [][]string{
		{"logs", "-t", name},
		{"logs", "-t", "--since=0", name},
	} {
		result := icmd.RunCommand(dockerBinary, cmd...)
		result.Assert(c, icmd.Success)
		for _, v := range expected {
			c.Assert(result.Combined(), checker.Contains, v)
		}
	}
}

// WaitInspectWithArgs waits for the specified expression to be equals to the specified expected string in the given time.
// FIXME(vdemeester) Attach this to the Daemon struct
func WaitInspectWithArgs(dockerBinary, name, expr, expected string, timeout time.Duration, arg ...string) error {
	after := time.After(timeout)

	args := append(arg, "inspect", "-f", expr, name)
	for {
		result := icmd.RunCommand(dockerBinary, args...)
		if result.Error != nil {
			if !strings.Contains(result.Stderr(), "No such") {
				return errors.Errorf("error executing docker inspect: %v\n%s",
					result.Stderr(), result.Stdout())
			}
			select {
			case <-after:
				return result.Error
			default:
				time.Sleep(10 * time.Millisecond)
				continue
			}
		}

		out := strings.TrimSpace(result.Stdout())
		if out == expected {
			break
		}

		select {
		case <-after:
			return errors.Errorf("condition \"%q == %q\" not true in time (%v)", out, expected, timeout)
		default:
		}

		time.Sleep(100 * time.Millisecond)
	}
	return nil
}

// blocking wait with 0 exit code
func (s *DockerSuite) TestWaitBlockedExitZero(c *check.C) {
	// Windows busybox does not support trap in this way, not sleep with sub-second
	// granularity. It will always exit 0x40010004.
	testRequires(c, DaemonIsLinux)
	out, _ := dockerCmd(c, "run", "-d", "busybox", "/bin/sh", "-c", "trap 'exit 0' TERM; while true; do usleep 10; done")
	containerID := strings.TrimSpace(out)

	c.Assert(waitRun(containerID), checker.IsNil)

	chWait := make(chan string)
	go func() {
		chWait <- ""
		out := icmd.RunCommand(dockerBinary, "wait", containerID).Combined()
		chWait <- out
	}()

	<-chWait // make sure the goroutine is started
	time.Sleep(100 * time.Millisecond)
	dockerCmd(c, "stop", containerID)

	select {
	case status := <-chWait:
		c.Assert(strings.TrimSpace(status), checker.Equals, "0", check.Commentf("expected exit 0, got %s", status))
	case <-time.After(2 * time.Second):
		c.Fatal("timeout waiting for `docker wait` to exit")
	}

}

func dockerCmd(c *check.C, args ...string) (string, int) {
	if err := validateArgs(args...); err != nil {
		c.Fatalf(err.Error())
	}
	result := icmd.RunCommand(dockerBinary, args...)
	c.Assert(result, icmd.Matches, icmd.Success)
	return result.Combined(), result.ExitCode
}

func inspectFilter(name, filter string) (string, error) {
	format := fmt.Sprintf("{{%s}}", filter)
	result := icmd.RunCommand(dockerBinary, "inspect", "-f", format, name)
	if result.Error != nil || result.ExitCode != 0 {
		return "", fmt.Errorf("failed to inspect %s: %s", name, result.Combined())
	}
	return strings.TrimSpace(result.Combined()), nil
}

func dockerCmdWithError(args ...string) (string, int, error) {
	if err := validateArgs(args...); err != nil {
		return "", 0, err
	}
	result := icmd.RunCommand(dockerBinary, args...)
	if result.Error != nil {
		return result.Combined(), result.ExitCode, result.Compare(icmd.Success)
	}
	return result.Combined(), result.ExitCode, result.Error
}

// FIXME(vdemeester) move this away are remove ignoreNoSuchContainer bool
func deleteContainer(ignoreNoSuchContainer bool, container ...string) error {
	result := icmd.RunCommand(dockerBinary, append([]string{"rm", "-fv"}, container...)...)
	if ignoreNoSuchContainer && result.Error != nil {
		// If the error is "No such container: ..." this means the container doesn't exists anymore,
		// we can safely ignore that one.
		if strings.Contains(result.Error.Error(), "No such container") {
			return nil
		}
	}
	return result.Compare(icmd.Success)
}

// FIXME(vdemeester) this should be a unit tests on cli/command/container package
func (s *DockerSuite) TestExecParseError(c *check.C) {
	// TODO Windows CI: Requires some extra work. Consider copying the
	// runSleepingContainer helper to have an exec version.
	testRequires(c, DaemonIsLinux)
	dockerCmd(c, "run", "-d", "--name", "top", "busybox", "top")

	// Test normal (non-detached) case first
	icmd.RunCommand(dockerBinary, "exec", "top").Assert(c, icmd.Expected{
		ExitCode: 1,
		Error:    "exit status 1",
		Err:      "See 'docker exec --help'",
	})
}

// FIXME(vdemeester) should be a unit test in cli/command/volume package
func (s *DockerSuite) TestVolumeCLINoArgs(c *check.C) {
	out, _ := dockerCmd(c, "volume")
	// no args should produce the cmd usage output
	usage := "Usage:	docker volume COMMAND"
	c.Assert(out, checker.Contains, usage)

	// invalid arg should error and show the command usage on stderr
	icmd.RunCommand(dockerBinary, "volume", "somearg").Assert(c, icmd.Expected{
		ExitCode: 1,
		Error:    "exit status 1",
		Err:      usage,
	})

	// invalid flag should error and show the flag error and cmd usage
	result := icmd.RunCommand(dockerBinary, "volume", "--no-such-flag")
	result.Assert(c, icmd.Expected{
		ExitCode: 125,
		Error:    "exit status 125",
		Err:      usage,
	})
	c.Assert(result.Stderr(), checker.Contains, "unknown flag: --no-such-flag")
}

func (s *DockerSuite) TestConfigHTTPHeader(c *check.C) {
	testRequires(c, UnixCli) // Can't set/unset HOME on windows right now
	// We either need a level of Go that supports Unsetenv (for cases
	// when HOME/USERPROFILE isn't set), or we need to be able to use
	// os/user but user.Current() only works if we aren't statically compiling

	var headers map[string][]string

	server := httptest.NewServer(http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("API-Version", api.DefaultVersion)
			headers = r.Header
		}))
	defer server.Close()

	homeKey := homedir.Key()
	homeVal := homedir.Get()
	tmpDir, err := ioutil.TempDir("", "fake-home")
	c.Assert(err, checker.IsNil)
	defer os.RemoveAll(tmpDir)

	dotDocker := filepath.Join(tmpDir, ".docker")
	os.Mkdir(dotDocker, 0600)
	tmpCfg := filepath.Join(dotDocker, "config.json")

	defer func() { os.Setenv(homeKey, homeVal) }()
	os.Setenv(homeKey, tmpDir)

	data := `{
		"HttpHeaders": { "MyHeader": "MyValue" }
	}`

	err = ioutil.WriteFile(tmpCfg, []byte(data), 0600)
	c.Assert(err, checker.IsNil)

	result := icmd.RunCommand(dockerBinary, "-H="+server.URL[7:], "ps")
	result.Assert(c, icmd.Expected{
		ExitCode: 1,
		Error:    "exit status 1",
	})

	c.Assert(headers["User-Agent"], checker.NotNil, check.Commentf("Missing User-Agent"))

	c.Assert(headers["User-Agent"][0], checker.Equals, "Docker-Client/"+dockerversion.Version+" ("+runtime.GOOS+")", check.Commentf("Badly formatted User-Agent,out:%v", result.Combined()))

	c.Assert(headers["Myheader"], checker.NotNil)
	c.Assert(headers["Myheader"][0], checker.Equals, "MyValue", check.Commentf("Missing/bad header,out:%v", result.Combined()))

}

// TestDaemonShutdownWithPlugins shuts down running plugins.
func (s *DockerDaemonSuite) TestDaemonShutdownWithPlugins(c *check.C) {
	testRequires(c, IsAmd64, Network, SameHostDaemon)

	s.d.Start(c)
	if out, err := s.d.Cmd("plugin", "install", "--grant-all-permissions", pName); err != nil {
		c.Fatalf("Could not install plugin: %v %s", err, out)
	}

	defer func() {
		s.d.Restart(c)
		if out, err := s.d.Cmd("plugin", "disable", pName); err != nil {
			c.Fatalf("Could not disable plugin: %v %s", err, out)
		}
		if out, err := s.d.Cmd("plugin", "remove", pName); err != nil {
			c.Fatalf("Could not remove plugin: %v %s", err, out)
		}
	}()

	if err := s.d.Interrupt(); err != nil {
		c.Fatalf("Could not kill daemon: %v", err)
	}

	for {
		if err := syscall.Kill(s.d.Pid(), 0); err == syscall.ESRCH {
			break
		}
	}

	icmd.RunCommand("pgrep", "-f", pluginProcessName).Assert(c, icmd.Expected{
		ExitCode: 1,
		Error:    "exit status 1",
	})

	s.d.Start(c, "--live-restore")
	icmd.RunCommand("pgrep", "-f", pluginProcessName).Assert(c, icmd.Success)
}

// Test case for #23716
func (s *DockerSuite) TestStartAttachWithRename(c *check.C) {
	testRequires(c, DaemonIsLinux)
	dockerCmd(c, "create", "-t", "--name", "before", "busybox")
	go func() {
		c.Assert(waitRun("before"), checker.IsNil)
		dockerCmd(c, "rename", "before", "after")
		dockerCmd(c, "stop", "--time=2", "after")
	}()
	// FIXME(vdemeester) the intent is not clear and potentially racey
	result := icmd.RunCommand(dockerBinary, "start", "-a", "before")
	result.Assert(c, icmd.Expected{
		ExitCode: 137,
		Error:    "exit status 137",
	})
	c.Assert(result.Stderr(), checker.Not(checker.Contains), "No such container")
}

// testConcurrentPullWholeRepo pulls the same repo concurrently.
func testConcurrentPullWholeRepo(c *check.C) {
	repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL)

	repos := []string{}
	for _, tag := range []string{"recent", "fresh", "todays"} {
		repo := fmt.Sprintf("%v:%v", repoName, tag)
		_, err := buildImage(repo, fmt.Sprintf(`
		    FROM busybox
		    ENTRYPOINT ["/bin/echo"]
		    ENV FOO foo
		    ENV BAR bar
		    CMD echo %s
		`, repo), true)
		c.Assert(err, checker.IsNil)
		dockerCmd(c, "push", repo)
		repos = append(repos, repo)
	}

	// Clear local images store.
	args := append([]string{"rmi"}, repos...)
	dockerCmd(c, args...)

	// Run multiple re-pulls concurrently
	results := make(chan error)
	numPulls := 3

	for i := 0; i != numPulls; i++ {
		go func() {
			result := icmd.RunCommand(dockerBinary, "pull", "-a", repoName)
			results <- result.Error
		}()
	}

	// These checks are separate from the loop above because the check
	// package is not goroutine-safe.
	for i := 0; i != numPulls; i++ {
		err := <-results
		c.Assert(err, checker.IsNil, check.Commentf("concurrent pull failed with error: %v", err))
	}

	// Ensure all tags were pulled successfully
	for _, repo := range repos {
		dockerCmd(c, "inspect", repo)
		out, _ := dockerCmd(c, "run", "--rm", repo)
		c.Assert(strings.TrimSpace(out), checker.Equals, "/bin/sh -c echo "+repo)
	}
}

// testConcurrentPush pushes multiple tags to the same repo
// concurrently.
func testConcurrentPush(c *check.C) {
	repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL)

	repos := []string{}
	for _, tag := range []string{"push1", "push2", "push3"} {
		repo := fmt.Sprintf("%v:%v", repoName, tag)
		_, err := buildImage(repo, fmt.Sprintf(`
	FROM busybox
	ENTRYPOINT ["/bin/echo"]
	ENV FOO foo
	ENV BAR bar
	CMD echo %s
`, repo), true)
		c.Assert(err, checker.IsNil)
		repos = append(repos, repo)
	}

	// Push tags, in parallel
	results := make(chan error)

	for _, repo := range repos {
		go func(repo string) {
			result := icmd.RunCommand(dockerBinary, "push", repo)
			results <- result.Error
		}(repo)
	}

	for range repos {
		err := <-results
		c.Assert(err, checker.IsNil, check.Commentf("concurrent push failed with error: %v", err))
	}

	// Clear local images store.
	args := append([]string{"rmi"}, repos...)
	dockerCmd(c, args...)

	// Re-pull and run individual tags, to make sure pushes succeeded
	for _, repo := range repos {
		dockerCmd(c, "pull", repo)
		dockerCmd(c, "inspect", repo)
		out, _ := dockerCmd(c, "run", "--rm", repo)
		c.Assert(strings.TrimSpace(out), checker.Equals, "/bin/sh -c echo "+repo)
	}
}

// Used to test output flag in the export command
func (s *DockerSuite) TestExportContainerWithOutputAndImportImage(c *check.C) {
	testRequires(c, DaemonIsLinux)
	containerID := "testexportcontainerwithoutputandimportimage"

	dockerCmd(c, "run", "--name", containerID, "busybox", "true")
	dockerCmd(c, "export", "--output=testexp.tar", containerID)
	defer os.Remove("testexp.tar")

	resultCat := icmd.RunCommand("cat", "testexp.tar")
	resultCat.Assert(c, icmd.Success)

	result := icmd.RunCmd(icmd.Cmd{
		Command: []string{dockerBinary, "import", "-", "repo/testexp:v1"},
		Stdin:   strings.NewReader(resultCat.Combined()),
	})
	result.Assert(c, icmd.Success)

	cleanedImageID := strings.TrimSpace(result.Combined())
	c.Assert(cleanedImageID, checker.Not(checker.Equals), "", check.Commentf("output should have been an image id"))
}

func (s *DockerSuite) TestVolumeCLIRmForce(c *check.C) {
	testRequires(c, SameHostDaemon, DaemonIsLinux)

	name := "test"
	out, _ := dockerCmd(c, "volume", "create", name)
	id := strings.TrimSpace(out)
	c.Assert(id, checker.Equals, name)

	out, _ = dockerCmd(c, "volume", "inspect", "--format", "{{.Mountpoint}}", name)
	c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "")
	// Mountpoint is in the form of "/var/lib/docker/volumes/.../_data", removing `/_data`
	path := strings.TrimSuffix(strings.TrimSpace(out), "/_data")
	icmd.RunCommand("rm", "-rf", path).Assert(c, icmd.Success)

	dockerCmd(c, "volume", "rm", "-f", "test")
	out, _ = dockerCmd(c, "volume", "ls")
	c.Assert(out, checker.Not(checker.Contains), name)
	dockerCmd(c, "volume", "create", "test")
	out, _ = dockerCmd(c, "volume", "ls")
	c.Assert(out, checker.Contains, name)
}

// testConcurrentFailingPull tries a concurrent pull that doesn't succeed.
func testConcurrentFailingPull(c *check.C) {
	repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL)

	// Run multiple pulls concurrently
	results := make(chan error)
	numPulls := 3

	for i := 0; i != numPulls; i++ {
		go func() {
			result := icmd.RunCommand(dockerBinary, "pull", repoName+":asdfasdf")
			results <- result.Error
		}()
	}

	// These checks are separate from the loop above because the check
	// package is not goroutine-safe.
	for i := 0; i != numPulls; i++ {
		err := <-results
		c.Assert(err, checker.NotNil, check.Commentf("expected pull to fail"))
	}
}

func (s *DockerSuite) TestEventsImageLoad(c *check.C) {
	testRequires(c, DaemonIsLinux)
	myImageName := "footest:v1"
	dockerCmd(c, "tag", "busybox", myImageName)
	since := daemonUnixTime(c)

	out, _ := dockerCmd(c, "images", "-q", "--no-trunc", myImageName)
	longImageID := strings.TrimSpace(out)
	c.Assert(longImageID, checker.Not(check.Equals), "", check.Commentf("Id should not be empty"))

	dockerCmd(c, "save", "-o", "saveimg.tar", myImageName)
	dockerCmd(c, "rmi", myImageName)
	out, _ = dockerCmd(c, "images", "-q", myImageName)
	noImageID := strings.TrimSpace(out)
	c.Assert(noImageID, checker.Equals, "", check.Commentf("Should not have any image"))
	dockerCmd(c, "load", "-i", "saveimg.tar")

	result := icmd.RunCommand("rm", "-rf", "saveimg.tar")
	c.Assert(result, icmd.Matches, icmd.Success)

	out, _ = dockerCmd(c, "images", "-q", "--no-trunc", myImageName)
	imageID := strings.TrimSpace(out)
	c.Assert(imageID, checker.Equals, longImageID, check.Commentf("Should have same image id as before"))

	out, _ = dockerCmd(c, "events", "--since", since, "--until", daemonUnixTime(c), "--filter", "event=load")
	events := strings.Split(strings.TrimSpace(out), "\n")
	c.Assert(events, checker.HasLen, 1)
	matches := eventstestutils.ScanMap(events[0])
	c.Assert(matches["id"], checker.Equals, imageID, check.Commentf("matches: %v\nout:\n%s\n", matches, out))
	c.Assert(matches["action"], checker.Equals, "load", check.Commentf("matches: %v\nout:\n%s\n", matches, out))

	out, _ = dockerCmd(c, "events", "--since", since, "--until", daemonUnixTime(c), "--filter", "event=save")
	events = strings.Split(strings.TrimSpace(out), "\n")
	c.Assert(events, checker.HasLen, 1)
	matches = eventstestutils.ScanMap(events[0])
	c.Assert(matches["id"], checker.Equals, imageID, check.Commentf("matches: %v\nout:\n%s\n", matches, out))
	c.Assert(matches["action"], checker.Equals, "save", check.Commentf("matches: %v\nout:\n%s\n", matches, out))
}