func SetupDataManagerTest(t *testing.T) {
	os.Setenv("ARVADOS_API_HOST_INSECURE", "true")

	// start api and keep servers
	arvadostest.ResetEnv()
	arvadostest.StartAPI()
	arvadostest.StartKeep(2, false)

	var err error
	arv, err = arvadosclient.MakeArvadosClient()
	if err != nil {
		t.Fatalf("Error making arvados client: %s", err)
	}
	arv.ApiToken = arvadostest.DataManagerToken

	// keep client
	keepClient = &keepclient.KeepClient{
		Arvados:       &arv,
		Want_replicas: 2,
		Client:        &http.Client{},
	}

	// discover keep services
	if err = keepClient.DiscoverKeepServers(); err != nil {
		t.Fatalf("Error discovering keep services: %s", err)
	}
	keepServers = []string{}
	for _, host := range keepClient.LocalRoots() {
		keepServers = append(keepServers, host)
	}
}

func (s *NoKeepServerSuite) SetUpSuite(c *C) {
	arvadostest.StartAPI()
	// We need API to have some keep services listed, but the
	// services themselves should be unresponsive.
	arvadostest.StartKeep(2, false)
	arvadostest.StopKeep(2)
}

func (s *ServerRequiredSuite) SetUpSuite(c *C) {
	if *no_server {
		c.Skip("Skipping tests that require server")
		return
	}
	arvadostest.StartAPI()
	arvadostest.StartKeep()
}

func (s *ServerNotRequiredSuite) TestSetupKeepClient_NoBlobSignatureTTL(c *C) {
	var srcConfig apiConfig
	srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
	srcConfig.APIToken = arvadostest.DataManagerToken
	srcConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
	arvadostest.StartKeep(2, false)

	_, ttl, err := setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, 0)
	c.Check(err, IsNil)
	c.Assert(ttl, Equals, blobSignatureTTL)
}

func (s *DoMainTestSuite) Test_doMain_WithNoSuchBlockHashFile(c *C) {
	config := setupConfigFile(c, "config")
	defer os.Remove(config)

	arvadostest.StartKeep(2, false)
	defer arvadostest.StopKeep(2)

	args := []string{"-config", config, "-block-hash-file", "no-such-file"}
	err := doMain(args)
	c.Assert(strings.Contains(err.Error(), "no such file or directory"), Equals, true)
}

func (s *DoMainTestSuite) Test_doMain_WithNoBlockHashFile(c *C) {
	config := setupConfigFile(c, "config")
	defer os.Remove(config)

	// Start keepservers.
	arvadostest.StartKeep(2, false)
	defer arvadostest.StopKeep(2)

	args := []string{"-config", config}
	err := doMain(args)
	c.Assert(strings.Contains(err.Error(), "block-hash-file not specified"), Equals, true)
}

func (s *IntegrationSuite) SetUpSuite(c *check.C) {
	arvadostest.StartAPI()
	arvadostest.StartKeep(2, true)

	arv, err := arvadosclient.MakeArvadosClient()
	c.Assert(err, check.Equals, nil)
	arv.ApiToken = arvadostest.ActiveToken
	kc, err := keepclient.MakeKeepClient(&arv)
	c.Assert(err, check.Equals, nil)
	kc.PutB([]byte("Hello world\n"))
	kc.PutB([]byte("foo"))
	kc.PutB([]byte("foobar"))
}

func (s *DoMainTestSuite) Test_doMainWithSrcAndDstConfig(c *C) {
	srcConfig := setupConfigFile(c, "src")
	dstConfig := setupConfigFile(c, "dst")
	args := []string{"-src", srcConfig.Name(), "-dst", dstConfig.Name()}
	os.Args = append(os.Args, args...)

	// Start keepservers. Since we are not doing any tweaking as
	// in setupRsync func, kcSrc and kcDst will be the same and no
	// actual copying to dst will happen, but that's ok.
	arvadostest.StartKeep(2, false)
	defer arvadostest.StopKeep(2)

	err := doMain()
	c.Check(err, IsNil)
}

func setupKeepBlockCheckWithTTL(c *C, enforcePermissions bool, keepServicesJSON string, ttl time.Duration) {
	var config apiConfig
	config.APIHost = os.Getenv("ARVADOS_API_HOST")
	config.APIToken = arvadostest.DataManagerToken
	config.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))

	// Start Keep servers
	arvadostest.StartKeep(2, enforcePermissions)

	// setup keepclients
	var err error
	kc, ttl, err = setupKeepClient(config, keepServicesJSON, ttl)
	c.Assert(ttl, Equals, blobSignatureTTL)
	c.Check(err, IsNil)
}

func SetupPullWorkerIntegrationTest(t *testing.T, testData PullWorkIntegrationTestData, wantData bool) PullRequest {
	os.Setenv("ARVADOS_API_HOST_INSECURE", "true")

	// start api and keep servers
	arvadostest.StartAPI()
	arvadostest.StartKeep()

	// make arvadosclient
	arv, err := arvadosclient.MakeArvadosClient()
	if err != nil {
		t.Error("Error creating arv")
	}

	// keep client
	keepClient = &keepclient.KeepClient{
		Arvados:       &arv,
		Want_replicas: 1,
		Using_proxy:   true,
		Client:        &http.Client{},
	}

	// discover keep services
	var servers []string
	if err := keepClient.DiscoverKeepServers(); err != nil {
		t.Error("Error discovering keep services")
	}
	for _, host := range keepClient.LocalRoots() {
		servers = append(servers, host)
	}

	// Put content if the test needs it
	if wantData {
		locator, _, err := keepClient.PutB([]byte(testData.Content))
		if err != nil {
			t.Errorf("Error putting test data in setup for %s %s %v", testData.Content, locator, err)
		}
		if locator == "" {
			t.Errorf("No locator found after putting test data")
		}
	}

	// Create pullRequest for the test
	pullRequest := PullRequest{
		Locator: testData.Locator,
		Servers: servers,
	}
	return pullRequest
}

func (s *DoMainTestSuite) Test_doMain(c *C) {
	// Start keepservers.
	arvadostest.StartKeep(2, false)
	defer arvadostest.StopKeep(2)

	config := setupConfigFile(c, "config")
	defer os.Remove(config)

	locatorFile := setupBlockHashFile(c, "block-hash", []string{TestHash, TestHash2})
	defer os.Remove(locatorFile)

	args := []string{"-config", config, "-block-hash-file", locatorFile, "-v"}
	err := doMain(args)
	c.Check(err, NotNil)
	c.Assert(err.Error(), Equals, "Block verification failed for 2 out of 2 blocks with matching prefix.")
	checkErrorLog(c, []string{TestHash, TestHash2}, "Error verifying block", "Block not found")
	c.Assert(strings.Contains(logBuffer.String(), "Verifying block 1 of 2"), Equals, true)
}

func (s *integrationSuite) SetUpSuite(c *check.C) {
	if testing.Short() {
		c.Skip("-short")
	}
	arvadostest.ResetEnv()
	arvadostest.StartAPI()
	arvadostest.StartKeep(4, true)

	arv, err := arvadosclient.MakeArvadosClient()
	arv.ApiToken = arvadostest.DataManagerToken
	c.Assert(err, check.IsNil)
	s.keepClient = &keepclient.KeepClient{
		Arvados: &arv,
		Client:  &http.Client{},
	}
	c.Assert(s.keepClient.DiscoverKeepServers(), check.IsNil)
	s.putReplicas(c, "foo", 4)
	s.putReplicas(c, "bar", 1)
}

// Testing keep-rsync needs two sets of keep services: src and dst.
// The test setup hence creates 3 servers instead of the default 2,
// and uses the first 2 as src and the 3rd as dst keep servers.
func setupRsync(c *C, enforcePermissions bool, replications int) {
	// srcConfig
	var srcConfig apiConfig
	srcConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
	srcConfig.APIToken = arvadostest.DataManagerToken
	srcConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))

	// dstConfig
	var dstConfig apiConfig
	dstConfig.APIHost = os.Getenv("ARVADOS_API_HOST")
	dstConfig.APIToken = arvadostest.DataManagerToken
	dstConfig.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))

	if enforcePermissions {
		blobSigningKey = arvadostest.BlobSigningKey
	}

	// Start Keep servers
	arvadostest.StartKeep(3, enforcePermissions)

	// setup keepclients
	var err error
	kcSrc, _, err = setupKeepClient(srcConfig, srcKeepServicesJSON, false, 0, blobSignatureTTL)
	c.Check(err, IsNil)

	kcDst, _, err = setupKeepClient(dstConfig, dstKeepServicesJSON, true, replications, 0)
	c.Check(err, IsNil)

	for uuid := range kcSrc.LocalRoots() {
		if strings.HasSuffix(uuid, "02") {
			delete(kcSrc.LocalRoots(), uuid)
		}
	}
	for uuid := range kcSrc.GatewayRoots() {
		if strings.HasSuffix(uuid, "02") {
			delete(kcSrc.GatewayRoots(), uuid)
		}
	}
	for uuid := range kcSrc.WritableLocalRoots() {
		if strings.HasSuffix(uuid, "02") {
			delete(kcSrc.WritableLocalRoots(), uuid)
		}
	}

	for uuid := range kcDst.LocalRoots() {
		if strings.HasSuffix(uuid, "00") || strings.HasSuffix(uuid, "01") {
			delete(kcDst.LocalRoots(), uuid)
		}
	}
	for uuid := range kcDst.GatewayRoots() {
		if strings.HasSuffix(uuid, "00") || strings.HasSuffix(uuid, "01") {
			delete(kcDst.GatewayRoots(), uuid)
		}
	}
	for uuid := range kcDst.WritableLocalRoots() {
		if strings.HasSuffix(uuid, "00") || strings.HasSuffix(uuid, "01") {
			delete(kcDst.WritableLocalRoots(), uuid)
		}
	}

	if replications == 0 {
		// Must have got default replications value of 2 from dst discovery document
		c.Assert(kcDst.Want_replicas, Equals, 2)
	} else {
		// Since replications value is provided, it is used
		c.Assert(kcDst.Want_replicas, Equals, replications)
	}
}