func TestRawResponseData(t *testing.T) {
	// nodeGetHandler is defined in dpn_rest_client_test.go
	testServer := httptest.NewServer(http.HandlerFunc(nodeGetHandler))
	defer testServer.Close()

	// configFile is defined in dpn_rest_client_test.go
	config, err := models.LoadConfigFile(configFile)
	require.Nil(t, err)
	client, err := network.NewDPNRestClient(
		testServer.URL,
		"",
		"",
		config.DPN.LocalNode,
		config.DPN)
	if err != nil {
		t.Error(err)
		return
	}
	resp := client.NodeGet("luna")

	// Should be able to call repeatedly without error.
	// Incorrect implementation would try to read from
	// closed network socket.
	for i := 0; i < 3; i++ {
		bytes, err := resp.RawResponseData()
		assert.NotNil(t, bytes)
		assert.NotEmpty(t, bytes)
		assert.Nil(t, err)
	}
}

func main() {
	pathToConfigFile := parseCommandLine()
	config, err := models.LoadConfigFile(pathToConfigFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	_context := context.NewContext(config)
	_context.MessageLog.Info("Connecting to NSQLookupd at %s", _context.Config.NsqLookupd)
	_context.MessageLog.Info("NSQDHttpAddress is %s", _context.Config.NsqdHttpAddress)
	consumer, err := apt_workers.CreateNsqConsumer(_context.Config, &_context.Config.DPN.DPNIngestStoreWorker)
	if err != nil {
		_context.MessageLog.Fatalf(err.Error())
	}
	storer, err := workers.NewDPNIngestStorer(_context)
	if err != nil {
		_context.MessageLog.Error(err.Error())
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	_context.MessageLog.Info("dpn_ingest_store started")
	consumer.AddHandler(storer)
	consumer.ConnectToNSQLookupd(_context.Config.NsqLookupd)

	// This reader blocks until we get an interrupt, so our program does not exit.
	<-consumer.StopChan
}

func TestEnsurePharosConfig(t *testing.T) {
	configFile := filepath.Join("config", "test.json")
	config, err := models.LoadConfigFile(configFile)
	require.Nil(t, err)

	url := config.PharosURL
	config.PharosURL = ""
	err = config.EnsurePharosConfig()
	assert.Equal(t, "PharosUrl is missing from config file", err.Error())

	config.PharosURL = url
	apiUser := os.Getenv("PHAROS_API_USER")
	os.Setenv("PHAROS_API_USER", "")
	err = config.EnsurePharosConfig()
	assert.Equal(t, "Environment variable PHAROS_API_USER is not set", err.Error())

	os.Setenv("PHAROS_API_USER", "Bogus value set by config_test.go")
	apiKey := os.Getenv("PHAROS_API_KEY")
	os.Setenv("PHAROS_API_KEY", "")
	err = config.EnsurePharosConfig()
	assert.Equal(t, "Environment variable PHAROS_API_KEY is not set", err.Error())

	os.Setenv("PHAROS_API_USER", apiUser)
	os.Setenv("PHAROS_API_KEY", apiKey)
}

// GetContext returns a context object initialized with the specified
// config file. Param configFile should be the name of a JSON config
// file in the exchange/config directory. For example, "test.json"
// or "integration.json".
func GetContext(configFile string) (*context.Context, error) {
	configPath := filepath.Join("config", configFile)
	config, err := models.LoadConfigFile(configPath)
	if err != nil {
		return nil, err
	}
	config.ExpandFilePaths()
	return context.NewContext(config), nil
}

func TestGetRemoteClient(t *testing.T) {
	if runRestTests(t) == false {
		return
	}
	config, err := apt_models.LoadConfigFile(configFile)
	require.Nil(t, err)
	client := getClient(t)
	nodes := []string{"chron", "hathi", "sdr", "tdr"}
	for _, node := range nodes {
		_, err := client.GetRemoteClient(node, config.DPN)
		assert.Nil(t, err)
	}
}

func getPostTestClient(t *testing.T) *network.DPNRestClient {
	// If you want to debug, change ioutil.Discard to os.Stdout
	// to see log output from the client.
	config, err := apt_models.LoadConfigFile(filepath.Join("config", "integration.json"))
	require.Nil(t, err)
	client, err := network.NewDPNRestClient(
		config.DPN.RestClient.LocalServiceURL,
		config.DPN.RestClient.LocalAPIRoot,
		config.DPN.RestClient.LocalAuthToken,
		config.DPN.LocalNode,
		config.DPN)
	require.Nil(t, err)
	return client
}

func main() {
	pathToConfigFile := parseCommandLine()
	config, err := models.LoadConfigFile(pathToConfigFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	_context := context.NewContext(config)
	dpnSync, err := workers.NewDPNSync(_context)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	dpnSync.Run()
}

func createBagBuilder(t *testing.T) (builder *util.BagBuilder) {
	obj := intelObj(t)
	configFile := filepath.Join("config", "test.json")
	config, err := models.LoadConfigFile(configFile)
	require.Nil(t, err)

	builder, err = util.NewBagBuilder(testBagPath(), obj, config.DPN.DefaultMetadata)
	if err != nil {
		tearDown()
		t.Errorf("Could not create bag builder: %s", err.Error())
		return nil
	}
	builder.Bag.Save()
	return builder
}

func TestLoad(t *testing.T) {
	configFile := filepath.Join("config", "test.json")
	config, err := models.LoadConfigFile(configFile)
	require.Nil(t, err)

	// Spot check a few settings.
	assert.Equal(t, 60, config.MaxDaysSinceFixityCheck)
	assert.Equal(t, "http://localhost:3000", config.PharosURL)
	assert.Equal(t, "10s", config.FetchWorker.HeartbeatInterval)
	assert.Equal(t, 18, len(config.ReceivingBuckets))
	assert.Equal(t, configFile, config.ActiveConfig)
	assert.Equal(t, 24, config.BucketReaderCacheHours)
	assert.Equal(t, "api-v2", config.DPN.DPNAPIVersion)
	assert.Equal(t, "us-east-1", config.DPN.DPNGlacierRegion)
}

func runRestTests(t *testing.T) bool {
	config, err := apt_models.LoadConfigFile(configFile)
	require.Nil(t, err)
	_, err = http.Get(config.DPN.RestClient.LocalServiceURL)
	if err != nil {
		if skipRestMessagePrinted == false {
			skipRestMessagePrinted = true
			fmt.Printf("Skipping DPN REST integration tests: "+
				"DPN REST server is not running at %s\n",
				config.DPN.RestClient.LocalServiceURL)
		}
		return false
	}
	return true
}

// See printUsage for a description.
func main() {
	pathToConfigFile := parseCommandLine()
	config, err := models.LoadConfigFile(pathToConfigFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	_context := context.NewContext(config)
	_context.MessageLog.Info("apt_volume_service started")

	volumeService := service.NewVolumeService(
		_context.Config.VolumeServicePort,
		_context.MessageLog)
	volumeService.Serve()
}

func main() {
	pathToConfigFile, hours := parseCommandLine()
	config, err := models.LoadConfigFile(pathToConfigFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	_context := context.NewContext(config)
	dpnQueue, err := workers.NewDPNQueue(_context, hours)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	dpnQueue.Run()
}

func getClient(t *testing.T) *network.DPNRestClient {
	// If you want to debug, change ioutil.Discard to os.Stdout
	// to see log output from the client.
	config, err := apt_models.LoadConfigFile(configFile)
	require.Nil(t, err)
	client, err := network.NewDPNRestClient(
		config.DPN.RestClient.LocalServiceURL,
		config.DPN.RestClient.LocalAPIRoot,
		config.DPN.RestClient.LocalAuthToken,
		config.DPN.LocalNode,
		config.DPN)
	if err != nil {
		t.Errorf("Error constructing DPN REST client: %v", err)
	}
	return client
}

func TestBuildUrl(t *testing.T) {
	config, err := apt_models.LoadConfigFile(configFile)
	require.Nil(t, err)
	client := getClient(t)
	require.NotNil(t, client)
	relativeUrl := "/api-v1/popeye/olive/oyl/"
	expectedUrl := config.DPN.RestClient.LocalServiceURL + relativeUrl
	if client.BuildUrl(relativeUrl, nil) != expectedUrl {
		t.Errorf("BuildUrl returned '%s', expected '%s'",
			client.BuildUrl(relativeUrl, nil), expectedUrl)
	}
	params := url.Values{}
	params.Set("color", "blue")
	params.Set("material", "cotton")
	params.Set("size", "extra medium")
	actualUrl := client.BuildUrl(relativeUrl, params)
	expectedUrl = expectedUrl + "?color=blue&material=cotton&size=extra+medium"
	assert.Equal(t, expectedUrl, actualUrl)
}

// test_push_to_dpn is for integration testing only.
// It creates a few WorkItems in Pharos asking that
// a handful of bags be pushed to DPN. Subsequent
// integration tests (such as dpn_queue) depend on
// the WorkItems created by this test.
func TestPushToDPN(t *testing.T) {
	if !testutil.ShouldRunIntegrationTests() {
		t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
	}
	configFile := filepath.Join("config", "integration.json")
	config, err := models.LoadConfigFile(configFile)
	require.Nil(t, err)
	_context := context.NewContext(config)
	for _, s3Key := range testutil.INTEGRATION_GOOD_BAGS[0:7] {
		identifier := strings.Replace(s3Key, "aptrust.receiving.test.", "", 1)
		identifier = strings.Replace(identifier, ".tar", "", 1)
		resp := _context.PharosClient.IntellectualObjectPushToDPN(identifier)
		workItem := resp.WorkItem()
		require.Nil(t, resp.Error)
		require.NotNil(t, workItem)
		_context.MessageLog.Info("Created DPN work item #%d for %s",
			workItem.Id, workItem.ObjectIdentifier)
	}
}

func TestFetchResults(t *testing.T) {
	if !testutil.ShouldRunIntegrationTests() {
		t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
	}
	// Load config
	configFile := filepath.Join("config", "integration.json")
	config, err := models.LoadConfigFile(configFile)
	require.Nil(t, err)
	config.ExpandFilePaths()

	// Find the log file that apt_fetch created when it was running
	// with the "config/integration.json" config options. We'll read
	// that file.
	pathToJsonLog := filepath.Join(config.LogDirectory, "apt_fetch.json")
	for _, bagName := range testutil.INTEGRATION_GOOD_BAGS {
		ingestManifest, err := testutil.FindIngestManifestInLog(pathToJsonLog, bagName)
		assert.Nil(t, err)
		if err != nil {
			continue
		}
		fetcherTestCommon(t, bagName, ingestManifest)
		// TODO: Test WorkItem (stage, status, etc.) below
		fetcherTestGoodBagResult(t, bagName, ingestManifest)
		if bagName == "aptrust.receiving.test.test.edu/example.edu.tagsample_good.tar" {
			fetcherTestSpecifics(t, ingestManifest)
		}
	}
	for _, bagName := range testutil.INTEGRATION_BAD_BAGS {
		ingestManifest, err := testutil.FindIngestManifestInLog(pathToJsonLog, bagName)
		assert.Nil(t, err)
		if err != nil {
			continue
		}
		fetcherTestCommon(t, bagName, ingestManifest)
		// TODO: Test WorkItem (stage, status, etc.) below
		fetcherTestBadBagResult(t, bagName, ingestManifest)
	}
}

func TestEnsureDPNConfig(t *testing.T) {
	configFile := filepath.Join("config", "test.json")
	config, err := models.LoadConfigFile(configFile)
	require.Nil(t, err)

	assert.NotEmpty(t, config.DPN.DefaultMetadata.BagItVersion)
	assert.NotEmpty(t, config.DPN.DefaultMetadata.BagItEncoding)
	assert.NotEmpty(t, config.DPN.DefaultMetadata.IngestNodeName)
	assert.NotEmpty(t, config.DPN.DefaultMetadata.IngestNodeAddress)
	assert.NotEmpty(t, config.DPN.DefaultMetadata.IngestNodeContactName)
	assert.NotEmpty(t, config.DPN.DefaultMetadata.IngestNodeContactEmail)

	assert.NotEmpty(t, config.DPN.RestClient.LocalServiceURL)
	assert.NotEmpty(t, config.DPN.RestClient.LocalAPIRoot)
	assert.NotEmpty(t, config.DPN.RestClient.LocalAuthToken)

	assert.EqualValues(t, 3, config.DPN.DPNValidationWorker.MaxAttempts)
	assert.EqualValues(t, 3, config.DPN.DPNCopyWorker.MaxAttempts)

	assert.Equal(t, "chron_token", config.DPN.RemoteNodeAdminTokensForTesting["chron"])
	assert.Equal(t, "aptrust_token", config.DPN.RemoteNodeTokens["chron"])
	assert.Equal(t, "http://localhost:3002", config.DPN.RemoteNodeURLs["chron"])
}

// Returns two Stats objects: the expected stats, from our test data dir,
// and the actual stats, from the JSON file that the bucket reader dumped
// out last time it ran
func getBucketReaderOutputs(t *testing.T) (expected *stats.APTBucketReaderStats, actual *stats.APTBucketReaderStats) {
	configFile := filepath.Join("config", "integration.json")
	appConfig, err := models.LoadConfigFile(configFile)
	require.Nil(t, err)

	// This JSON file is part of our code repo.
	// It contains the output we expect from a success test run.
	pathToExpectedJson, err := fileutil.RelativeToAbsPath(filepath.Join("testdata",
		"integration_results", "bucket_reader_stats.json"))
	require.Nil(t, err)
	expected, err = stats.APTBucketReaderStatsLoadFromFile(pathToExpectedJson)
	require.Nil(t, err)

	// This JSON file is recreated every time we run apt_bucket_reader
	// as part of the integration tests. It contains the output of the
	// actual test run.
	pathToActualJson, err := fileutil.ExpandTilde(filepath.Join(appConfig.LogDirectory, "bucket_reader_stats.json"))
	require.Nil(t, err)
	actual, err = stats.APTBucketReaderStatsLoadFromFile(pathToActualJson)
	require.Nil(t, err)

	return expected, actual
}

func TestNewContext(t *testing.T) {
	configFile := filepath.Join("config", "test.json")
	appConfig, err := models.LoadConfigFile(configFile)
	require.Nil(t, err)

	// In some tests we want to log to STDERR, but in this case, if it
	// happens to be turned on, it just creates useless, annoying output.
	appConfig.LogToStderr = false

	_context := context.NewContext(appConfig)
	require.NotNil(t, _context)

	expectedPathToLogFile := filepath.Join(_context.Config.AbsLogDirectory(), path.Base(os.Args[0])+".log")
	expectedPathToJsonLog := filepath.Join(_context.Config.AbsLogDirectory(), path.Base(os.Args[0])+".json")

	assert.NotNil(t, _context.Config)
	assert.NotNil(t, _context.NSQClient)
	assert.NotNil(t, _context.PharosClient)
	assert.NotNil(t, _context.MessageLog)
	assert.NotNil(t, _context.JsonLog)
	assert.Equal(t, expectedPathToLogFile, _context.PathToLogFile())
	assert.Equal(t, expectedPathToJsonLog, _context.PathToJsonLog())
	assert.Equal(t, int64(0), _context.Succeeded())
	assert.Equal(t, int64(0), _context.Failed())

	assert.NotPanics(t, func() { _context.MessageLog.Info("Test INFO log message") })
	assert.NotPanics(t, func() { _context.MessageLog.Debug("Test DEBUG log message") })
	assert.NotPanics(t, func() { _context.JsonLog.Println(`{"message": "Test JSON log message"}`) })

	// Cleanup, but only if context was successfully created
	if _context != nil && _context.PathToLogFile() != "" {
		os.Remove(_context.PathToLogFile())
	}
	if _context != nil && _context.PathToJsonLog() != "" {
		os.Remove(_context.PathToJsonLog())
	}
}

func TestStoreResults(t *testing.T) {
	if !testutil.ShouldRunIntegrationTests() {
		t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
	}
	// Load config
	configFile := filepath.Join("config", "integration.json")
	config, err := models.LoadConfigFile(configFile)
	require.Nil(t, err)
	config.ExpandFilePaths()

	// Find the log file that apt_store created when it was running
	// with the "config/integration.json" config options. We'll read
	// that file.
	pathToJsonLog := filepath.Join(config.LogDirectory, "apt_store.json")
	for _, bagName := range testutil.INTEGRATION_GOOD_BAGS {
		ingestManifest, err := testutil.FindIngestManifestInLog(pathToJsonLog, bagName)
		assert.Nil(t, err)
		if err != nil {
			continue
		}
		// TODO: Test WorkItem (stage, status, etc.) below
		storeTestCommon(t, bagName, ingestManifest, config)
	}
}