// Perform a
func CbfsSanityCheck(config Configuration) error {

	uuid := NewUuid() // use uuid so other nodes on cluster don't conflict
	numAttempts := 20
	for i := 0; i < numAttempts; i++ {
		filename := fmt.Sprintf("env_check_%v_%v", uuid, i)
		content := fmt.Sprintf("Hello %v_%v", uuid, i)
		err := CbfsReadWriteFile(config, filename, content)
		if err == nil {
			logg.LogTo("ELASTIC_THOUGHT", "Cbfs sanity ok: %v", filename)
			return nil
		}
		logg.LogTo("ELASTIC_THOUGHT", "Cbfs sanity failed # %v: %v", i, filename)
		if i >= (numAttempts - 1) {
			logg.LogTo("ELASTIC_THOUGHT", "Cbfs sanity check giving up")
			return err
		} else {
			logg.LogTo("ELASTIC_THOUGHT", "Cbfs sanity check sleeping ..")
			time.Sleep(time.Duration(i) * time.Second)
			logg.LogTo("ELASTIC_THOUGHT", "Cbfs sanity check done sleeping")
		}
	}
	return fmt.Errorf("Exhausted attempts")

}

func (c *Checkerlution) CreateSensors() {

	sensorLayer := 0.0

	sensorFuncGameState := func(syncCounter int) []float64 {
		logg.LogTo("MAIN", "sensor func game state called")
		return c.currentGameState
	}
	sensorGameStateNodeId := ng.NewSensorId("SensorGameState", sensorLayer)
	sensorGameState := &ng.Sensor{
		NodeId:         sensorGameStateNodeId,
		VectorLength:   32,
		SensorFunction: sensorFuncGameState,
	}

	sensorFuncPossibleMove := func(syncCounter int) []float64 {
		logg.LogTo("MAIN", "sensor func possible move called")
		return c.currentPossibleMove.VectorRepresentation()
	}
	sensorPossibleMoveNodeId := ng.NewSensorId("SensorPossibleMove", sensorLayer)
	sensorPossibleMove := &ng.Sensor{
		NodeId:         sensorPossibleMoveNodeId,
		VectorLength:   5, // start_location, is_king, final_location, will_be_king, amt_would_capture
		SensorFunction: sensorFuncPossibleMove,
	}
	c.cortex.SetSensors([]*ng.Sensor{sensorGameState, sensorPossibleMove})

}

func TestUntarGzWithToc(t *testing.T) {

	// Create a test tar archive
	buf := new(bytes.Buffer)

	var files = []tarFile{
		{"foo/1.txt", "."},
		{"foo/2.txt", "."},
		{"bar/1.txt", "."},
		{"bar/2.txt", "."},
		{"bar/3.txt", "."},
		{"bar/4.txt", "."},
		{"bar/5.txt", "."},
	}
	createArchive(buf, files)
	reader := bytes.NewReader(buf.Bytes())

	tempDir := TempDir()
	logg.LogTo("TEST", "tempDir: %v", tempDir)
	toc, err := untarWithToc(reader, tempDir)
	assert.True(t, err == nil)

	logg.LogTo("TEST", "toc: %v, err: %v", toc, err)

	// TODO: add asserations

}

func (c OcrRpcClient) handleRpcResponse(deliveries <-chan amqp.Delivery, correlationUuid string, rpcResponseChan chan OcrResult) {
	logg.LogTo("OCR_CLIENT", "looping over deliveries..")
	for d := range deliveries {
		if d.CorrelationId == correlationUuid {
			logg.LogTo(
				"OCR_CLIENT",
				"got %dB delivery: [%v] %q.  Reply to: %v",
				len(d.Body),
				d.DeliveryTag,
				d.Body,
				d.ReplyTo,
			)

			ocrResult := OcrResult{
				Text: string(d.Body),
			}

			logg.LogTo("OCR_CLIENT", "send result to rpcResponseChan")
			rpcResponseChan <- ocrResult
			logg.LogTo("OCR_CLIENT", "sent result to rpcResponseChan")

			return

		} else {
			logg.LogTo("OCR_CLIENT", "ignoring delivery w/ correlation id: %v", d.CorrelationId)
		}

	}
}

// Follow changes feed.  This will typically be run in its own goroutine.
func (c ChangesListener) FollowChangesFeed() {

	logg.LogTo("CHANGES", "going to follow changes feed")

	var since interface{}

	handleChange := func(reader io.Reader) interface{} {
		logg.LogTo("CHANGES", "handleChange() callback called")
		changes, err := decodeChanges(reader)
		if err != nil {
			// it's very common for this to timeout while waiting for new changes.
			// since we want to follow the changes feed forever, just log an error
			logg.LogTo("CHANGES", "%T decoding changes: %v.", err, err)
			return since
		}
		c.processChanges(changes)

		since = changes.LastSequence
		logg.LogTo("CHANGES", "returning since: %v", since)
		return since

	}

	options := map[string]interface{}{}
	options["feed"] = "longpoll"

	logg.LogTo("CHANGES", "Following changes feed: %+v.", options)

	// this will block until the handleChange callback returns nil
	c.Database.Changes(handleChange, options)

	logg.LogPanic("Changes listener died -- this should never happen")

}

func (w *OcrRpcWorker) handle(deliveries <-chan amqp.Delivery, done chan error) {
	for d := range deliveries {
		logg.LogTo(
			"OCR_WORKER",
			"got %d byte delivery: [%v]. Routing key: %v  Reply to: %v",
			len(d.Body),
			d.DeliveryTag,
			d.RoutingKey,
			d.ReplyTo,
		)

		ocrResult, err := w.resultForDelivery(d)
		if err != nil {
			msg := "Error generating ocr result.  Error: %v"
			logg.LogError(fmt.Errorf(msg, err))
		}

		logg.LogTo("OCR_WORKER", "Sending rpc response: %v", ocrResult)
		err = w.sendRpcResponse(ocrResult, d.ReplyTo, d.CorrelationId)
		if err != nil {
			msg := "Error returning ocr result: %v.  Error: %v"
			logg.LogError(fmt.Errorf(msg, ocrResult, err))
			// if we can't send our response, let's just abort
			done <- err
			break
		}

	}
	logg.LogTo("OCR_WORKER", "handle: deliveries channel closed")
	done <- fmt.Errorf("handle: deliveries channel closed")
}

func TestChooseBestMove(t *testing.T) {

	ng.SeedRandom()
	logg.LogKeys["MAIN"] = true

	checkerlution := &Checkerlution{}
	checkerlution.ourTeamId = RED_TEAM

	checkerlution.CreateNeurgoCortex()
	cortex := checkerlution.cortex
	cortex.Run()

	gameState, possibleMoves := FakeGameDocument()
	bestMove := checkerlution.chooseBestMove(gameState, possibleMoves)
	logg.LogTo("TEST", "bestMove: %v", &bestMove)

	found := false
	for _, possibleMove := range possibleMoves {
		logg.LogTo("TEST", "possibleMove: %v", &possibleMove)
		if possibleMove.Equals(bestMove) {
			found = true
		}
	}
	assert.True(t, found)

	cortex.Shutdown()

}

func TopologyOrWeightMutator(cortex *ng.Cortex) (success bool, result MutateResult) {

	randomNumber := ng.RandomIntInRange(0, 100)
	didMutate := false
	var mutators []CortexMutator
	if randomNumber > 90 {
		mutators = []CortexMutator{MutateActivation}
	} else if randomNumber > 80 {
		mutators = []CortexMutator{MutateAllWeightsBellCurve}
	} else if randomNumber > 20 {
		// apply topological mutation
		includeNonTopological := false
		mutators = CortexMutatorsNonRecurrent(includeNonTopological)
	} else {
		mutators = CortexMutatorsNonTopological()
	}
	// before we mutate the cortex, we need to init it,
	// otherwise things like Outsplice will fail because
	// there are no DataChan's.
	cortex.Init()
	for i := 0; i <= 100; i++ {
		randInt := RandomIntInRange(0, len(mutators))
		mutator := mutators[randInt]
		didMutate, _ = mutator(cortex)
		if !didMutate {
			logg.LogTo("NEURVOLVE", "Mutate didn't work, retrying...")
			continue
		}
		break
	}
	logg.LogTo("NEURVOLVE", "did mutate: %v", didMutate)
	success = didMutate
	result = "nothing"
	return
}

// Run this job
func (j *TrainingJob) Run(wg *sync.WaitGroup) {

	defer wg.Done()

	logg.LogTo("TRAINING_JOB", "Run() called!")

	updatedState, err := j.UpdateProcessingState(Processing)
	if err != nil {
		j.recordProcessingError(err)
		return
	}

	if !updatedState {
		logg.LogTo("TRAINING_JOB", "%+v already processed.  Ignoring.", j)
		return
	}

	j.StdOutUrl = j.getStdOutCbfsUrl()
	j.StdErrUrl = j.getStdErrCbfsUrl()

	if err := j.extractData(); err != nil {
		j.recordProcessingError(err)
		return
	}

	if err := j.runCaffe(); err != nil {
		j.recordProcessingError(err)
		return
	}

	j.FinishedSuccessfully(j.Configuration.DbConnection(), "")

}

func (s *OcrHttpMultipartHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {

	defer req.Body.Close()

	ocrRequest, err := s.extractParts(req)
	if err != nil {
		logg.LogError(err)
		errStr := fmt.Sprintf("Error extracting multipart/related parts: %v", err)
		http.Error(w, errStr, 500)
		return
	}

	logg.LogTo("OCR_HTTP", "ocrRequest: %v", ocrRequest)

	ocrResult, err := HandleOcrRequest(ocrRequest, s.RabbitConfig)

	if err != nil {
		msg := "Unable to perform OCR decode.  Error: %v"
		errMsg := fmt.Sprintf(msg, err)
		logg.LogError(fmt.Errorf(errMsg))
		http.Error(w, errMsg, 500)
		return
	}

	logg.LogTo("OCR_HTTP", "ocrResult: %v", ocrResult)

	fmt.Fprintf(w, ocrResult.Text)

}

func RunPopulationTrainer(pt *nv.PopulationTrainer) bool {

	population := getInitialPopulation()
	scape := getScape()

	fitPopulation, succeeded := pt.Train(population, scape, nv.NewNullRecorder())

	if succeeded {
		logg.LogTo("MAIN", "Successfully trained!")

		fittestCortex := fitPopulation[0]
		logg.LogTo("MAIN", "Fitness: %v", fittestCortex.Fitness)

		filename := fmt.Sprintf("/tmp/checkerlution-%v.json", time.Now().Unix())
		logg.LogTo("MAIN", "Saving Cortex to %v", filename)
		cortex := fittestCortex.Cortex
		cortex.MarshalJSONToFile(filename)

		// verify it can now solve the training set
		verified := cortex.Verify(ng.XnorTrainingSamples())
		if !verified {
			logg.LogTo("MAIN", "Failed to verify neural net")
			succeeded = false
		}

	}

	if !succeeded {
		logg.LogTo("MAIN", "Failed to train neural net")
	}

	return succeeded

}

// Save a new version of Datafile to the db
func (d Datafile) Save(db couch.Database) (*Datafile, error) {

	idToRetrieve := ""

	switch d.HasValidId() {
	case true:
		logg.LogTo("MODEL", "calling db.Edit()")
		_, err := db.Edit(d)
		if err != nil {
			return nil, err
		}
		idToRetrieve = d.Id
	default:
		logg.LogTo("MODEL", "calling db.Insert()")
		id, _, err := db.Insert(d)
		if err != nil {
			return nil, err
		}
		idToRetrieve = id
	}

	// load latest version from db to get the _id and _rev fields
	datafile := &Datafile{}
	err := db.Retrieve(idToRetrieve, datafile)
	if err != nil {
		return nil, err
	}

	return datafile, nil

}

// Creates datasets from a datafile
func (e EndpointContext) CreateDataSetsEndpoint(c *gin.Context) {

	user := c.MustGet(MIDDLEWARE_KEY_USER).(User)
	db := c.MustGet(MIDDLEWARE_KEY_DB).(couch.Database)
	logg.LogTo("REST", "user: %v db: %v", user, db)

	dataset := NewDataset(e.Configuration)

	// bind the input struct to the JSON request
	if ok := c.Bind(dataset); !ok {
		errMsg := fmt.Sprintf("Invalid input")
		c.String(400, errMsg)
		return
	}

	logg.LogTo("REST", "dataset: %+v", dataset)

	// save dataset in db
	if err := dataset.Insert(); err != nil {
		c.String(500, err.Error())
		return
	}

	// the changes listener will see new datafile and download to cbfs

	// update with urls of training/testing artifacts (which don't exist yet)
	if err := dataset.AddArtifactUrls(); err != nil {
		errMsg := fmt.Sprintf("Error updating dataset: %+v.  Err: %v", dataset, err)
		c.String(500, errMsg)
		return
	}

	c.JSON(201, dataset)

}

func RunPopulationTrainerLoop(maxIterations int) bool {

	// create population trainer ...
	pt := &nv.PopulationTrainer{
		FitnessThreshold: ng.FITNESS_THRESHOLD,
		MaxGenerations:   1000,
		CortexMutator:    nv.MutateAllWeightsBellCurve,
		// CortexMutator: nv.MutateWeights,
		// CortexMutator: RandomNeuronMutator,
		// CortexMutator:       nv.TopologyOrWeightMutator,
		NumOpponents:        5,
		SnapshotRequestChan: make(chan chan nv.EvaluatedCortexes),
	}
	nv.RegisterHandlers(pt)

	for i := 0; i < maxIterations; i++ {
		succeeded := RunPopulationTrainer(pt)
		if !succeeded {
			logg.LogTo("MAIN", "Population trainer succeeded %d times and then failed this time", i)
			return false
		} else {
			logg.LogTo("MAIN", "Population trainer succeeded %d times", i)

		}
	}
	logg.LogTo("MAIN", "Population succeeded %d times", maxIterations)
	return true
}

func TestTesseractEngineWithJson(t *testing.T) {

	if testing.Short() {
		t.Skip("skipping test in short mode.")
	}

	testJsons := []string{}
	testJsons = append(testJsons, `{"engine":"tesseract"}`)
	testJsons = append(testJsons, `{"engine":"tesseract", "engine_args":{}}`)
	testJsons = append(testJsons, `{"engine":"tesseract", "engine_args":null}`)
	testJsons = append(testJsons, `{"engine":"tesseract", "engine_args":{"config_vars":{"tessedit_char_whitelist":"0123456789"}, "psm":"1"}}`)
	testJsons = append(testJsons, `{"engine":"tesseract", "engine_args":{"config_vars":{"tessedit_create_hocr":"1", "tessedit_pageseg_mode":"1"}, "psm":"3"}}`)

	for _, testJson := range testJsons {
		logg.LogTo("TEST", "testJson: %v", testJson)
		ocrRequest := OcrRequest{}
		err := json.Unmarshal([]byte(testJson), &ocrRequest)
		assert.True(t, err == nil)
		bytes, err := ioutil.ReadFile("docs/testimage.png")
		assert.True(t, err == nil)
		ocrRequest.ImgBytes = bytes
		engine := NewOcrEngine(ocrRequest.EngineType)
		result, err := engine.ProcessRequest(ocrRequest)
		logg.LogTo("TEST", "err: %v", err)
		assert.True(t, err == nil)
		logg.LogTo("TEST", "result: %v", result)

	}

}

func stressTest(doneChannel chan<- bool) {

	imageUrls := imageUrls()
	logg.LogTo("CLI", "imageUrls: %v", imageUrls)
	logg.LogTo("CLI", "numIterations: %v", *numIterations)

	openOcrUrl := *ocrUrl
	client := ocrclient.NewHttpClient(openOcrUrl)

	for i := 0; i < *numIterations; i++ {
		index := randomIntInRange(0, numTestImages)
		imageUrl := imageUrls[index]
		logg.LogTo("CLI", "OCR decoding: %v.  index: %d", imageUrl, index)

		ocrRequest := ocrclient.OcrRequest{
			ImgUrl:     imageUrl,
			EngineType: ocrclient.ENGINE_TESSERACT,
		}

		ocrDecoded, err := client.DecodeImageUrl(ocrRequest)
		if err != nil {
			logg.LogError(fmt.Errorf("Error decoding image: %v", err))
		} else {
			logg.LogTo("CLI", "OCR decoded: %v", ocrDecoded)
		}
	}

	doneChannel <- true

}

func (t TesseractEngine) ProcessRequest(ocrRequest OcrRequest) (OcrResult, error) {

	tmpFileName, err := func() (string, error) {
		if ocrRequest.ImgUrl != "" {
			return t.tmpFileFromImageUrl(ocrRequest.ImgUrl)
		} else {
			return t.tmpFileFromImageBytes(ocrRequest.ImgBytes)
		}

	}()

	if err != nil {
		logg.LogTo("OCR_TESSERACT", "error getting tmpFileName")
		return OcrResult{}, err
	}

	defer os.Remove(tmpFileName)

	engineArgs, err := NewTesseractEngineArgs(ocrRequest)
	if err != nil {
		logg.LogTo("OCR_TESSERACT", "error getting engineArgs")
		return OcrResult{}, err
	}

	ocrResult, err := t.processImageFile(tmpFileName, *engineArgs)

	return ocrResult, err

}

func (s *OcrHttpHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {

	logg.LogTo("OCR_HTTP", "serveHttp called")
	defer req.Body.Close()

	ocrRequest := OcrRequest{}
	decoder := json.NewDecoder(req.Body)
	err := decoder.Decode(&ocrRequest)
	if err != nil {
		logg.LogError(err)
		http.Error(w, "Unable to unmarshal json", 500)
		return
	}

	ocrResult, err := HandleOcrRequest(ocrRequest, s.RabbitConfig)

	if err != nil {
		msg := "Unable to perform OCR decode.  Error: %v"
		errMsg := fmt.Sprintf(msg, err)
		logg.LogError(fmt.Errorf(errMsg))
		http.Error(w, errMsg, 500)
		return
	}

	logg.LogTo("OCR_HTTP", "ocrResult: %v", ocrResult)

	fmt.Fprintf(w, ocrResult.Text)

}

func logWeights(neuron *Neuron) {
	for _, inboundConnection := range neuron.Inbound {
		logmsg := fmt.Sprintf("%v -> %v weights: %v", inboundConnection.NodeId.UUID, neuron.NodeId.UUID, inboundConnection.Weights)
		logg.LogTo("NODE_STATE", logmsg)
		logmsg = fmt.Sprintf("%v bias: %v", neuron.NodeId.UUID, neuron.Bias)
		logg.LogTo("NODE_STATE", logmsg)
	}
}

func confirmDelivery(ack, nack chan uint64) {
	select {
	case tag := <-ack:
		logg.LogTo("OCR_CLIENT", "confirmed delivery, tag: %v", tag)
	case tag := <-nack:
		logg.LogTo("OCR_CLIENT", "failed to confirm delivery: %v", tag)
	}
}