func (um *Model) GetRegisteredUsers() (users map[string]*User) {
	if rows, err := um.table.Scan(nil); err == nil {
		users = make(map[string]*User)
		for _, row := range rows {
			uid := row["uid"].Value
			user := &User{
				uid:      uid,
				key:      row["key"].Value,
				Enabled:  row["enabled"].Value,
				logs:     make(map[string][]*LogLine),
				billHist: []*Billing{},
				md:       um,
			}
			if err := json.Unmarshal([]byte(row["info"].Value), &user); err != nil {
				log.Error("Problem trying to retieve the user information for user:", user.uid, "Error:", err)
				return nil
			}
			if err = json.Unmarshal([]byte(row["logs"].Value), &user.logs); err != nil {
				log.Error("Problem trying to unmarshal the user logs for user:", user.uid, "Error:", err)
				return nil
			}
			if err = json.Unmarshal([]byte(row["bill_hist"].Value), &user.billHist); err != nil {
				log.Error("Problem trying to unmarshal the billing history for user:", user.uid, "Error:", err)
				return nil
			}
			users[uid] = user
		}
	}

	return
}

func (bi *Billing) getToken() {
	client := &http.Client{}

	req, err := http.NewRequest("POST", bi.baseUri+"oauth2/token", strings.NewReader("grant_type=client_credentials"))
	req.Header.Add("Accept", `application/json`)
	req.Header.Add("Accept-Language", `en_US`)
	req.SetBasicAuth(bi.clientId, bi.secret)
	resp, err := client.Do(req)
	if err != nil {
		log.Error("Problem trying to request a new token from the PayPal API, Error:", err)
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Error("Problem trying to parse the response form the PayPal API, Error:", err)
	}

	responseInfo := make(map[string]interface{})
	err = json.Unmarshal(body, &responseInfo)
	if err != nil {
		log.Error("Problem trying to read token from PayPal API, Error:", err)
	}

	bi.token = responseInfo["access_token"].(string)
	bi.tokenExpireTs = time.Now().Unix() + int64(responseInfo["expires_in"].(float64))
}

func GetTrainerCorrelations(trainingFile string, TimeRangeToStudySecs int64) TrainerInt {
	log.Debug("Initializing trainer...")

	TimeRangeToStudySecs *= tsMultToSecs
	feedsFile, err := os.Open(trainingFile)
	log.Debug("File:", trainingFile)
	if err != nil {
		log.Fatal("Problem reading the logs file")
	}

	scanner := bufio.NewScanner(feedsFile)

	feeds := &TrainerCorrelations{
		feeds:                 make(map[string][]*charont.CurrVal),
		centroidsCurr:         make(map[string][][]float64),
		centroidsCurrSell:     make(map[string][][]float64),
		centroidsForAsk:       make(map[string][]int),
		centroidsForSell:      make(map[string][]int),
		maxWinByCentroid:      make(map[string][]float64),
		maxLossByCentroid:     make(map[string][]float64),
		maxWinByCentroidSell:  make(map[string][]float64),
		maxLossByCentroidSell: make(map[string][]float64),
	}

	i := 0
	for {
		var feed *charont.CurrVal

		if !scanner.Scan() {
			break
		}
		lineParts := strings.SplitN(scanner.Text(), ":", 2)
		curr := lineParts[0]
		if len(lineParts) < 2 {
			log.Error("The line:", i, "can't be parsed")
			continue
		}
		if err := json.Unmarshal([]byte(lineParts[1]), &feed); err != nil {
			log.Error("The feeds response body is not a JSON valid, Error:", err, "Line:", i)
			continue
		}

		if _, ok := feeds.feeds[curr]; !ok {
			feeds.feeds[curr] = []*charont.CurrVal{}
		}
		feeds.feeds[curr] = append(feeds.feeds[curr], feed)
		i++
		if i%10000 == 0 {
			log.Debug("Lines:", i)
		}
	}

	for curr, scores := range feeds.feeds {
		log.Debug("Curr:", curr, "Scores:", len(scores))
	}

	feeds.studyCurrencies(TimeRangeToStudySecs)

	return feeds
}

func (im *Model) updateInstances() {
	if rows, err := im.table.Scan(nil); err == nil {
		instances := []string{}
		for _, row := range rows {
			lastTs, _ := strconv.ParseInt(row["ts"].Value, 10, 64)
			if lastTs, _ = strconv.ParseInt(row["ts"].Value, 10, 64); lastTs+cTTL > time.Now().Unix() {
				instances = append(instances, row[cPrimKey].Value)
			} else if row[cPrimKey].Value != hostName {
				log.Info("Outdated instance detected, removing it, name:", row[cPrimKey].Value)
				attKey := &dynamodb.Key{
					HashKey:  row[cPrimKey].Value,
					RangeKey: "",
				}

				_, err = im.table.DeleteItem(attKey)
				if err != nil {
					log.Error("The instance:", row[cPrimKey].Value, "can't be removed, Error:", err)
				}
			}
		}

		sort.Sort(byName(instances))
		im.mutex.Lock()
		im.instancesAlive = instances
		im.mutex.Unlock()
	} else {
		log.Error("Problem trying to get the list of instances from Dynamo DB, Error:", err)
	}
}

func (um *Model) delTable() {
	if tableDesc, err := um.conn.DescribeTable(um.tableName); err == nil {
		if _, err = um.conn.DeleteTable(*tableDesc); err != nil {
			log.Error("Can't remove Dynamo table:", um.tableName, "Error:", err)
		}
	} else {
		log.Error("Can't remove Dynamo table:", um.tableName, "Error:", err)
	}
}

func (rc *Recommender) uncompress(data []byte) (result []byte) {
	gz, err := gzip.NewReader(strings.NewReader(string(data)))
	if err != nil {
		log.Error("The data can't be uncompressed on shard:", rc.identifier, "Error:", err)
	}
	defer gz.Close()
	if result, err = ioutil.ReadAll(gz); err != nil {
		log.Error("The data can't be uncompressed on shard:", rc.identifier, "Error:", err)
	}

	return
}

func (rc *Recommender) compress(data []byte) (result []byte) {
	var b bytes.Buffer
	gz := gzip.NewWriter(&b)
	if _, err := gz.Write(data); err != nil {
		log.Error("The data can't be compressed on shard:", rc.identifier, "Error:", err)
	}
	if err := gz.Flush(); err != nil {
		log.Error("The data can't be compressed on shard:", rc.identifier, "Error:", err)
	}
	if err := gz.Close(); err != nil {
		log.Error("The data can't be compressed on shard:", rc.identifier, "Error:", err)
	}

	return b.Bytes()
}

func (rc *Recommender) LoadBackup() (success bool) {
	log.Info("Loading backup from S3:", rc.identifier)
	auth, err := aws.EnvAuth()
	if err != nil {
		log.Error("Problem trying to connect with AWS:", err)
		return false
	}

	s := s3.New(auth, rc.s3Region)
	bucket := s.Bucket(S3BUCKET)

	jsonData, err := bucket.Get(rc.getS3Path())
	if err != nil {
		log.Info("Problem trying to get backup from S3:", err)
		return false
	}

	dataFromJSON := [][]uint64{}
	json.Unmarshal(rc.uncompress(jsonData), &dataFromJSON)

	log.Info("Data loaded from S3:", rc.identifier, "len:", len(dataFromJSON))
	recs := 0
	for _, record := range dataFromJSON {
		scores := make(map[uint64]uint8)
		for i := 1; i < len(record); i += 2 {
			scores[record[i]] = uint8(record[i+1])
		}
		recs += len(scores)
		rc.AddRecord(record[0], scores)
	}

	return true
}

func (mg *Manager) SendEmail(to, body, subject string) (success bool) {
	auth := smtp.PlainAuth(
		mg.mailFromAddr,
		mg.mailFromAddr,
		os.Getenv("PIT_MAIL_PASS"),
		mg.mailServerAddr,
	)

	body = fmt.Sprintf(
		"Subject: %s\r\n\r\n\r\n%s",
		subject,
		[]byte(body))

	err := smtp.SendMail(
		fmt.Sprintf("%s:%d", mg.mailServerAddr, mg.mailServerPort),
		auth,
		mg.mailFromAddr,
		[]string{to},
		[]byte(body),
	)

	if err != nil {
		log.Error("Problem trying to send a verification e-mail:", err)
		return false
	}
	return true
}

func InitAndKeepAlive(prefix string, awsRegion string, keepAlive bool) (im *Model) {
	if awsAuth, err := aws.EnvAuth(); err == nil {
		im = &Model{
			prefix:    prefix,
			tableName: fmt.Sprintf("%s_%s", prefix, cTable),
			conn: &dynamodb.Server{
				Auth:   awsAuth,
				Region: aws.Regions[awsRegion],
			},
		}
		im.initTable()

		if keepAlive {
			im.registerHostName(hostName)
		}
		im.updateInstances()
		if keepAlive {
			go func() {
				for {
					im.registerHostName(hostName)
					im.updateInstances()
					time.Sleep(time.Second)
				}
			}()
		}
	} else {
		log.Error("Problem trying to connect with DynamoDB, Error:", err)
		return
	}

	return
}

// GetFloat Returns the value of the section, subsection as float
func GetFloat(sec, subsec string) (v float64) {
	if v, err := strconv.ParseFloat(loadSection(sec).ValueOf(subsec), 64); err == nil {
		return v
	}

	log.Error("Configuration parameter:", sec, subsec, "can't be parsed as integer")
	return
}

func InitOandaApi(endpoint string, authToken string, accountId int, currencies []string, currLogsFile string) (api *Oanda, err error) {
	var resp []byte

	api = &Oanda{
		endpoint:        endpoint,
		openOrders:      make(map[int64]*Order),
		authToken:       authToken,
		currencies:      currencies,
		listeners:       make(map[string][]func(currency string, ts int64)),
		currentWin:      0,
		simulatedOrders: 0,
	}

	if currLogsFile != "" {
		api.currLogsFile, err = os.Create(currLogsFile)
		if err != nil {
			log.Error("Currency logs file can't be open, Error:", err)
			return
		}
	}
	api.mutex.Lock()

	if accountId == -1 {
		var accInfo map[string]interface{}

		respHttp, err := http.PostForm(fmt.Sprintf(FAKE_GENERATE_ACCOUNT_URL, api.endpoint), nil)
		if err != nil {
			return nil, err
		}
		body, err := ioutil.ReadAll(respHttp.Body)
		if err != nil {
			return nil, err
		}

		err = json.Unmarshal(body, &accInfo)
		if err != nil {
			return nil, err
		}
		resp, err = api.doRequest("GET", fmt.Sprintf(ACCOUNT_INFO_URL, api.endpoint), nil)

		log.Info("New account generated:", int(accInfo["accountId"].(float64)))
	} else {
		resp, err = api.doRequest("GET", fmt.Sprintf("%s%d", fmt.Sprintf(ACCOUNT_INFO_URL, api.endpoint), accountId), nil)
	}

	if err != nil {
		return
	}

	err = json.Unmarshal(resp, &api.account)
	if err != nil {
		return
	}

	api.mutex.Unlock()

	return
}

func (um *Model) AdminGetUserInfoByID(uid string) (user *User) {
	um.mutex.Lock()
	defer um.mutex.Unlock()
	if us, ok := um.cache[uid]; ok {
		return us
	}

	attKey := &dynamodb.Key{
		HashKey:  uid,
		RangeKey: "",
	}

	if data, err := um.table.GetItemConsistent(attKey, false); err == nil {
		user = &User{
			uid:      uid,
			key:      data["key"].Value,
			Enabled:  data["enabled"].Value,
			logs:     make(map[string][]*LogLine),
			billHist: []*Billing{},
			md:       um,
		}
		if err := json.Unmarshal([]byte(data["info"].Value), &user); err != nil {
			log.Error("Problem trying to retieve the user information for user:", uid, "Error:", err)
			return nil
		}
		if _, ok := data["logs"]; ok {
			if err = json.Unmarshal([]byte(data["logs"].Value), &user.logs); err != nil {
				log.Error("Problem trying to unmarshal the user logs for user:", uid, "Error:", err)
			}
		}
		if _, ok := data["bill_hist"]; ok {
			if err = json.Unmarshal([]byte(data["bill_hist"].Value), &user.billHist); err != nil {
				log.Error("Problem trying to unmarshal the user billing history for user:", uid, "Error:", err)
			}
		}
	} else {
		log.Error("Problem trying to read the user information for user:", uid, "Error:", err)
	}

	um.cache[uid] = user
	return
}

func (rc *Recommender) SaveBackup() {
	log.Info("Storing backup on S3:", rc.identifier)
	rc.mutex.Lock()
	records := make([][]uint64, len(rc.records))
	i := 0
	for recID, record := range rc.records {
		records[i] = make([]uint64, len(record.scores)*2+1)
		records[i][0] = recID
		elemPos := 1
		for k, v := range record.scores {
			records[i][elemPos] = k
			records[i][elemPos+1] = uint64(v)
			elemPos += 2
		}
		i++
	}
	rc.mutex.Unlock()

	jsonToUpload, err := json.Marshal(records)

	auth, err := aws.EnvAuth()
	if err != nil {
		log.Error("Problem trying to connect with AWS:", err)
		return
	}

	s := s3.New(auth, rc.s3Region)
	bucket := s.Bucket(S3BUCKET)

	err = bucket.Put(
		rc.getS3Path(),
		rc.compress(jsonToUpload),
		"text/plain",
		s3.BucketOwnerFull,
		s3.Options{})
	if err != nil {
		log.Error("Problem trying to upload backup to S3 from:", rc.identifier, "Error:", err)
	}

	log.Info("New backup stored on S3, bucket:", S3BUCKET, "Path:", rc.getS3Path())
}

func (mock *Mock) ratesCollector() {
	mock.mutex.Lock()
	mock.currencyValues = make(map[string][]*CurrVal)
	for _, curr := range mock.currencies {
		mock.mutexCurr[curr] = new(sync.Mutex)
	}
	mock.mutex.Unlock()

	scanner := bufio.NewScanner(mock.currLogsFile)
	scanner.Scan()
	log.Info("Parsing currencies from the mock file...")

	i := 0
	lastWinVal := mock.currentWin
	for {
		var feed CurrVal

		scanner.Scan()
		lineParts := strings.SplitN(scanner.Text(), ":", 2)
		curr := lineParts[0]
		if err := json.Unmarshal([]byte(lineParts[1]), &feed); err != nil {
			log.Error("The feeds response body is not a JSON valid, Error:", err)
			continue
		}

		mock.mutexCurr[curr].Lock()
		//log.Debug("New price for currency:", curr, "Bid:", feed.Bid, "Ask:", feed.Ask)
		mock.currencyValues[curr] = append(mock.currencyValues[curr], &CurrVal{
			Ts:  feed.Ts,
			Bid: feed.Bid,
			Ask: feed.Ask,
		})

		if listeners, ok := mock.listeners[curr]; ok {
			for _, listener := range listeners {
				listener(curr, feed.Ts)
			}
		}
		if len(mock.currencyValues[curr]) > MAX_RATES_TO_STORE {
			mock.currencyValues[curr] = mock.currencyValues[curr][1:]
		}
		mock.mutexCurr[curr].Unlock()

		if lastWinVal != mock.currentWin {
			log.Info("CurrentWin:", i, mock.currentWin)
			lastWinVal = mock.currentWin
		}
		i++
	}
}

func (bi *Billing) callToApi(uri string, method string, reqBody string) (body []byte, err error) {
	if time.Now().Unix() > bi.tokenExpireTs {
		bi.getToken()
	}

	client := &http.Client{}

	url := bi.baseUri + uri
	log.Info(reqBody)
	req, err := http.NewRequest(method, url, strings.NewReader(reqBody))
	req.Header.Add("Content-Type", `application/json`)
	req.Header.Add("Authorization", "Bearer "+bi.token)
	resp, err := client.Do(req)
	if err != nil {
		log.Error("Problem trying to do a request to the PayPal API, Url:", url, "Error:", err)
	}

	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Error("Problem trying to parse the response form the PayPal API, Error:", err)
	}

	return
}

func (um *Model) initTable() {
	pKey := dynamodb.PrimaryKey{dynamodb.NewStringAttribute(cPrimKey, ""), nil}
	um.table = um.conn.NewTable(um.tableName, pKey)

	res, err := um.table.DescribeTable()
	if err != nil {
		log.Info("Creating a new table on DynamoDB:", um.tableName)
		td := dynamodb.TableDescriptionT{
			TableName: um.tableName,
			AttributeDefinitions: []dynamodb.AttributeDefinitionT{
				dynamodb.AttributeDefinitionT{cPrimKey, "S"},
			},
			KeySchema: []dynamodb.KeySchemaT{
				dynamodb.KeySchemaT{cPrimKey, "HASH"},
			},
			ProvisionedThroughput: dynamodb.ProvisionedThroughputT{
				ReadCapacityUnits:  cDefaultWRCapacity,
				WriteCapacityUnits: cDefaultWRCapacity,
			},
		}

		if _, err := um.conn.CreateTable(td); err != nil {
			log.Error("Error trying to create a table on Dynamo DB, table:", um.tableName, "Error:", err)
		}
		if res, err = um.table.DescribeTable(); err != nil {
			log.Error("Error trying to describe a table on Dynamo DB, table:", um.tableName, "Error:", err)
		}
	}
	for "ACTIVE" != res.TableStatus {
		if res, err = um.table.DescribeTable(); err != nil {
			log.Error("Can't describe Dynamo DB instances table, Error:", err)
		}
		log.Debug("Waiting for active table, current status:", res.TableStatus)
		time.Sleep(time.Second)
	}
}

func TestCollabInsertion(t *testing.T) {
	runtime.GOMAXPROCS(runtime.NumCPU())

	f, err := os.Open(TESTSET)
	if err != nil {
		log.Error("Can't read the the test set file:", TESTSET, "Error:", err)
		t.Fail()
	}
	r := bufio.NewReader(f)
	s, e := Readln(r)
	records := []map[uint64]uint8{}
	log.Info("Parsing test file...")
	for i := 0; e == nil && i < 10000; i++ {
		//for i := 0; e == nil && i < 480187; i++ {
		s, e = Readln(r)
		_, scores := parseLine(s)
		records = append(records, scores)
	}
	log.Info("Generating tree...")
	tr, _ := ProcessNewTrees(records, 50, MAXSCORE, 3)
	tr.setTestMode()
	log.Info("Tree generated...")

	quadError := 0.0
	comparedItems := 0
	for i := 0; e == nil && i < 1000; i++ {
		s, e = Readln(r)
		_, scores := parseLine(s)
		elements := tr.GetBestRecommendation(scores, 10)

		for _, elemID := range elements {
			if score, rated := scores[elemID]; rated {
				quadError += (1.0 - (float64(score) / MAXSCORE)) * (1.0 - (float64(score) / MAXSCORE))
				comparedItems++
			}
		}
	}

	// Estimate the Root-mean-square deviation, we will use 0.3 for this test because the training set and the number of trees is too low
	rmsd := math.Sqrt(quadError / float64(comparedItems))
	if rmsd > 0.3 {
		t.Error("The RMSD is bigger than the expected, obtained:", rmsd)
	}

	return
}

func (rc *Recommender) DestroyS3Backup() (success bool) {
	log.Info("Destroying backup on S3:", rc.identifier)
	auth, err := aws.EnvAuth()
	if err != nil {
		log.Error("Problem trying to connect with AWS:", err)
		return false
	}

	s := s3.New(auth, rc.s3Region)
	bucket := s.Bucket(S3BUCKET)

	if err := bucket.Del(rc.getS3Path()); err != nil {
		log.Info("Problem trying to remove backup from S3:", err)
		return false
	}

	return true
}

func GetMock(feedsFile string, feedsBySecond int, currencies []string, httpPort int) (mock *Mock) {
	var err error

	mock = &Mock{
		orders:        0,
		currentWin:    0,
		feedsBySecond: feedsBySecond,
		mutexCurr:     make(map[string]*sync.Mutex),
		ordersByCurr:  make(map[string][]*Order),
		currencies:    currencies,
		listeners:     make(map[string][]func(currency string, ts int64)),
		openOrders:    make(map[int64]*Order),
	}

	for _, curr := range currencies {
		mock.ordersByCurr[curr] = []*Order{}
	}

	if feedsFile != "" {
		mock.currLogsFile, err = os.Open(feedsFile)
		if err != nil {
			log.Error("Currency logs file can't be open, Error:", err)
			return
		}
	}

	http.HandleFunc("/get_curr_values_orders", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")
		w.Header().Set("Content-Type", "application/json")
		curr := r.FormValue("curr")
		info, _ := json.Marshal(&currOpsInfo{
			Prices: mock.currencyValues[curr],
			Orders: mock.ordersByCurr[curr],
		})
		w.Write(info)
	})
	go http.ListenAndServe(fmt.Sprintf(":%d", httpPort), nil)
	log.Info("Mock HTTP server listening on:", httpPort)

	return
}