void test_sentence(PersonRecog &pr, const char *sentence)
{
    Words words;
	pr.recog(sentence, words);
    for (int i=0; i<words.size(); i+=1)
		printf("%s\n", words[i].c_str());
}

void Dictionary::printInteractiveSuggestions(Words const& active_words, unsigned const show) const
{
	assert(!active_words.empty());
	assert(show > 0);
	vector<pair<double, Word>> suggestions;
	suggestions.reserve(active_words.size());
	puts("Suggestions:");
	{
		DividedWordList dl(wordLength());
		auto it = keys_.cbegin();
		do {
			dl.build(*this, active_words, *it);
			suggestions.push_back(make_pair(-dl.entropy(), *it));
		} while (++it != keys_.cend());
	}
	sort(suggestions.begin(), suggestions.end());
	{
		auto it = suggestions.cbegin();
		unsigned i = 0;
		do {
			printf("\t%2u: ", ++i);
			describeWord(stdout, it->second);
			putchar('\n');
		} while (++it != suggestions.cend() && i < show);
	}
	putchar('\n');
}

// When player has entered something, it is parsed elsewhere
uint8 AgiEngine::testSaid(uint8 nwords, uint8 *cc) {
	AgiGame *state = &_game;
	AgiEngine *vm = state->_vm;
	Words *words = vm->_words;
	int c, n = words->getEgoWordCount();
	int z = 0;

	if (vm->getFlag(VM_FLAG_SAID_ACCEPTED_INPUT) || !vm->getFlag(VM_FLAG_ENTERED_CLI))
		return false;

	// FR:
	// I think the reason for the code below is to add some speed....
	//
	//      if (nwords != num_ego_words)
	//              return false;
	//
	// In the disco scene in Larry 1 when you type "examine blonde",
	// inside the logic is expected ( said("examine", "blonde", "rol") )
	// where word("rol") = 9999
	//
	// According to the interpreter code 9999 means that whatever the
	// user typed should be correct, but it looks like code 9999 means that
	// if the string is empty at this point, the entry is also correct...
	//
	// With the removal of this code, the behavior of the scene was
	// corrected

	for (c = 0; nwords && n; c++, nwords--, n--) {
		z = READ_LE_UINT16(cc);
		cc += 2;

		switch (z) {
		case 9999:  // rest of line (empty string counts to...)
			nwords = 1;
			break;
		case 1: // any word
			break;
		default:
			if (words->getEgoWordId(c) != z)
				return false;
			break;
		}
	}

	// The entry string should be entirely parsed, or last word = 9999
	if (n && z != 9999)
		return false;

	// The interpreter string shouldn't be entirely parsed, but next
	// word must be 9999.
	if (nwords != 0 && READ_LE_UINT16(cc) != 9999)
		return false;

	setFlag(VM_FLAG_SAID_ACCEPTED_INPUT, true);

	return true;
}

void tokenize(const char* string, Words& words)
{

	enum State
	{
		ScanningForWordBeginning,
		ScanningForWordEnd,
	};

	State state = ScanningForWordBeginning;

	const char* wordStart = string;
	const char* wordEnd = string;

	while (true)
	{
		char ch = *wordEnd;

		switch (state)
		{
		case ScanningForWordBeginning:
			{
				if (ch == '\0')
				{
					return;
				}
				else if (ch != ' ' && ch != '\t')
				{
					wordStart = wordEnd;
					wordEnd++;
					state = ScanningForWordEnd;
				}
			}
			break;

		case ScanningForWordEnd:
			{
				if (ch == '\0')
				{
					words.push_back(std::string(wordStart));
					return;
				}
				if (ch != ' ' && ch != '\t')
				{
					wordEnd++;
				}
				else
				{
					words.push_back(std::string(wordStart, wordEnd - wordStart));
					wordEnd++;
					state = ScanningForWordBeginning;
				}
			}
			break;
		}
	}
}

bool Matches::addMatches( char *s, int32_t slen, mf_t flags ) {
	// . do not breach
	// . happens a lot with a lot of link info text
	if ( m_numMatchGroups >= MAX_MATCHGROUPS ) {
		return true;
	}

	// get some new ptrs for this match group
	Words    *wp = &m_wordsArray    [ m_numMatchGroups ];
	Bits     *bp = &m_bitsArray     [ m_numMatchGroups ];
	Pos      *pb = &m_posArray      [ m_numMatchGroups ];

	// set the words class for this match group
	if ( !wp->set( s, slen, true ) ) {
		return false;
	}

	// bits vector
	if ( ! bp->setForSummary ( wp ) ) {
		return false;
	}

	// position vector
	if ( ! pb->set ( wp ) ) {
		return false;
	}

	// record the start
	int32_t startNumMatches = m_numMatches;
	// sometimes it returns true w/o incrementing this
	int32_t n = m_numMatchGroups;
	// . add all the Match classes from this match group
	// . this increments m_numMatchGroups on success
	bool status = addMatches( wp, NULL, NULL, bp, pb, flags );

	// if this matchgroup had some, matches, then keep it
	if ( m_numMatches > startNumMatches ) {
		return status;
	}

	// otherwise, reset it, useless
	wp->reset();
	bp->reset();
	pb->reset();

	// do not decrement the counter if we never incremented it
	if ( n == m_numMatchGroups ) {
		return status;
	}

	// ok, remove it
	m_numMatchGroups--;

	return status;
}

int main( int argc, char * argv[] ) {
    char from[10] = "aablls?";

    if( argc >= 2 ) {
        std::string arg(argv[1]);
        std::sort( arg.begin(), arg.end() );
        strcpy( from, arg.c_str() );
    }

    if( argc >= 3 ) {
        min_length = atoi( argv[2] );
    }


    logf( 1, CLEAR "Score %s -> %i\n", from, ScoreString( from ) );

    const int MAX_LEN = 32;
    int lengths[MAX_LEN] = {0};
    if( FILE * fp = fopen( "enable1.txt", "rt" ) ) {
        double start = PerfTime();
        while( !feof( fp ) ) {
            char buf[MAX_LEN];
            char * got = fgets( buf, 63, fp );

            if( got ) {
                char *end = buf + strlen(buf) -1;
                while( end > buf && !isalpha( *end ) ) {
                    *end = 0;
                    end -= 1;
                }

                int l = strlen(buf);
                lengths[l] += 1;

                if( l <= 15 ) {
                    words.push_back(std::string(buf));
                }
            }
        }
        double loaded = PerfTime();

        double runTime = PerfTime();
        Words gathered = GatherAngrams( from );
        std::sort( gathered.begin(), gathered.end(), XScoresLessThanY );
        for( auto s : gathered ) {
            logf( 1, CLEAR "%s - %i\n", s.c_str(), ScoreString( s ) );
        }
        double done = PerfTime();
        logf( 1, CLEAR "Timing (%f s) Loading\n", loaded-start );
        //logf( 1, CLEAR "Timing (%f s) Preparing\n", prepared-prepping);
        logf( 1, CLEAR "Timing (%f s) running\n", done-runTime );
    }

    return 0;
}

void parse_doc_icu(char *s, int len, bool doHash, char *charset){
	Xml xml;
	xml.set( s, len, TITLEREC_CURRENT_VERSION, 0, CT_HTML );

	// Extract text from (x)html
	char *text_buf = (char*)malloc(64*1024);
	int32_t textLen = xml.getText( text_buf, 64 * 1024, 0, 99999999, doFilterSpaces );
	Words w;
	w.set(text_buf, textLen, doHash);
	free(text_buf);
}

void parse_doc_8859_1(char *s, int len, bool doHash,char *charset)
{
	Xml xml;
	xml.set( s, len, TITLEREC_CURRENT_VERSION, 0, CT_HTML );

	// Extract text from (x)html
	char *text_buf = (char*)malloc(len+1);
	xml.getText( text_buf, len, 0, 99999999, doFilterSpaces );
	Words words;

	// just tokenize words
	words.set(text_buf, len, doHash);
	free(text_buf);
}

void DividedWordList::build(Dictionary const& dictionary, Words const& ws, Word const guess)
{
	assert(size() == dictionary.wordLength() + 1);
	assert(!ws.empty());
	for (auto it = begin(); it != end(); ++it)
		it->clear();
	{
		auto it = ws.cbegin();
		do {
			unsigned const c = correct_letters(*it, guess);
			assert(c <= dictionary.wordLength());
			(*this)[c].update(dictionary, *it);
		} while (++it != ws.cend());
	}
}

void RconConnection::EnableEvents(void)
{
	Words command;
	command.clear();
	command.push_back("admin.eventsEnabled");
	command.push_back("true");

	if(sendRequest(command))
		throw string("sendRequest failed :: eventsEnabled");

	TextRconPacket response = getResponse();

	if(!response.m_isResponse || response.m_data[0] != "OK")
		throw string("eventsEnabled failed");
}

	virtual void onServerResponse(const Words& words)
	{
		if (words.size() == 3 && words[0] == "OK")
			printf("Server version: Game %s, build ID %s\n", words[1].c_str(), words[2].c_str());
		else
			printf("Invalid response to version query\n");
	}

bool submissionsSequence(const Words& words, unsigned int& i)
{
    if (i + 1 >= words.size()) return false;

    if (words[i] == "From"   ||
        words[i] == "from"   ||
        words[i] == "From:"  ||
        words[i] == "from:"  ||
        words[i] == "Merged" ||
        words[i] == "Integrated") return true;

    if (i + 2 >= words.size()) return false;

    if (words[i] == "submitted" && words[i + 1] == "by")
    {
        i += 1;
        return true;
    }

    if (words[i] == "Folded" && words[i + 1] == "in")
    {
        i += 1;
        return true;
    }

    if (words[i] == "Rolled" && words[i + 1] == "in")
    {
        i += 1;
        return true;
    }

    if (words[i] == "Checked" && words[i + 1] == "in")
    {
        i += 1;
        return true;
    }

    if (i + 3 >= words.size()) return false;

    if (words[i] == "sent" && words[i + 1] == "in" && words[i + 2] == "by")
    {
        i += 2;
        return true;
    }

    return false;
}

void RconConnection::Login(void)
{
	Words loginCommand;
	loginCommand.push_back("login.hashed");

	if(sendRequest(loginCommand))
		throw string("Login failed");

	TextRconPacket response = getResponse();

	if(response.m_isResponse && response.m_data[0] == "OK")
	{
		string salt = response.m_data[1];

		const char* hex_str = salt.c_str();
		string hash, saltHex;
		uint32_t ch;

		for( ; sscanf( hex_str, "%2x", &ch) == 1 ; hex_str += 2)
			hash += ch;

		saltHex = hash;

		hash.append( this->password );
		hash = MD5String( (char*)hash.c_str() );

		boost::to_upper(hash);

		loginCommand.clear();
		loginCommand.push_back("login.hashed");
		loginCommand.push_back(hash);

		if(sendRequest(loginCommand))
			throw string("sendRequest failed :: Login");

		response = getResponse();

		if(response.m_isResponse && response.m_data[0] == "InvalidPasswordHash")
			throw string("Login failed :: InvalidPasswordHash (Salt: " + salt + " | SaltHex: "+ saltHex +" | Hash: " + hash + ")");

	}
	else
		throw string("Login failed");


}

void test_file(PersonRecog &pr, const char *file)
{
	ifstream fi(file);
	string line;
	Words words;
	if (!fi) {
		printf("can not open file: %s\n", file);
		return;
	}
	while (getline(fi, line)) {
		pr.recog(line.c_str(), words);
		//printf("%s\n", line.c_str());
		for (int i=0; i<words.size(); i+=1)
			printf("%s\n", words[i].c_str());
	}
    fi.close();
}

void parse_doc_icu(char *s, int len, bool doHash, char *charset){
	Xml xml;
	xml.set(csUTF8,s,len,false, 0,false, TITLEREC_CURRENT_VERSION);
	//fprintf(stderr,"\nparse_doc_icu\n");	
	// Extract text from (x)html
	char *text_buf = (char*)malloc(64*1024);
	long textLen = xml.getText(text_buf, 
				   64*1024, 
				   0,
				   99999999,
				   false,
				   true,
				   false,
				   doFilterSpaces,
				   false);
	Words w;
	w.set(true,false, text_buf, textLen, TITLEREC_CURRENT_VERSION,doHash);
	free(text_buf);
}

void Output::print(const Words& message, bool useComma, bool useAnd)
{
	if (!_streams[_curStream].print())
		return;
	for (int i = 0; i < message.length(); ++i)
	{
		stream() << message[i];
		if (useComma)
		{
			if ((message.length() > 2) && (i != message.length() - 1))
				stream() << ",";
			if ((i == message.length() - 2) && (useAnd))
				stream() << " and";
		}
		if (i != message.length() - 1)
			stream() << " ";
	}
	stream() << flush;
}

Words GatherAngrams( const char * from ) {
    int wild = 0;
    int find_freq[26] = {0};
    uint32_t find_have = 0;
    for( const char *s = from; *s; ++s ) {
        if( *s == '?' ) {
            wild += 1;
        } else {
            const int ord = *s - 'a';
            find_freq[ord] += 1;
            find_have |= (1<<ord);
        }
    }
    uint32_t find_dont = ~find_have;

    Words matchList;
    const int count = words.size();
    for( int i = 0; i < count; ++i ) {
        const char *s = words[i].c_str();
        if( strlen( s ) < min_length )
            continue;
        int freq[26] = {0};
        uint32_t have = 0;
        for( ; *s; ++s ) {
            const int ord = *s - 'a';
            freq[ord] += 1;
            have |= (1<<ord);
        }

        if( wild == 0 && find_dont & have )
            continue;
        int over = 0;
        for( int c = 0; c < 26; ++c ) {
            if( freq[c] > find_freq[c] )
                over += freq[c] - find_freq[c];
        }
        if( over <= wild ) {
            logf( 1, RED "From (%s) -> (%s)\n", from, words[i].c_str() );
            matchList.push_back( words[i] );
        }
    }
    return matchList;
}

static void generateSummary( Summary &summary, char *htmlInput, const char *queryStr, const char *urlStr ) {
	Xml xml;
	ASSERT_TRUE(xml.set(htmlInput, strlen(htmlInput), 0, CT_HTML));

	Words words;
	ASSERT_TRUE(words.set(&xml, true));

	Bits bits;
	ASSERT_TRUE(bits.set(&words));

	Url url;
	url.set(urlStr);

	Sections sections;
	ASSERT_TRUE(sections.set(&words, &bits, &url, "", CT_HTML));

	Query query;
	ASSERT_TRUE(query.set2(queryStr, langEnglish, true));

	LinkInfo linkInfo;
	memset ( &linkInfo , 0 , sizeof(LinkInfo) );
	linkInfo.m_lisize = sizeof(LinkInfo);

	Title title;
	ASSERT_TRUE(title.setTitle(&xml, &words, 80, &query, &linkInfo, &url, NULL, 0, CT_HTML, langEnglish));

	Pos pos;
	ASSERT_TRUE(pos.set(&words));

	Bits bitsForSummary;
	ASSERT_TRUE(bitsForSummary.setForSummary(&words));

	Phrases phrases;
	ASSERT_TRUE(phrases.set(&words, &bits));

	Matches matches;
	matches.setQuery(&query);
	ASSERT_TRUE(matches.set(&words, &phrases, &sections, &bitsForSummary, &pos, &xml, &title, &url, &linkInfo));

	summary.setSummary(&xml, &words, &sections, &pos, &query, 180, 3, 3, 180, &url, &matches, title.getTitle(), title.getTitleLen());
}

SDataForSolver CSolver::ExtractData(const std::string & inputString)
{
	SDataForSolver result;
	Words arguments = SplitWords(inputString);

	if (arguments.size() == AMOUNT_ARGUMENTS)
	{
		result.firstPoint.x = stof(arguments[0]);
		result.firstPoint.y = stof(arguments[1]);

		result.secondPoint.x = stof(arguments[2]);
		result.secondPoint.y = stof(arguments[3]);

		result.radiusCircle = stof(arguments[4]);
	}
	else
	{
		throw std::invalid_argument(MESSAGE_INCORRECT_AMOUNT_ARGUMENTS);
	}

	return result;
}

void Dictionary::recursiveSolve(AbstractSolutionAcceptor * const acceptor, Words const& active_words, AnswerSequence * const runningAnswersp) const
{
	assert(!active_words.empty());
	if (active_words.size() == 1) {	// We found an end node
		acceptor->acceptSolution(*runningAnswersp, active_words.front());
	} else {
		DividedWordList dl(wordLength());
		double best_entropy = -1.0;
		Word best_word = 0;
		// Of all of the words in active_words, find the one that
		// will gives us the most even split (and thus the highest entropy)
		{
			auto it = keys_.cbegin();
			do {
				dl.build(*this, active_words, *it);
				double const entropy = dl.entropy();
				if (entropy > best_entropy) {
					best_entropy = entropy;
					best_word = *it;
				}
			} while (++it != keys_.cend());
		}
		assert(popcount32(best_word) == wordLength());
		// Now that we have found our best word, recompute the split
		dl.build(*this, active_words, best_word);
		assert(dl.entropy() == best_entropy);
		acceptor->acceptBranch(*runningAnswersp, best_word, best_entropy, dl);
		// Now recursively descend
		for (unsigned i = 0; i < wordLength() + 1; i++) {
			const WordsWithTotalChoices& wwtc = dl[i];
			if (wwtc.count() > 0) {
				runningAnswersp->push_back(i);
				recursiveSolve(acceptor, wwtc.choices(), runningAnswersp);
				runningAnswersp->pop_back();
			}
		}
	}
}