bool Lattice::assign(utils::piece::const_iterator& iter, utils::piece::const_iterator end)
  {
    namespace qi = boost::spirit::qi;
    namespace standard = boost::spirit::standard;
    
    clear();
    
    // empty lattice...
    if (iter == end) return true;
    
    utils::piece::const_iterator iter_back = iter;
    
    if (qi::phrase_parse(iter, end, lattice_grammar_parser_impl::instance(), standard::space, lattice)) {
      initialize_distance();
      return true;
    } else {
      clear();
      
      // fallback to sentence...
      iter = iter_back;
      Sentence sentence;
      
      if (sentence.assign(iter, end)) {
	Sentence::const_iterator iter_end = sentence.end();
	for (Sentence::const_iterator iter = sentence.begin(); iter != iter_end; ++ iter)
	  lattice.push_back(arc_set_type(1, arc_type(*iter)));
	
	return true;
      } else {
	clear();
	return false;
      }
    }
  }

bool isPELCNFLiteral(const Sentence& sentence) {
    if (sentence.getTypeCode() == Atom::TypeCode
            || sentence.getTypeCode() == BoolLit::TypeCode
            || sentence.getTypeCode() == LiquidOp::TypeCode) {
        return true;
    }
    if (sentence.getTypeCode() == Negation::TypeCode) {
        const Negation& neg = static_cast<const Negation&>(sentence);

        // TODO: necessary to check for double negation?
        if (neg.sentence()->getTypeCode() == Negation::TypeCode) return false;
        return isPELCNFLiteral(*neg.sentence());
    }
    if (sentence.getTypeCode() == DiamondOp::TypeCode) {
        const DiamondOp& dia = static_cast<const DiamondOp&>(sentence);
        if (       dia.sentence()->getTypeCode()    == Atom::TypeCode
                || dia.sentence()->getTypeCode()    == BoolLit::TypeCode
                || dia.sentence()->getTypeCode()    == LiquidOp::TypeCode) {  // TODO add liquidop
            return true;
        }
        return false;
    }
    if (sentence.getTypeCode() == Conjunction::TypeCode) {
        const Conjunction& con = static_cast<const Conjunction&>(sentence);
        if ((con.left()->getTypeCode() == Atom::TypeCode
                || con.left()->getTypeCode() == BoolLit::TypeCode)
              && (con.right()->getTypeCode() == Atom::TypeCode
                || con.right()->getTypeCode() == BoolLit::TypeCode)) {
            return true;
        }
        return false;
    }

    return false;
}

IChoiceTag* IChoiceTag::Create(Sentence& sentence)
{
    static Sentence kahnsOrDragons("As THIS enters the battlefield, choose Khans or Dragons.");
    static Sentence opponentChoose("An opponent chooses one " HYPEN);

    if (kahnsOrDragons.IsEqualTo(sentence))
    {
        sentence.Clear();
        return new KahnsOrDragons;
    }

    if (opponentChoose.IsEqualTo(sentence))
    {
        sentence.Clear();
        return new OpponentChoose;
    }

	auto first = sentence.Find(
		[](IObjectTag* tag) -> bool { return tag->GetType() == ObjectType::Choose; });

	if (first.found)
	{
		auto second = sentence.Find(
			[](IObjectTag* tag) -> bool { return tag->GetType() == ObjectType::BigSeperator; }, 
			first.iterator);

		if (second.found)
		{
			return new ChoiceBegin(sentence.Splice(second.iterator));
		}
	}
    return nullptr;
}

bool KnowLedgeBasedAgent::askOr(string query, vector<string>& theta)
{
    Predicate p = TextParser::GetPredicate(query);
    if (predicateMap.find(p.name) != predicateMap.end())
    {
        vector<int>rules = predicateMap[p.name];
        bool flag = false;
        for (int ruleIterator = 0; ruleIterator < rules.size(); ruleIterator++)
        {
            Sentence sentence = KnowledgeBase[rules[ruleIterator]-1];
            vector<string> premise = sentence.GetPremise();
            string conclusion = sentence.GetConclusion();
            Predicate q = TextParser::GetPredicate(conclusion);
            string substitution = "";
            bool unificationResult = Unify(p, q, substitution);
            if (!unificationResult) continue;
            if (find(theta.begin(), theta.end(), substitution) == theta.end())
            {
                theta.push_back(*new string(substitution));
            }
            bool validity = askAnd(premise, theta);
            if (validity) flag = true;
            else flag = false;
        }
        if(flag) return true;
        else return false;
    }
    else return false;
}

void KnowLedgeBasedAgent::Tell(string percept)
{
    t = t+1;
    Sentence sentence = MakePerceptSentence(percept);
    KnowledgeBase.push_back(sentence);
    Predicate conclusion = TextParser::GetPredicate(sentence.GetConclusion());
    if (predicateMap.find(conclusion.name) == predicateMap.end())
    {
        vector<int> conclusionId;
        conclusionId.push_back(t);
        predicateMap[conclusion.name] = conclusionId;
        if (conclusion.arg1[0] != 'x' && find(constants.begin(), constants.end(), conclusion.arg1) == constants.end())
            constants.push_back(conclusion.arg1);
        if (conclusion.arg2[0] != 'x' && find(constants.begin(), constants.end(), conclusion.arg2) == constants.end())
            constants.push_back(conclusion.arg2);
        }
    else
    {
        (predicateMap[conclusion.name]).push_back(t);
        if (conclusion.arg1[0] != 'x' && find(constants.begin(), constants.end(), conclusion.arg1) == constants.end())
            constants.push_back(conclusion.arg1);
        if (conclusion.arg2[0] != 'x' && find(constants.begin(), constants.end(), conclusion.arg2) == constants.end())
            constants.push_back(conclusion.arg2);
    }
    vector<string> premises = sentence.GetPremise();
    for (int i=0; i<premises.size(); i++)
    {
        Predicate premise = TextParser::GetPredicate(premises[i]);
        if (premise.arg1[0] != 'x' && find(constants.begin(), constants.end(), premise.arg1) == constants.end())
            constants.push_back(premise.arg1);
        if (premise.arg2[0] != 'x' && find(constants.begin(), constants.end(), premise.arg2) == constants.end())
            constants.push_back(premise.arg2);
    }
}

 void readWordFile(string filePath)
 {
     ifstream in(filePath.c_str());
     int sentenceId, wordNo, wordId;
     SentenceLine sentenceline;
     int lastSentenceId = 1;
     while (in>>sentenceId>>wordNo>>wordId)
     {
         if (sentenceId!=lastSentenceId&&sentenceId>1)
         {
             Sentence s;
             s.ReadLine(sentenceline);
             if (sentenceId-lastSentenceId>1) // some files the sentenceId is not continous, treat it as zero length sentence
             {
                 for (int i=0; i<sentenceId-lastSentenceId-1; i++)
                 {
                     Sentence tmp;
                     AppendSentence(tmp);
                 }
             }
             AppendSentence(s);  //note that byte is zero, need to correct and Combine Step
             sentenceline.clear();
             lastSentenceId= sentenceId;
         }
         sentenceline.push_back(wordId);
     }
     Sentence s;
     s.ReadLine(sentenceline);
     AppendSentence(s);
     in.close();
 }

long DataManager::readPOSFile(const char* filePath, const std::function<void (Sentence*)> placementFunction)
{
    Sentence *sentence = nullptr;
    long counter = 0;
    decltype(readLine()) line = nullptr;

    if (openFile(filePath, true)) {
        while ((line = readLine()) != nullptr) {
            if (strlen(line) == 0) {
                if (sentence) {
                    placementFunction(sentence);
                    sentence = nullptr;
                }
                continue;
            }

            auto indexOfTab = line;

            for (; *indexOfTab != '\0'; indexOfTab++) {
                if (*indexOfTab == '\t') {
                    *(indexOfTab++) = '\0';
                    break;
                }
            }

            if (!sentence)
                sentence = new Sentence();

            sentence->addWord(line, indexOfTab);
        }
    }

    return counter;
}

    void Run::parse(const std::string &sInputFile, const std::string &sOutputFile,
                    const std::string &sFeatureFile) const {

        Sentence sentence;
        DependencyTree tree;

        std::cout << "Parsing started" << std::endl;

        auto time_begin = time(NULL);

        std::unique_ptr<DepParser> parser(new DepParser(sFeatureFile, sFeatureFile, ParserState::PARSE));
        std::ifstream input(sInputFile);
        std::ofstream output(sOutputFile);
        if (input) {
            while (input >> sentence) {
                if (sentence.size() < MAX_SENTENCE_SIZE) {
                    parser->parse(sentence, &tree);
                    output << tree;
                    tree.clear();
                }
            }
        }
        input.close();
        output.close();

        auto time_end = time(NULL);

        auto seconds = difftime(time_end, time_begin);

        std::cout << "Parsing has finished successfully. Total time taken is: " << difftime(time_end, time_begin) <<
        "s" << std::endl;
    }

// "((this|next|last) year)|(\d\d\d\d)"
int year_is_found( Sentence& mSentence, struct tm* mFillin=NULL )
{
    int retval=0;
    retval = mSentence.is_found_in_sentence("this year");
    if (retval)
    {
        if (mFillin) mFillin->tm_year = retval = bd_now.tm_year;
        return retval;
    } else if ((retval = mSentence.is_found_in_sentence("next year")))
    {
        if (mFillin) mFillin->tm_year = retval = bd_now.tm_year+1;
        return retval;
    } else if ((retval = mSentence.is_found_in_sentence("last year")))
    {
        if (mFillin) mFillin->tm_year = retval = bd_now.tm_year-1;
        return retval;
    }
    /* Scan numbers 1900 to 2016 */
    char year_ascii[4];
    for (int year=1900; year<2100; year++) {
        sprintf(year_ascii, "%d", year);
        if (mSentence.is_found_in_sentence( year_ascii )) {
            if (mFillin) mFillin->tm_year = year-1900;
            return mSentence.get_word_index(year_ascii);
        }
    }
    return retval;
}

// 인자로 주어진 파일로부터 한문장씩 읽어 이를 번역
void doTranslate(char* pInFile, Manual& out, int wantedLang)
{
	ifstream ifs(pInFile);
	if (!ifs)
	{
		cout << "Can't open file : " << pInFile << endl;
		return;
	}
	string result;
	Sentence next;
	while (!(next = getSentence(ifs)).getString().empty())
	{
		switch (next.getType())
		{
		case NORMAL_SENTENCE: // 평서문
			result = TransNormalSentence(next.getString(), wantedLang);
			break;
		case INTERROGATIVE_SENTENCE: // 의문문
			result = TransInterrogativeSentence(next.getString(), wantedLang);
			break;
		case IMPERATIVE_SENTENCE: // 명령문
			result = TransImperativeSentence(next.getString(), wantedLang);
			break;
		default:
			cout << "Untranslatable sentence type" << endl;
			return;
		}
		out.addContents(result);

	}
}

VbCodeExpressionPtr VbCodeExpressionFactory::CreateConstantExpression(const Sentence& sentence)
{
	if (sentence.GetNodes().size() != 1)
		throw std::runtime_error("Literal should contain exactly 1 token.");
	auto& value = VbCodeValueFactory::Create(sentence.GetNodes()[0]->AsToken());
	return std::make_shared<VbCodeConstantExpression>(value);
}

/** constructor; just initialize the base class */
TranslationOptionCollectionText::TranslationOptionCollectionText(Sentence const &input, size_t maxNoTransOptPerCoverage, float translationOptionThreshold)
  : TranslationOptionCollection(input, maxNoTransOptPerCoverage, translationOptionThreshold)
{
  size_t size = input.GetSize();
  m_inputPathMatrix.resize(size);
  for (size_t phaseSize = 1; phaseSize <= size; ++phaseSize) {
    for (size_t startPos = 0; startPos < size - phaseSize + 1; ++startPos) {
      size_t endPos = startPos + phaseSize -1;
      vector<InputPath*> &vec = m_inputPathMatrix[startPos];

      WordsRange range(startPos, endPos);
      Phrase subphrase(input.GetSubString(WordsRange(startPos, endPos)));
      const NonTerminalSet &labels = input.GetLabelSet(startPos, endPos);

      InputPath *node;
      if (range.GetNumWordsCovered() == 1) {
        node = new InputPath(subphrase, labels, range, NULL, NULL);
        vec.push_back(node);
      } else {
        const InputPath &prevNode = GetInputPath(startPos, endPos - 1);
        node = new InputPath(subphrase, labels, range, &prevNode, NULL);
        vec.push_back(node);
      }

      m_phraseDictionaryQueue.push_back(node);
    }
  }
}

	// make N-gram
	Sentence makeNGram(const Sentence& sent, Position begin, unsigned int n)
	{
		Sentence ret;
		auto it1 = sent.begin() + begin;
		auto it2 = it1 + n;
		ret.insert(ret.end(), it1, it2);
		return move(ret);
	}

void ParallelCorpus::PrintSentence(
    const Sentence& sentence, const Vocab& vocab, std::ostream& out) const {
  if (sentence.size() > 0) {
    out << vocab.GetWord(sentence.at(0));
  }
  for (int i = 1; i < sentence.size(); ++i) {
    out << " " << vocab.GetWord(sentence.at(i));
  }
}

// render a sentence in words
void putSentence(const Wordlist& wlist, const Sentence sent)
{
  for (Sentence::const_iterator i = sent.begin() ; i != sent.end() ; ++i) {
    if (*i != 0) 
      cout << wlist[*i-1] << " " ;
  }
  cout << endl ;

}

int parse_qualitative_1_time( Sentence& mSentence, time_t& sTime )
{
	// Parse Qualitative time:
	parsed_qualitative_duration = mSentence.are_found_in_sentence( "in the last %d (day|days|hour|hours|minute|minutes|seconds)" ) || 
								  mSentence.are_found_in_sentence( "between" );	// hour, day, 5 hours, 5 minutes.
	
	parsed_qualitative_duration  = mSentence.are_found_in_sentence( "at [specific time ie five thirty]" );
	parsed_qualitative_time = parsed_qualitative_duration;	
}

void SRLBaselineExt::ExtractPrgFeatures(vector< vector<string> >& vecPrgFeatures) const
{
    vecPrgFeatures.clear();

    Sentence sentence;

    vector<string> vecRows;
    convert2ConllFormat(vecRows);

    sentence.from_corpus_block(vecRows);
    const size_t row_count = sentence.get_row_count();

    m_featureExtractor->set_target_sentence(sentence);

    m_featureExtractor->calc_node_features();

    vector< vector<string> > vec_feature_values;
    for (size_t i = 0; i < m_prgFeatureNumbers.size(); ++i)
    {
        vector<string> feature_values;

        const int feature_number = m_prgFeatureNumbers[i];
        const string& feature_prefix = m_prgFeaturePrefixes[i];
        bool feature_empty_flag = false;
        try {
            m_featureExtractor->get_feature_for_rows(feature_number, feature_values);
        } catch (...) {
            feature_empty_flag = true;
        }

        if (feature_empty_flag)
        {
            feature_values.clear();
            for (size_t row = 0; row <= row_count; ++row)
            {
                feature_values.push_back("");
            }
        }

        vec_feature_values.push_back(feature_values);
    }

    for (size_t row = 1; row <= row_count; ++row)
    {
        vector<string> instance;
        for (size_t i = 0; i < m_prgFeatureNumbers.size(); ++i)
        {
            string feature = m_prgFeaturePrefixes[i] + "@"
                + vec_feature_values[i][row];
            instance.push_back(feature);
        }
        vecPrgFeatures.push_back(instance);
    }
}

int RestaurantOrder::is_order_complete( Sentence& mSentence )
{
    orderFoundComplete = mSentence.are_found_in_sentence( "no more" ) ||
    mSentence.are_found_in_sentence( "that's it" ) ||
    mSentence.are_found_in_sentence( "that's all" ) ||
    mSentence.is_found_in_sentence( "no" );    
    if (orderFoundComplete)
        return 1;
    else
        return 0;
}

void
crf_decode_forward_backward(Sentence & s, CRF_Model & m, vector< map<string, double> > & tagp)
{
  CRF_Sequence cs;
  for (size_t j = 0; j < s.size(); j++) cs.add_state(crfstate(s, j));

  m.decode_forward_backward(cs, tagp);
  //  m.decode_viterbi(cs);

  for (size_t k = 0; k < s.size(); k++) s[k].prd = cs.vs[k].label;
}

void
crf_decode_nbest(Sentence & s, CRF_Model & m, 
		 vector<pair<double, vector<string> > > & nbest_seqs, int n)
{
  CRF_Sequence cs;
  for (size_t j = 0; j < s.size(); j++) cs.add_state(crfstate(s, j));

  m.decode_nbest(cs, nbest_seqs, n, 0);

  for (size_t k = 0; k < s.size(); k++) s[k].prd = cs.vs[k].label;

}