void paramCb(const TokenPtr& msg)
 {
     if (std::dynamic_pointer_cast<EndOfSequenceMessage const>(msg->getTokenData())) {
         return;
     }
     params_.push_back(msg->getTokenData());
 }

void IOBlock::preprocess() {
	// get rid of the beginning and end parens
	this->get_unprocessed()->erase(this->get_unprocessed()->begin());
	this->get_unprocessed()->erase(this->get_unprocessed()->end() - 1);
	if (this->action == IO_WRITE) {
		SymbolListPtr pre = SymbolListPtr(new SymbolList());
		for (auto i = this->get_unprocessed()->begin();
			 i != this->get_unprocessed()->end(); i++) {
			TokenPtr t = *i;
			if (t->get_token() != MP_COMMA) {
				SymbolPtr p = translate(t);
				pre->push_back(p);
			} else {
				// reset
				this->expressions->push_back(pre);
				pre = SymbolListPtr(new SymbolList());
			}
		}
		this->expressions->push_back(pre);
	} else if (this->action == IO_READ) {
		for (auto i = this->get_unprocessed()->begin();
			 i != this->get_unprocessed()->end(); i++) {
			TokenPtr t = *i;
			if (t->get_token() != MP_COMMA) {
				SymbolPtr p = translate(t);
				this->get_symbol_list()->push_back(p);
			}
		}
	}
}

void Document::_processBlocksItems(TokenPtr inTokenContainer) {
	if (!inTokenContainer->isContainer()) return;

	token::Container *tokens=dynamic_cast<token::Container*>(inTokenContainer.get());
	assert(tokens!=0);

	TokenGroup processed;

	for (TokenGroup::const_iterator ii=tokens->subTokens().begin(),
		iie=tokens->subTokens().end(); ii!=iie; ++ii)
	{
		if ((*ii)->text()) {
			optional<TokenPtr> subitem;
			if (!subitem) subitem=parseHeader(ii, iie);
			if (!subitem) subitem=parseHorizontalRule(ii, iie);
			if (!subitem) subitem=parseListBlock(ii, iie);
			if (!subitem) subitem=parseBlockQuote(ii, iie);
			if (!subitem) subitem=parseCodeBlock(ii, iie);

			if (subitem) {
				_processBlocksItems(*subitem);
				processed.push_back(*subitem);
				if (ii==iie) break;
				continue;
			} else processed.push_back(*ii);
		} else if ((*ii)->isContainer()) {
			_processBlocksItems(*ii);
			processed.push_back(*ii);
		}
	}
	tokens->swapSubtokens(processed);
}

/** \brief Define a binary operator.
        \param a_pCallback Pointer to the callback object
  */
void ParserXBase::DefineOprt(const TokenPtr<IOprtBin> &oprt)
{
    if (IsOprtDefined(oprt->GetIdent()))
        throw ParserError(ErrorContext(ecFUNOPRT_DEFINED, 0, oprt->GetIdent()));

    oprt->SetParent(this);
    m_OprtDef[oprt->GetIdent()] = ptr_tok_type(oprt->Clone());
}

/**
 * Style text based on the token for the length given
 */
void FbEditor::style(int length, TokenPtr token)
{
    if (token->getKind() == TokenKind::Invalid || !token->isValid()) {
        SetIndicatorCurrent(ErrorIndicator);
        IndicatorFillRange(GetEndStyled(), length);
    }
    style(length, _tokenStyles[(int)token->getKind()]);
}

void LoopBlock::generate_pre() {
	if (this->type == RPTUNTLLOOP) {
		write_raw(this->body_label + ":\n");
	} else if (this->type == WHILELOOP) {
		write_raw(this->cond_label + ":\n");
		VarType result = this->generate_expr(this->get_symbol_list());
		if (result != BOOLEAN) {
			report_error_lc("Semantic Error",
							"Conditional expression doesn't evaluate to boolean value.",
							(*this->get_symbol_list()->begin())->get_row(),
							(*this->get_symbol_list()->begin())->get_col());
		}
		write_raw("\nBRTS " + this->body_label);
		write_raw("BR " + this->exit_label);
		write_raw(this->body_label + ":\n");
	} else if (this->type == FORLOOP) {
		// parse the assignment
		// process the assignment
		AssignmentBlockPtr assignment = AssignmentBlockPtr(new AssignmentBlock(false));
		assignment->set_analyzer(this->get_analyzer());
		for (auto i = 0; i < 3; i++) {
			assignment->catch_token((*this->get_unprocessed())[i]);
		}
		// generate its code
		assignment->preprocess();
		assignment->generate_pre();
		assignment->generate_post();
		// generate the condition label
		write_raw(this->cond_label + ":\n");
		// process the ordinal expression
		AssignmentBlockPtr ordinal_expr = AssignmentBlockPtr(new AssignmentBlock(true));
		ordinal_expr->set_analyzer(this->get_analyzer());
		for (unsigned int i = 4; i < this->get_unprocessed()->size(); i++) {
			ordinal_expr->catch_token((*this->get_unprocessed())[i]);
		}
		// get the comparison components for the ordinal expr
		TokenPtr incrementer = (*this->get_unprocessed())[3];
		if (incrementer->get_token() == MP_TO) {
			ordinal_expr->catch_token(TokenPtr(new Token(MP_EQUALS, "=", -1, -1)));
		} else if (incrementer->get_token() == MP_DOWNTO) {
			ordinal_expr->catch_token(TokenPtr(new Token(MP_EQUALS, "=", -1, -1)));
		}
		ordinal_expr->catch_token((*this->get_unprocessed())[0]);
		if (incrementer->get_token() == MP_TO) {
			ordinal_expr->catch_token(TokenPtr(new Token (MP_MINUS, "-", -1, -1)));
			ordinal_expr->catch_token(TokenPtr(new Token (MP_INT_LITERAL, "1", -1, -1)));
		} else if (incrementer->get_token() == MP_DOWNTO) {
			ordinal_expr->catch_token(TokenPtr(new Token (MP_PLUS, "+", -1, -1)));
			ordinal_expr->catch_token(TokenPtr(new Token (MP_INT_LITERAL, "1", -1, -1)));
		}
		// generate its code
		ordinal_expr->preprocess();
		ordinal_expr->generate_pre();
		write_raw("\nBRFS " + this->body_label);
		write_raw("BR " + this->exit_label + "\n");
		write_raw(this->body_label + ":\n");
	}
}

TEST_F(AutoGenerateTest, ExplicitTypesAreDetected)
{
    factory.registerNodeType(GenericNodeFactory::createConstructorFromFunction(f1, "f1"));

    UUID node_id = UUIDProvider::makeUUID_without_parent("foobarbaz");
    NodeFacadeImplementationPtr node = factory.makeNode("f1", node_id, uuid_provider);

    ASSERT_TRUE(node != nullptr);
    ASSERT_EQ(node_id, node->getUUID());

    ASSERT_EQ(node->getParameters().size(), 1);

    ASSERT_EQ(node->getInputs().size(), 2 + node->getParameters().size());
    ASSERT_EQ(node->getOutputs().size(), 1 + node->getParameters().size());

    GenericValueMessage<int>::Ptr a(new GenericValueMessage<int>);
    GenericValueMessage<int>::Ptr b(new GenericValueMessage<int>);

    a->value = 23;
    b->value = 42;

    InputPtr i1 = node->getNodeHandle()->getInput(UUIDProvider::makeDerivedUUID_forced(node_id, "in_0"));
    ASSERT_TRUE(i1 != nullptr);

    InputPtr i2 = node->getNodeHandle()->getInput(UUIDProvider::makeDerivedUUID_forced(node_id, "in_1"));
    ASSERT_TRUE(i2 != nullptr);

    TokenPtr ta = std::make_shared<Token>(a);
    TokenPtr tb = std::make_shared<Token>(b);

    param::ParameterPtr p = node->getParameter("param 2");
    ASSERT_TRUE(p != nullptr);

    p->set<int>(1337);

    param::ValueParameter::Ptr vp = std::dynamic_pointer_cast<param::ValueParameter>(p);
    ASSERT_TRUE(vp != nullptr);

    i1->setToken(ta);
    i2->setToken(tb);

    NodePtr n = node->getNode();
    n->process(*node->getNodeHandle(), *n);

    OutputPtr o = node->getNodeHandle()->getOutput(UUIDProvider::makeDerivedUUID_forced(node_id, "out_0"));
    ASSERT_TRUE(o != nullptr);

    o->commitMessages(false);

    TokenPtr to = o->getToken();
    ASSERT_TRUE(to != nullptr);
    ASSERT_TRUE(to->getTokenData() != nullptr);

    GenericValueMessage<int>::ConstPtr result = std::dynamic_pointer_cast<GenericValueMessage<int> const>(to->getTokenData());
    ASSERT_TRUE(result != nullptr);

    ASSERT_EQ((a->value + b->value + vp->as<int>()), result->value);
}

void SubgraphNode::deactivation()
{
    if (deactivation_event_) {
        TokenDataConstPtr data(new connection_types::AnyMessage);
        TokenPtr token = std::make_shared<Token>(data);
        token->setActivityModifier(ActivityModifier::DEACTIVATE);
        deactivation_event_->triggerWith(token);
    }
}

void IOBlock::catch_token(TokenPtr token) {
	// no filtering, since the capture is for
	// comma separated expressions
	if (token->get_token() != MP_WRITE
		&& token->get_token() != MP_WRITELN
		&& token->get_token() != MP_READ
		&& token->get_token() != MP_READLN)
	this->get_unprocessed()->push_back(token);
}

/** \brief Add a user defined operator.
      \param a_pOprt Pointer to a unary postfix operator object. The parser will
             become the new owner of this object hence will destroy it.
  */
void ParserXBase::DefineInfixOprt(const TokenPtr<IOprtInfix> &oprt)
{
    if (IsInfixOprtDefined(oprt->GetIdent()))
        throw ParserError(ErrorContext(ecFUNOPRT_DEFINED, 0, oprt->GetIdent()));

    // Function is not added yet, add it.
    oprt->SetParent(this);
    m_InfixOprtDef[oprt->GetIdent()] = ptr_tok_type(oprt->Clone());
}

void ParserMixin::initilizeObject(const ParseContextSPtr& context, const TokenPtr& token, ObjectSPtr object)
{
    if (!token)
    {
        initilizeObject(context, object);
        return;
    }
    object->set_sourceId(context->mSourceId);
    object->set_line(token->line());
    object->set_column(token->beginColumn());
}

    void fitnessCb(const TokenPtr& data)
    {
        if (auto val = std::dynamic_pointer_cast<GenericValueMessage<double> const>(data->getTokenData())) {
            //            std::unique_lock<std::mutex> lock(data_available_mutex_);
            fitness_.push_back(val->value);

        } else if (std::dynamic_pointer_cast<EndOfSequenceMessage const>(data->getTokenData())) {
        } else {
            throw std::runtime_error("unkown message recieved: " + data->getTokenData()->typeName());
        }
    }

bool ProjectParserMixin::parseProjectStatement(const ProjectParseContextSPtr& context, const CommentSPtr& comment)
{
    TokenizerPtr& tokenizer = context->tokenizer;

    TokenPtr core;

    if (tokenizer->check(Token::TYPE_IDENTIFIER, "core"))
    {
        core = tokenizer->current();
        tokenizer->shift();
        skipComments(context);
    }

    if (tokenizer->check(Token::TYPE_IDENTIFIER, "section"))
    {
        SectionSPtr section = parseSection(context, comment);
        if (section)
        {
            context->mProject << section;
        }
    }
    else
    if (tokenizer->check(Token::TYPE_IDENTIFIER, "package"))
    {
        if (!core)
        {
            *context <<= errorMessage(context, Message::p_unknownStatment)
                         << Message::Context("package")
                         << Message::Options("core");
        }
        PackageSPtr package = parsePackage(context);
        if (!core || !package)
            return false;

        core.reset();
        context->mProject->set_corePackage(package);
    }
    else
    {
        *context <<= errorMessage(context, Message::p_unknownStatment)
                     << Message::Context("top")
                     << Message::Options("section");
        tokenizer->shift();
    }

    return true;
}

        typename parser_result<self_t, Scanner>::type
        parse(Scanner const& scan) const
        {
          if (!scan.at_end())
          {
            TokenPtr t = *scan;

            if(ReferenceCounting::strict_cast<Type> (t) != 0 &&
               lexeme_ == t->lexeme ())
            {
              Iterator save(scan.first);
              ++scan;
              return scan.create_match(1, t, save, scan.first);
            }
          }
          return scan.no_match();
        }

void PageTemplate::yield_if(const ArgumentList& arguments, const ArrayArgument::Item* arrayItem, const TokenPtr& token, std::string& result) const
{
	const IfToken* iftoken = static_cast<const IfToken*>(token.get());
	const std::string& value = get_variable(arguments, arrayItem, iftoken->data);
	if (value.empty())
		yield_tokens(arguments, arrayItem, iftoken->if_false, result);
	else
		yield_tokens(arguments, arrayItem, iftoken->if_true, result);
}

/** Add a DataSet of specified type, set it up and return pointer to it. 
  * \param inType type of DataSet to add.
  * \param metaIn DataSet MetaData.
  * \return pointer to successfully set-up DataSet or 0 if error.
  */ 
DataSet* DataSetList::AddSet(DataSet::DataType inType, MetaData const& metaIn)
{ // TODO Always generate default name if empty?
  // Do not add to a list with copies
  if (hasCopies_) {
    mprinterr("Internal Error: Attempting to add DataSet (%s) to DataSetList with copies.\n",
              metaIn.PrintName().c_str());
    return 0;
  }
  MetaData meta( metaIn );
  meta.SetEnsembleNum( ensembleNum_ );
  // Check if DataSet with same attributes already present.
  DataSet* DS = CheckForSet(meta);
  if (DS != 0) {
    mprintf("Warning: DataSet '%s' already present.\n", DS->Meta().PrintName().c_str());
    // NOTE: Should return found dataset?
    return 0; 
  }
  TokenPtr token = &(DataArray[inType]);
  if ( token->Alloc == 0) {
    mprinterr("Internal Error: No allocator for DataSet type [%s]\n", token->Description);
    return 0;
  }
  DS = (DataSet*)token->Alloc();
  if (DS==0) {
    mprinterr("Internal Error: DataSet %s memory allocation failed.\n", meta.PrintName().c_str());
    return 0;
  }
  // If 1 dim set and time series status not set, set to true.
  if (meta.TimeSeries() == MetaData::UNKNOWN_TS && DS->Ndim() == 1) {
    meta.SetTimeSeries( MetaData::IS_TS );
    // Also set dimension default
    DS->SetDim(Dimension::X, Dimension(1.0, 1.0, "Frame") );
  }
  // Set up dataset 
  if ( DS->SetMeta( meta ) ) {
    mprinterr("Error setting up data set %s.\n", meta.PrintName().c_str());
    delete DS;
    return 0;
  }

  Push_Back(DS); 
  //fprintf(stderr,"ADDED dataset %s\n",dsetName);
  return DS;
}

// Program Block stuff
void ProgramBlock::generate_pre() {
	// get the program id and set it as the open file
	TokenPtr p = *this->get_unprocessed()->begin();
	// open a file to write to
	this->get_analyzer()->open_file(p->get_lexeme());
	// generate program entry point
	write_raw("MOV SP D0");
	// push begin symbols
	for (auto i = get_symbol_list()->begin(); i != get_symbol_list()->end(); i++) {
		if (static_pointer_cast<SymData>(*i)->get_var_type() == STRING) {
			write_raw("PUSH #\"\"");
		} else if (static_pointer_cast<SymData>(*i)->get_var_type() == FLOATING) {
			write_raw("PUSH #0.0");
		} else {
			write_raw("PUSH #0");
		}
	}
	write_raw("");
}

int scanner_test(string filename) {
	cout << "[ Scanner Test ]" << endl;
	shared_ptr<Input> test_input = Input::open_file(filename);
    if (test_input != nullptr) {
        shared_ptr<Scanner> scanner = shared_ptr<Scanner>(new Scanner(test_input));
        TokenPtr t;
        do {
            t = scanner->scan_one();
            if (t == nullptr) {
                report_error("Scan Error", "Token returned was invalid");
            }
        } while (t->get_token() != MP_EOF);
        scanner->display_tokens();
    } else {
        return -1;
    }
	cout << "[ End ]" << endl;
	return 0;
}

UUID SubgraphNode::addForwardingOutput(const UUID& internal_uuid, const TokenDataConstPtr& type, const std::string& label)
{
    graph_->registerUUID(internal_uuid);

    Output* external_output = VariadicOutputs::createVariadicOutput(type, label);

    InputPtr relay = createInternalInput(type, internal_uuid, label, true);

    crossConnectLabelChange(external_output, relay.get());

    std::weak_ptr<Output> external_output_weak = std::dynamic_pointer_cast<Output>(external_output->shared_from_this());
    relay->message_set.connect([this, external_output_weak, relay](Connectable*) {
        if (auto external_output = external_output_weak.lock()) {
            TokenPtr token = relay->getToken();
            if (is_iterating_) {
                connection_types::GenericVectorMessage::Ptr vector;
                if (!external_output->hasMessage()) {
                    vector = connection_types::GenericVectorMessage::make(token->getTokenData());
                } else {
                    auto collected = external_output->getAddedToken()->getTokenData()->cloneAs<TokenData>();
                    vector = std::dynamic_pointer_cast<connection_types::GenericVectorMessage>(collected);
                }
                apex_assert(vector);
                vector->addNestedValue(token->getTokenData());
                msg::publish(external_output.get(), vector);

                external_output->setType(vector);

            } else {
                msg::publish(external_output.get(), token->getTokenData());
            }
        }
    });

    external_to_internal_inputs_[external_output->getUUID()] = relay;

    relay_to_external_output_[internal_uuid] = external_output->getUUID();

    forwarding_connector_added(relay);

    return external_output->getUUID();
}

// ------------------------------------------------------------------------------------------------
Scope::Scope(Parser& parser,bool topLevel)
{
    if(!topLevel) {
        TokenPtr t = parser.CurrentToken();
        if (t->Type() != TokenType_OPEN_BRACKET) {
            ParseError("expected open bracket",t);
        }
    }

    TokenPtr n = parser.AdvanceToNextToken();
    if(n == NULL) {
        ParseError("unexpected end of file");
    }

    // note: empty scopes are allowed
    while(n->Type() != TokenType_CLOSE_BRACKET) {
        if (n->Type() != TokenType_KEY) {
            ParseError("unexpected token, expected TOK_KEY",n);
        }

        const std::string& str = n->StringContents();
        elements.insert(ElementMap::value_type(str,new_Element(*n,parser)));

        // Element() should stop at the next Key token (or right after a Close token)
        n = parser.CurrentToken();
        if(n == NULL) {
            if (topLevel) {
                return;
            }
            ParseError("unexpected end of file",parser.LastToken());
        }
    }
}