heuristic_trainer::heuristic_trainer(container & heuristic_word_files,
			super_container & file_names_by_subject,
			std::vector<std::string> & output_names)
				: bayesian_trainer(output_names) {

	//initialise the maps with the words
	container heuristic_words;
	probability_maps = std::list<word_map>(3);
	files_to_words(heuristic_word_files.begin(), heuristic_word_files.end(),
			std::back_inserter(heuristic_words));
	for(word_map & map : probability_maps) {
		std::for_each(heuristic_words.begin(), heuristic_words.end(),
				words_to_map(map));
	}

	// calculate n and vocab size and fill in the corpera containers
	corpera_by_subject = super_container(3);
	std::set<std::string> vocab_set;
	super_container::iterator corpera_it = corpera_by_subject.begin();
	for (container & subject : file_names_by_subject) {
		files_to_words(subject.begin(), subject.end(),
				std::back_inserter<container>(*corpera_it));
		vocab_set.insert<container::iterator>(corpera_it->begin(), corpera_it->end());
		++corpera_it;
	}
}

    static
    void
    hex_to_binary (fwditer first, fwditer last, container& out)
    {
        struct table
        {
            int val[256];
            table ()
            {
                std::fill (val, val+256, 0);
                for (int i = 0; i < 10; ++i)
                    val ['0'+i] = i;
                for (int i = 0; i < 6; ++i)
                {
                    val ['a'+i] = 10 + i;
                    val ['a'+i] = 10 + i;
                }
            }
            int operator[] (int i)
            {
               return val[i];
            }
        };

        static table lut;
        out.reserve (std::distance (first, last) / 2);
        while (first != last)
        {
            auto const hi (lut[(*first++)]);
            auto const lo (lut[(*first++)]);
            out.push_back ((hi*16)+lo);
        }
    }

	void insert(It b, It e) {
		size_t d = std::distance(b, e);
		if (d == 1) {
			insert(*b);
			return;
		}

		static container merged;
		merged.resize(0);
		merged.reserve(elements.size() + d);

		if (detail::is_sorted(b, e, comp)) {
			std::merge(elements.begin(), elements.end(), b, e, std::back_inserter(merged), comp);
		}
		else {
			static container sorted;
			sorted.assign(b, e);
			std::sort(sorted.begin(), sorted.end(), comp);
			std::merge(elements.begin(), elements.end(), sorted.begin(), sorted.end(), std::back_inserter(merged), comp);
		}

		merged.swap(elements);
		iterator it = std::unique(elements.begin(), elements.end());
		elements.erase(it, elements.end());
	}

void make_vector_zeroes( container & vec, const typename container::size_type & d1)
{
	vec.resize(d1);
	for(auto it=vec.begin(); it!=vec.end(); ++it)
	{
		set_zero(*it);
	}
}

void make_vector_default( container & vec, const typename container::size_type & d1, const typename container::size_type & d2, Args... remaining_dims)
{
	vec.clear();
	vec.resize(d1);
	for(auto it=vec.begin(); it!=vec.end(); ++it)
	{
		make_vector_default(*it, d2, remaining_dims...);
	}
}

 matrix(size_type N, size_type M, container<T> vector)
 : vec(N * M),
   rows(N),
   columns(M)
 {
     if (vector.size() == N * M) {
         std::copy(vector.begin(), vector.end(), vec.begin());
     }
 }

	vector_valuer(container & vec, const value_type & val, const other_container & other_vec)
	{
		vec.resize(other_vec.size());
		auto o_it = other_vec.begin();

		for(auto it=vec.begin(); it!=vec.end(); ++it)
		{
			vector_valuer<d-1,decltype(*it),value_type,decltype(*o_it)>(*it,val,*o_it);
		}
	}

	vector_defaulter(container & vec, const other_container & other_vec)
	{
		vec.resize(other_vec.size());
		auto o_it = other_vec.begin();

		for(auto it=vec.begin(); it!=vec.end(); ++it)
		{
			vector_defaulter<d-1,decltype(*it),decltype(*o_it)>(*it,*o_it);
		}
	}

inline string Join(const string& delim, const container& c) 
{
	ostringstream ret;
	typedef typename container::const_iterator iterator;
	for( iterator q = c.begin(); q != c.end(); ++q ) {
		if( q != c.begin() ) ret << delim;
		ret << *q;
	}
	return ret.str();
}

 boost::python::list
 multimap_algorithms<ContainerTraits, Ovr>::get (container &c, index_param ix)
 {
   boost::python::list l;
   typedef BOOST_DEDUCED_TYPENAME container::iterator iter_type;
   for( iter_type index = c.lower_bound( ix ); index != c.upper_bound( ix ); ++index ){
       boost::python::object v( index->second );
       l.append( v );
   }
   return l;
 }

Country Countries::find(const Key &_k,
						SetFunc &_func,
						const container&_container)const{

	Country c;
	_func(c, _k);
	auto iter = _container.find(c);
	if (iter != _container.end())
		return *iter;
	return Country();
}

void make_vector_function( container & vec, const func_type & func, const typename container::size_type & d1)
{
	vec.clear();
	vec.reserve(d1);

	typename container::size_type i(0);

	for(i=0; i<d1; ++i)
	{
		vec.push_back(func(i));
	}
}

template <template <class> class container, class T> void print(container<T>& vec)
{
  printf("vector - size(%u) capacity(%u)\n", vec.size(), vec.capacity());

  for (auto it = vec.begin(); it != vec.end(); ++it)
    print(*it);
  
  //test const version
  printf("test const version\n");
  const container<T>& const_vec = vec;
  for (auto it : vec)
    print(it);
}

	vector_zeroer(container & vec, const other_container & other_vec)
	{
		if(!other_vec.empty())
		{
			vec.resize(1);
			set_zero(vec.front());
			vec.resize(other_vec.size(),vec.front());
		}
		else
		{
			vec.resize(0);
		}
	}

 void verify_dest(const std::string& cmd_dest, container& backup_dir) {
   
   if (not is_remote(cmd_dest)) {
     fs::path dest_dir(fs::initial_path());
     dest_dir = fs::system_complete(fs::path(cmd_dest, fs::native));
     backup_dir.destination(dest_dir.native_file_string());
     if (not fs::exists(dest_dir)) throw Backup::Error::Directory_Error("Destination directory does not exist!");
     if (not fs::is_directory(dest_dir)) throw Backup::Error::Directory_Error("Destination specified is not a directory!");
   }
   else {
     backup_dir.destination(cmd_dest);
   }
 }

void insert(container& c, int key, int val)
{
	auto itr = c.insert(make_pair(key,val));
	if(!itr.second)
	{
		cout << "re-insert " << "[" << key << "," << val << "]" << "!!!" << endl;
#if 0
		c.erase(itr.first);
		c.insert(make_pair(key,val));
#else
		itr.first->second = val;
#endif
	}
}

void container::copy(size_t i, container const &other, size_t l, size_t r)
{
    if (r <= l)
        throw std::invalid_argument(
                "big_integer: container: in function copy(): left bound is larger than the right one"
        );

    if (r - l == 1 && sz == 0) {
        sz = 1;
        data_short = *(other.data_long->begin() + l);
        return;
    }

    if (sz == 1)
        data_long = std::make_shared< std::vector<uint32_t> > (1, data_short);
    else
        real_copy();

    if (other.size() == 1)
        data_long->insert(data_long->begin() + i, other.data_short);
    else
        std::copy(other.data_long->begin() + l,
                  other.data_long->begin() + r,
                  this->data_long->begin() + i
        );

    sz = data_long->size();
}

	vector_functioner(container & vec, const func_type & func, const other_container & other_vec)
	{
		const typename container::size_type i;
		auto new_func = [&] (Args... args)
		{
			return func(i,args...);
		};

		vec.clear();
		vec.reserve(other_vec.size());
		for(i=0; i<other_vec.size(); ++i)
		{
			vector_functioner<d-1,decltype(vec[i]),decltype(new_func),decltype(other_vec[i])>
				(vec[i],new_func,other_vec[i]);
		}
	}

 static std::set<core::identifier_string> find_variable_names(const container& equations)
 {
   std::set<pbes_system::propositional_variable_instantiation> occ;
   auto oit = std::inserter(occ, occ.end());
   std::set<core::identifier_string> occ_ids;
   for(auto it = equations.begin(); it != equations.end(); ++it)
   {
     pbes_system::detail::make_find_propositional_variables_traverser<pbes_system::pbes_expression_traverser>(oit).apply(it->formula());
     occ_ids.insert(it->variable().name());
   }
   for(auto it = occ.begin(); it != occ.end(); ++it)
   {
     occ_ids.insert(it->name());
   }
   return occ_ids;
 }

 void verify_source(const std::string& cmd_source, container& backup_dir) {
   fs::path source_dir(fs::initial_path());
   source_dir = fs::system_complete(fs::path(cmd_source, fs::native));
   backup_dir.source(source_dir.native_file_string());
   if (not fs::exists(source_dir)) throw Backup::Error::Directory_Error("Source directory does not exist!");      
   if (not fs::is_directory(source_dir)) throw Backup::Error::Directory_Error("Source specified is not a directory!");
   if (fs::is_empty(source_dir)) throw Backup::Error::Directory_Error("Source directory is empty!");
 }