void FactorAnalysisStat::substractChannelStats(){
	if (verbose) cout <<"(FactorAnalysisStat) Compute and Substract Channel FA Stats... "<<endl;	
	RealVector <double> UX;UX.setSize(_supervsize); UX.setAllValues(0.0);double* ux=UX.getArray();		
		
	// Compute Occupations and Statistics
	unsigned long loc=0;
	unsigned long sent=0; 
	XLine *pline; String *pFile; 
	fileList.rewind();
	
	double *super_mean=_super_mean.getArray();
	double *N_h=_matN_h.getArray(); 
	double *S_X=_matS_X.getArray();

	while((pline=fileList.getLine())!=NULL) { 		
		while((pFile=pline->getElement())!=NULL) {
			this->getUX(UX,*pFile);
			for(unsigned long k=0;k<_supervsize;k++) 
				ux[k]+=super_mean[k];
			
			for(unsigned long k=0;k<_mixsize;k++)
				for (unsigned long i=0;i<_vsize;i++)
					S_X[loc*_supervsize+(k*_vsize+i)]-= N_h[sent*_mixsize+k]*ux[i+k*_vsize];
			sent++;
		}
		loc++;
	}
};

void FactorAnalysisStat::getXEstimate(){
	if (verbose) cout << "(FactorAnalysisStat) Compute X Estimate "<<endl;
	RealVector <double> AUX;
	AUX.setSize(_rang);
	XLine *pline;
	String *pFile;

	_matX.setAllValues(0.0);

	double *X=_matX.getArray();	
	double *U=_matU.getArray();
	double *S_X_h=_matS_X_h.getArray();
	double *aux=AUX.getArray();
	double *super_invvar=_super_invvar.getArray();
	
	fileList.rewind();
	while((pline=fileList.getLine())!=NULL) {
		while((pFile=pline->getElement())!=NULL) {
		    unsigned long sent=_ndxTable.sessionNb(*pFile);
			AUX.setAllValues(0.0);
			for(unsigned long i=0;i<_rang;i++)
				for(unsigned long k=0;k<_supervsize;k++) 
					aux[i]+= U[k*_rang+i]*super_invvar[k]*S_X_h[sent*_supervsize+k];	
			double *l_h_inv=_l_h_inv[sent].getArray();	
			for(unsigned long i=0;i<_rang;i++)
				for(unsigned long k=0;k<_rang;k++) 
					X[sent*_rang+i]+=l_h_inv[i*_rang+k]*aux[k];
			sent++;
		}
	}
};

//-------------------------------------------------------------------------
// Load the NGRAM table, selecting the nbSelected first
void NGram::load(const String filename,Config &config){
  XList input(filename,config);
  XLine *linep;                                                          // Pointor on the current test line
  input.getLine(0);
 
  unsigned long idx=0;
  while (((linep=input.getLine()) != NULL)&&(idx<getSize())){
    for (unsigned long i=0;i<getOrder();i++){
      short int a=linep->getElement(i).toLong();
      setSymbol(idx,i,a);
    }
     if (linep->getElementCount()==(getOrder()+1)){
	unsigned long count=linep->getElement(getOrder()).toLong();
	setCount(idx,count);_totalCount+=count;}
     else setCount(idx,0);
    idx++;
  }
  if (idx!=getSize()){
    cout << "WARNING ! Number of ngram in the file["<<idx<<"] < to the number requested ["<<getSize()<<"]"<<endl;
    setSize(idx);
  }
  if (verboseLevel>1){
    cout <<"load symbol table from ["<<filename <<"]"<<endl;
    showTable();
  }
}

// Information on the quantity of data available for each client
// Outputs a list with the selected files for a defined quantity of data
int ExtractTargetDataInfo(Config& config)
{
	String inputClientListFileName = config.getParam("targetIdList");
	bool fixedLabelSelectedFrame;
	String labelSelectedFrames;
	if (config.existsParam("useIdForSelectedFrame"))      // the ID of each speaker is used as labelSelectedFrame
		fixedLabelSelectedFrame=false;
	else{                                                // the label is decided by the command line and is unique for the run
		labelSelectedFrames=config.getParam("labelSelectedFrames");
		if (verbose) cout << "Computing on" << labelSelectedFrames << " label" << endl;
		fixedLabelSelectedFrame=true;
	}
	unsigned long maxFrame=config.getParam("maxFrame").toLong();
	String outputFilename=config.getParam("outputFilename");
	
	
	ofstream outputFile(outputFilename.c_str(),ios::out| ios::trunc);
	try{
		XList inputClientList(inputClientListFileName,config);          // read the Id + filenames for each client
		XLine * linep;
		if (verbose) cout << "InfoTarget" << endl;
		// *********** Target loop *****************
		while ((linep=inputClientList.getLine()) != NULL){             // linep gives the XLine with the Id of a given client and the list of files
			String *id=linep->getElement();                              // Get the Client ID (id)
			outputFile<<*id;
			String currentFile="";
			XLine featureFileListp=linep->getElements();	           // Get the list of feature file for the client (end of the line)
			if (verbose) cout << "Info model ["<<*id<<"]"<<endl;
			if (!fixedLabelSelectedFrame){                                // the ID is used as label for selecting the frame
				labelSelectedFrames=*id;
				if (debug) cout <<*id<<" is used for label selected frames"<<endl;
			}
			// label files reading - It creates, for each file and each label, a cluster of segments - will be integrated witth the featre s - asap
			SegServer segmentsServer;                                    // Reading the segmentation files for each feature input file
			LabelServer labelServer;
			initializeClusters(featureFileListp,segmentsServer,labelServer,config);           // Reading the segmentation files for each feature input file
			unsigned long codeSelectedFrame=labelServer.getLabelIndexByString(labelSelectedFrames);            // Get the index of the cluster with in interest audio segments
			SegCluster& selectedSegments=segmentsServer.getCluster(codeSelectedFrame); // Gives the cluster of the selected/used segments
			Seg *seg;                                                                  // Will give the current segment
			unsigned long frameCount=0;
			selectedSegments.rewind();                                                 // at the begin of the selected segments list
			while(((seg=selectedSegments.getSeg())!=NULL) && (frameCount<maxFrame)){   // For each of the selected segments until the amount of data is get
				frameCount+=seg->length();
				cout << seg->sourceName()<<" "<<seg->begin()<<" "<<seg->length()<<" Total time="<<frameCount<<endl;
				if (seg->sourceName()!=currentFile){
					outputFile<<" "<<seg->sourceName();
					currentFile=seg->sourceName();
				}
			}                                                                          // end of the initial Train Iteration loop
			outputFile<<endl;
			if (verbose) cout << "Save info client ["<<*id<<"]" << endl;
		}                                                                            // end of the the target loop
	} // fin try
	
	catch (Exception& e)
	{
		cout << e.toString().c_str() << endl;
	}
	return 0;
}

FactorAnalysisStat::FactorAnalysisStat(String & featFilename,FeatureServer & fs,Config & config):_ms(config),_ss(config){ // constructor for a single file
	XList faNdx;	
	XLine featLine;
	featLine.addElement(featFilename);
        faNdx.addLine()=featLine;
        _init(faNdx,fs,config);
}

//-------------------------------------------------------------------------
// Get a tab with indexes of speakers with maximum likelihood (mdtm and etf only)
void getTarClientIdx(Config & config, XList & inputList, unsigned long nbLoc, unsigned long * tarTab)
{
	XLine *linep;
	unsigned long cpt=0;
	unsigned long cpttab=0;
	double minLLK=config.getParam("minLLK").toDouble();
	double maxScore=minLLK;
	unsigned long idxTar=0;
    bool verbose=config.existsParam("verbose");
    
	while((linep=inputList.getLine())!=NULL)
	{
		double score=linep->getElement(6).toDouble();
		if (score>=maxScore)
		{
			maxScore=score;		// this is the maximum score
			idxTar=cpt;			// index is just the line
			if (verbose) {cout << "giving highest score to " << linep->getElement(1) << " "<<maxScore << endl;} 
		}                         
		if (cpt%nbLoc==(nbLoc-1))	// when changing segment
		{
			tarTab[cpttab]=idxTar;   // idx of the target goes in the tab
			if (verbose) {cout << linep->getElement(1) << " max score: "<<maxScore <<"idx: "<<idxTar<<"cpt: "<<cpt<< endl;}
			cpttab++;	
			maxScore=minLLK; 	//reset maxScore
		}
		cpt++;
	}
}

/*
 * Checks what users match a given vector of ELines and sets their ban exempt flag accordingly.
 */
void XLineManager::CheckELines()
{
    ContainerIter n = lookup_lines.find("E");

    if (n == lookup_lines.end())
        return;

    XLineLookup& ELines = n->second;

    if (ELines.empty())
        return;

    for (LocalUserList::const_iterator u2 = ServerInstance->Users->local_users.begin(); u2 != ServerInstance->Users->local_users.end(); u2++)
    {
        LocalUser* u = *u2;

        /* This uses safe iteration to ensure that if a line expires here, it doenst trash the iterator */
        LookupIter safei;

        for (LookupIter i = ELines.begin(); i != ELines.end(); )
        {
            safei = i;
            safei++;

            XLine *e = i->second;
            u->exempt = e->Matches(u);

            i = safei;
        }
    }
}

	EventReturn OnCheckKick(User *u, Channel *c, Anope::string &mask, Anope::string &reason) override
	{
		XLine *x = this->sqlines.CheckChannel(c);
		if (x)
		{
			this->sqlines.OnMatch(u, x);
			reason = x->GetReason();
			return EVENT_STOP;
		}

		return EVENT_CONTINUE;
	}

	ModResult OnUserRegister(LocalUser* user) override
	{
		// Apply lines on user connect
		XLine *rl = ServerInstance->XLines->MatchesLine("R", user);

		if (rl)
		{
			// Bang. :P
			rl->Apply(user);
			return MOD_RES_DENY;
		}
		return MOD_RES_PASSTHRU;
	}

CmdResult CommandAddLine::Handle(User* usr, std::vector<std::string>& params)
{
	XLineFactory* xlf = ServerInstance->XLines->GetFactory(params[0]);
	const std::string& setter = usr->nick;

	if (!xlf)
	{
		ServerInstance->SNO->WriteToSnoMask('d',"%s sent me an unknown ADDLINE type (%s).",setter.c_str(),params[0].c_str());
		return CMD_FAILURE;
	}

	XLine* xl = NULL;
	try
	{
		xl = xlf->Generate(ServerInstance->Time(), ConvToInt(params[4]), params[2], params[5], params[1]);
	}
	catch (ModuleException &e)
	{
		ServerInstance->SNO->WriteToSnoMask('d',"Unable to ADDLINE type %s from %s: %s", params[0].c_str(), setter.c_str(), e.GetReason().c_str());
		return CMD_FAILURE;
	}
	xl->SetCreateTime(ConvToInt(params[3]));
	if (ServerInstance->XLines->AddLine(xl, NULL))
	{
		if (xl->duration)
		{
			std::string timestr = InspIRCd::TimeString(xl->expiry);
			ServerInstance->SNO->WriteToSnoMask('X',"%s added %s%s on %s to expire on %s: %s",setter.c_str(),params[0].c_str(),params[0].length() == 1 ? "-line" : "",
					params[1].c_str(), timestr.c_str(), params[5].c_str());
		}
		else
		{
			ServerInstance->SNO->WriteToSnoMask('X',"%s added permanent %s%s on %s: %s",setter.c_str(),params[0].c_str(),params[0].length() == 1 ? "-line" : "",
					params[1].c_str(),params[5].c_str());
		}

		TreeServer* remoteserver = TreeServer::Get(usr);

		if (!remoteserver->IsBursting())
		{
			ServerInstance->XLines->ApplyLines();
		}
		return CMD_SUCCESS;
	}
	else
	{
		delete xl;
		return CMD_FAILURE;
	}
}

	void OnUserPostNick(User *user, const std::string &oldnick) override
	{
		if (!IS_LOCAL(user))
			return;

		if (!MatchOnNickChange)
			return;

		XLine *rl = ServerInstance->XLines->MatchesLine("R", user);

		if (rl)
		{
			// Bang! :D
			rl->Apply(user);
		}
	}

bool LocalUser::CheckLines(bool doZline)
{
	const char* check[] = { "G" , "K", (doZline) ? "Z" : NULL, NULL };

	if (!this->exempt)
	{
		for (int n = 0; check[n]; ++n)
		{
			XLine *r = ServerInstance->XLines->MatchesLine(check[n], this);

			if (r)
			{
				r->Apply(this);
				return true;
			}
		}
	}

	return false;
}

// applies lines, removing clients and changing nicks etc as applicable
void XLineManager::ApplyLines()
{
    LocalUserList::reverse_iterator u2 = ServerInstance->Users->local_users.rbegin();
    while (u2 != ServerInstance->Users->local_users.rend())
    {
        LocalUser* u = *u2++;

        // Don't ban people who are exempt.
        if (u->exempt)
            continue;

        for (std::vector<XLine *>::iterator i = pending_lines.begin(); i != pending_lines.end(); i++)
        {
            XLine *x = *i;
            if (x->Matches(u))
                x->Apply(u);
        }
    }

    pending_lines.clear();
}

static void ReadXLine(ServerConfig* conf, const std::string& tag, const std::string& key, XLineFactory* make)
{
	insp::flat_set<std::string> configlines;

	ConfigTagList tags = conf->ConfTags(tag);
	for(ConfigIter i = tags.first; i != tags.second; ++i)
	{
		ConfigTag* ctag = i->second;
		std::string mask;
		if (!ctag->readString(key, mask))
			throw CoreException("<"+tag+":"+key+"> missing at " + ctag->getTagLocation());
		std::string reason = ctag->getString("reason", "<Config>");
		XLine* xl = make->Generate(ServerInstance->Time(), 0, "<Config>", reason, mask);
		xl->from_config = true;
		configlines.insert(xl->Displayable());
		if (!ServerInstance->XLines->AddLine(xl, NULL))
			delete xl;
	}

	ServerInstance->XLines->ExpireRemovedConfigLines(make->GetType(), configlines);
}

//-------------------------------------------------------------------------
// Produce a tab with mean and cov by segment without the maximum score(mdtm and etf only)
void getSegmentalMeanCovWithoutMax(Config & config, XList & inputList, unsigned long nbLoc, unsigned long * tarTab, double* meanTab, double * covTab)
{
	XLine *linep;
	unsigned long cpt=0;
	unsigned long cpttab=0;
	double minLLK=config.getParam("minLLK").toDouble();
	double maxScore=minLLK;
	double meanAcc=0.0;
	double covAcc=0.0;
	unsigned long idxTar=0;
	bool verbose=config.existsParam("verbose");

	while((linep=inputList.getLine())!=NULL)
	{
		double score=linep->getElement(6).toDouble();
		if (score>=maxScore)
		{
			maxScore=score;		// this is the maximum score
			idxTar=cpt;			// index is just the line
			if (verbose) {cout << "giving highest score to " << linep->getElement(1) << " "<<maxScore << endl;} 
		}
		meanAcc+=score;			// Accumulate mean and cov
		covAcc+=(score*score);
          
		if (cpt%nbLoc==(nbLoc-1))	// when changing segment
		{	
			tarTab[cpttab]=idxTar;
			meanAcc-=maxScore;	//remove max from Stats
			covAcc-=(maxScore*maxScore);
			meanTab[cpttab]=meanAcc;
			covTab[cpttab]=covAcc;
			if (verbose) {cout << linep->getElement(1) << " max score: "<<maxScore <<"idx: "<<idxTar<<"cpt: "<<cpt<< " meanA: "<<meanAcc<<" covA: "<<covAcc<<endl;}
			cpttab++;	
			maxScore=minLLK;
			meanAcc=0.0;
			covAcc=0.0;
		}
		cpt++;
	}
}  

	void SendAkill(User *u, XLine *x) override
	{
		if (x->IsRegex() || x->HasNickOrReal())
		{
			if (!u)
			{
				/*
				 * No user (this akill was just added), and contains nick and/or realname.
				 * Find users that match and ban them.
				 */
				for (user_map::const_iterator it = UserListByNick.begin(); it != UserListByNick.end(); ++it)
					if (x->manager->Check(it->second, x))
						this->SendAkill(it->second, x);

				return;
			}

			XLine *old = x;

			if (old->manager->HasEntry("*@" + u->host))
				return;

			/* We can't akill x as it has a nick and/or realname included, so create a new akill for *@host */
			XLine *xl = new XLine("*@" + u->host, old->GetBy(), old->GetExpires(), old->GetReason(), old->id);

			old->manager->AddXLine(xl);
			x = xl;

			Log(Config->GetClient("OperServ"), "akill") << "AKILL: Added an akill for " << x->GetMask() << " because " << u->GetMask() << "#"
					<< u->realname << " matches " << old->GetMask();
		}

		/* Calculate the time left before this would expire, capping it at 2 days */
		time_t timeleft = x->GetExpires() - Anope::CurTime;

		if (timeleft > 172800 || !x->GetExpires())
			timeleft = 172800;

		UplinkSocket::Message(Config->GetClient("OperServ")) << "KLINE * " << timeleft << " " << x->GetUser() << " " << x->GetHost() << " :" << x->GetReason();
	}

void FactorAnalysisStat::estimateAndInverseLUnThreaded(){
	if (verbose) cout << "(FactorAnalysisStat) Inverse L Matrix ... "<<endl;	
	unsigned long mk;
	DoubleSquareMatrix L(_rang);	
	L.setAllValues(0.0);
	RealVector <double> AUX;
	AUX.setSize(_rang);
	unsigned long sent=0;
	XLine *pline;
	String *pFile;
	fileList.rewind();

	double *N_h=_matN_h.getArray(); 
	double *U=_matU.getArray();
	double *LV=L.getArray();
	double *super_invvar=_super_invvar.getArray();
	
	while((pline=fileList.getLine())!=NULL) {
		while((pFile=pline->getElement())!=NULL) {
			L.setAllValues(0.0);
			AUX.setAllValues(0.0);
			for(unsigned long i=0;i<_rang;i++){
				for(unsigned long j=0;j<=i;j++){
					for(unsigned long k=0;k<_supervsize;k++){
						mk=k/_vsize;
						LV[i*_rang+j]+=N_h[sent*_mixsize+mk]*super_invvar[k]*U[k*_rang+i]*U[k*_rang+j];
						}
					}
				}
			for(unsigned long i=0;i<_rang;i++)
				for(unsigned long j=i+1;j<_rang;j++) 
					LV[i*_rang+j]=LV[j*_rang+i];
				
			for(unsigned long i=0;i<_rang;i++) 
				LV[i*_rang+i]+=1.0;
			L.invert(_l_h_inv[sent]);	
			sent++;
		}
	}
};

	void OnLookupComplete(const Query *record) override
	{
		if (!user || user->Quitting())
			return;

		const ResourceRecord &ans_record = record->answers[0];
		// Replies should be in 127.0.0.0/8
		if (ans_record.rdata.find("127.") != 0)
			return;

		sockaddrs sresult;
		sresult.pton(AF_INET, ans_record.rdata);
		int result = sresult.sa4.sin_addr.s_addr >> 24;

		Blacklist::Reply *reply = blacklist.Find(result);
		if (!blacklist.replies.empty() && !reply)
			return;

		if (reply && reply->allow_account && user->Account())
			return;

		Anope::string reason = this->blacklist.reason, addr = user->ip.addr();
		reason = reason.replace_all_cs("%n", user->nick);
		reason = reason.replace_all_cs("%u", user->GetIdent());
		reason = reason.replace_all_cs("%g", user->realname);
		reason = reason.replace_all_cs("%h", user->host);
		reason = reason.replace_all_cs("%i", addr);
		reason = reason.replace_all_cs("%r", reply ? reply->reason : "");
		reason = reason.replace_all_cs("%N", Config->GetBlock("networkinfo")->Get<Anope::string>("networkname"));

		ServiceBot *OperServ = Config->GetClient("OperServ");
		creator->logger.Category("dnsbl").Bot(OperServ).Log(_("{0} ({1}) appears in {2}"), user->GetMask(), addr, this->blacklist.name);

		XLine *x = Serialize::New<XLine *>();
		x->SetMask("*@" + addr);
		x->SetBy(OperServ ? OperServ->nick : "m_dnsbl");
		x->SetCreated(Anope::CurTime);
		x->SetExpires(Anope::CurTime + this->blacklist.bantime);
		x->SetReason(reason);
		x->SetID(XLineManager::GenerateUID());

		if (this->add_to_akill && akills)
		{
			akills->AddXLine(x);
			akills->Send(NULL, x);
		}
		else
		{
			IRCD->Send<messages::Akill>(nullptr, x);
			delete x;
		}
	}

//-------------------------------------------------------------------------
// Produce a tab with mean and cov by segment (mdtm and etf only)
void getSegmentalMeanCov(XList & inputList, unsigned long nbLoc, double* meanTab, double * covTab)
{
	XLine *linep;
	unsigned long cpt=0;
	unsigned long cpttab=0;
	double meanAcc=0.0;
	double covAcc=0.0;
     
	while((linep=inputList.getLine())!=NULL)
	{
		double score=linep->getElement(6).toDouble();
		meanAcc+=score;			// Accumulate mean and cov
		covAcc+=(score*score);
		if (cpt%nbLoc==(nbLoc-1))	// when changing segment
		{
			meanTab[cpttab]=meanAcc;
			covTab[cpttab]=covAcc;
			cpttab++;	
			meanAcc=0.0;
			covAcc=0.0;
		}
		cpt++;
	}
}

void FactorAnalysisStat::substractSpeakerStats(){
	if (verbose) cout <<"(FactorAnalysisStat) Compute and Substract Speaker FA Stats... " << endl;	
	RealVector <double> AUX1;AUX1.setSize(_supervsize); AUX1.setAllValues(0.0); double *aux1=AUX1.getArray();  
	
	// Compute Occupations and Statistics
	unsigned long loc=0; 
	unsigned long sent=0; 		
	XLine *pline; String *pFile; fileList.rewind();

	double *N_h=_matN_h.getArray(); 
	double *S_X_h=_matS_X_h.getArray();

	while((pline=fileList.getLine())!=NULL) { 
		while((pFile=pline->getElement())!=NULL) {
		this->getMplusDY(AUX1,*pFile);			
			for(unsigned long k=0;k<_mixsize;k++) 
				for (unsigned long i=0;i<_vsize;i++) 
					S_X_h[sent*_supervsize+(k*_vsize+i)]-= N_h[sent*_mixsize+k]*aux1[i+k*_vsize];
			sent++;
		}
		loc++;
	}
	if (verboseLevel >1) cout << "(FactorAnalysisStat) Done "<<endl;
};