import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
public static String findPopularTitleCERTH_old(Dysco dysco){
        List<Item> items=dysco.getItems();
        List<String> textItems=new ArrayList<String>();
        Set<String> entities=new HashSet<String>();
        List<Entity> ents=dysco.getEntities();
        for(Entity ent:ents)
            entities.add(ent.getName());
//        Logger.getRootLogger().info("Title extractor (case 3): Getting candidate sentences");
        for(Item item_tmp:items){
            List<String> sentences=getSentences1_old(item_tmp.getTitle(),entities);
            textItems.addAll(sentences);
        }
            
//        Logger.getRootLogger().info("Title extractor (case 3): Finding most popular sentence");
        String title=findPopularTitleNew(textItems);
        if(((title==null)||(title.trim().length()==0))&&(textItems.size()>0))
            title=extractor.cleanText(textItems.get(0));       
        if(((title==null)||(title.trim().length()==0))&&(textItems.size()>0))
            title=textItems.get(0);
        if(((title==null)||(title.trim().length()==0))&&(items.size()>0))
            title=items.get(0).getTitle();
        return title.replaceAll("\\s{2,}", " ");
        
    }
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
public void execute(Tuple input) {
	try {
		Item item = (Item)input.getValueByField("Item");
		if(item == null)
			return;
		
		String title = item.getTitle();
		if(title != null) {
			List<Entity> entities = extract(title);
			item.setEntities(entities);
		}
		_collector.emit(new Values(item));
	}
	catch(Exception e) {
		_logger.error(e);
	}
}
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
public void prepare(@SuppressWarnings("rawtypes") Map conf, TopologyContext context, 
		OutputCollector collector) {
	_logger = Logger.getLogger(ItemIndexerBolt.class);
	
	queue = new ArrayBlockingQueue<Item>(5000);
	try {
		_solrItemHandler = SolrItemHandler.getInstance(_service);
	} catch (Exception e) {
		e.printStackTrace();
		_solrItemHandler = null;
		_logger.error(e);
	}
	
	Thread t = new Thread(new TextIndexer());
	t.start();
}
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
public static List<Item> loadItemsFromFilePlain(String filename){
    List<Item> items=new ArrayList<Item>();
    try{
        BufferedReader br=new BufferedReader(new FileReader(filename));
        String line=null;
        int i=0;
        while((line=br.readLine())!=null){
            i=i+1;
            if(line.trim()!=""){
                Item new_item=new Item();
                String[] parts=line.split("%%%");
                new_item.setId(parts[0]);
                new_item.setTitle(parts[1].replaceAll("\"",""));
                items.add(new_item);
            }
        }
        br.close();
    }
    catch(Exception ex){
        ex.printStackTrace();
    }
    return items;
}
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
@Override
public void run() {
	while(true) {
		List<Item> items = itemDAO.getLatestItems(10000);
		System.out.println("Last items: " + items.size());
		
		List<Dysco> dyscos = dyscoCreator.createDyscos(items);
		System.out.println("Total dyscos found: " + dyscos.size());
		for(Dysco dysco : dyscos) {
			System.out.println("Items of dysco " + dysco.getId() + " " +  
					dysco.getItems().size());
		}
		
		return;
	}
	
}
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
public static List<Item> loadItemsFromFile(String filename){
    List<Item> items=new ArrayList<Item>();
    try{
        BufferedReader br=new BufferedReader(new FileReader(filename));
        String line=null;
        while((line=br.readLine())!=null){
            if(line.trim()!=""){
                Item new_item=new Item();
                DBObject dbObject = (DBObject) JSON.parse(line);
                String id=(String) dbObject.get("id_str");
                new_item.setId(id);
                String text=(String) dbObject.get("text");
                new_item.setTitle(text);
                DBObject tmp_obj=(DBObject) dbObject.get("user");
                String uploader=(String) tmp_obj.get("screen_name");
                new_item.setAuthorScreenName(uploader);
                items.add(new_item);
            }
        }
        br.close();
    }
    catch(Exception ex){
        ex.printStackTrace();
    }
    return items;
}
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
/**
 * Tester for the LDA class
 * @param args no args required
 * @throws Exception
 */
public static void main(String[] args) throws Exception
{
	System.out.println("Reading input");
	BufferedReader reader = new BufferedReader(new FileReader("./resources/docs-test"));
	String line = reader.readLine();
	ArrayList<Item> documents = new ArrayList<Item>();
	while (line!=null)
	{
		Item item = new Item();
		item.setText(line);
		line = reader.readLine();
		documents.add(item);
	}
	reader.close();
	
	System.out.println("Computing LDA");
	LDA lda = new LDA();
	List<LDATopic> topics = lda.run(documents,10,10,10);
	for (LDATopic t : topics)
	{
		System.out.println(t);
	}
	System.out.println("Done!");
}
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
public List<Dysco> createDyscos(List<Item> items) {
    Map<String,Item> items_map=new HashMap<String,Item>();
    for(int i=0;i<items.size();i++)
        items_map.put(items.get(i).getId(),items.get(i));
    Vocabulary vocabulary_corpus=Vocabulary.getCorpusVocabulary(items.iterator(),true);
    vocabulary_corpus.filterByNoOfOccurrences(2);
    Vocabulary vocabulary_reference=new Vocabulary();
    vocabulary_reference.load(true);
    System.out.println("Read reference vocabulary");
    VocabularyComparator vocabulary_comparator=new VocabularyComparator(vocabulary_corpus,vocabulary_reference);
    vocabulary_comparator.outputOrdered(vocabulary_comparator.vocabulary_new_corpus.directory+vocabulary_comparator.vocabulary_new_corpus.filename_start+".vocabulary_ratios");
    System.out.println("Getting topics");
    List<Dysco> dyscos=vocabulary_comparator.getDyscosBySFIM(items_map);
    List<Dysco> finalDyscos=new ArrayList<Dysco>();
    for(Dysco dysco:dyscos){
        if(dysco.getKeywords().size()<10)
            finalDyscos.add(dysco);
    }
    return finalDyscos;
}
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
public static Vocabulary getCorpusVocabulary(Iterator<Item> postIterator,boolean keepTweetIDs){
       Vocabulary vocabulary=new Vocabulary();
       Item tmp_post;
//        while((tmp_tweet=getNextTweetFromFile())!=null){
       Iterator<String> it;
        while(postIterator.hasNext()){
            tmp_post=postIterator.next();
            List<String> tmp_tokens=TweetPreprocessor.Tokenize(tmp_post.getTitle());
//            List<String> tmp_tokens=tmp_post.getTokens();
            String tmp_term;
            it = tmp_tokens.iterator();
            while (it.hasNext()) {
                tmp_term=(String) it.next();
                if(keepTweetIDs)
                    vocabulary.increaseTermFrequency(tmp_term,tmp_post.getId());
                else
                    vocabulary.increaseTermFrequency(tmp_term);
            }
        }
        return vocabulary;
    }
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
@Override
public List<Item> getItemsSince(long date) {
    Selector query = new Selector();
    query.selectGreaterThan("publicationTime", date);
    long l = System.currentTimeMillis();
    List<String> jsonItems = mongoHandler.findMany(query, 0);
    l = System.currentTimeMillis() - l;
    System.out.println("Fetch time: " + l + " msecs");
    l = System.currentTimeMillis() - l;

    List<Item> results = new ArrayList<Item>();
    for (String json : jsonItems) {
        results.add(ItemFactory.create(json));
    }
    l = System.currentTimeMillis() - l;
    System.out.println("List time: " + l + " msecs");
    return results;
}
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
@Override
public List<Item> getUnindexedItems(int max) {
    Selector query = new Selector();
    query.select("indexed", Boolean.FALSE);
    query.select("original", Boolean.TRUE);

    List<String> jsonItems = mongoHandler.findManyNoSorting(query, max);
    List<Item> items = new ArrayList<Item>();
    Gson gson = new GsonBuilder()
            .excludeFieldsWithoutExposeAnnotation()
            .create();

    for (String json : jsonItems) {
        Item item = gson.fromJson(json, Item.class);

        items.add(item);
    }

    return items;
}
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
@Override
public List<Item> readItemsByStatus() {
    Selector query = new Selector();
    query.select("isSearched", Boolean.FALSE);
    List<String> jsonItems = mongoHandler.findMany(query, -1);
    List<Item> items = new ArrayList<Item>();
    Gson gson = new GsonBuilder()
            .excludeFieldsWithoutExposeAnnotation()
            .create();

    for (String json : jsonItems) {
        Item item = gson.fromJson(json, Item.class);

        items.add(item);
    }

    return items;
}
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
public static List<RankedTitleRGU>getTweetScores(Dysco dysco) {
	
	List<Item> items = dysco.getItems();
	List<String> allText = new ArrayList<String>(items.size());
               HashMap<String, Item> itemsMap=new HashMap<String,Item>();
	for (Item item : items) {
		allText.add(item.getTitle());
                       itemsMap.put(item.getTitle(), item);
	}
	HashMap<String, Integer> uniqueTexts = new HashMap<String, Integer>();
	for (String text : allText) {
		Integer count = uniqueTexts.get(text);
		if (count == null)
			uniqueTexts.put(text, 1);
		else
			uniqueTexts.put(text, count + 1);
	}

	HashMap<String, Integer> uniqueTerms = getUniqueTerms(dysco); //combine keywords, entities, hashtags

	// other parameters
	int numberOfTopicTerms = uniqueTerms.size();
	double a = 0.8;
	int numberOfTweets = dysco.getItems().size();

	//variable to fill in 
	List<RankedTitleRGU> tweetScores = new ArrayList<RankedTitleRGU>(numberOfTweets);
	
	for (String title : allText){
		RankedTitleRGU rankedTitle = new RankedTitleRGU(title,itemsMap.get(title));
		int numberOfDuplicatesLikeCurrentTweet = uniqueTexts.get(title);
		int numberOfTermsContainedInTweet = calculateNumberOfTermsContainedInTweet(uniqueTerms, title);
		rankedTitle.calculateScore(a, numberOfTermsContainedInTweet, numberOfTopicTerms, numberOfDuplicatesLikeCurrentTweet, numberOfTweets);
		tweetScores.add(rankedTitle);
	}

	return tweetScores;

}
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
public Set<String> getTerms(Item item){
        String tmp_item_text=item.getTitle();
        Set<String> tmp_urls=getURLs(tmp_item_text);
        Iterator it = tmp_urls.iterator();
        while (it.hasNext()) {
            String next_url=(String) it.next();
            tmp_item_text=tmp_item_text.replaceAll(next_url, "");
        }

        Set<String> tmp_set=new HashSet<String>();
        Scanner tokenize;
        String tmp_word;
        if(tmp_item_text!=null){
            tokenize = new Scanner(tmp_item_text);
            //Punctuation and white space is used for 
            //splitting the text in tokens.
            tokenize.useDelimiter(delimiters);
            while (tokenize.hasNext()) {
                tmp_word=tokenize.next().toLowerCase();
                tmp_word.replaceAll(delimiters, "");
                //The following lines try to remove punctuation.
                //Maybe it should be skipped or revised
                String[] tmp_str=tmp_word.split(punctuation);
                while((tmp_word.length()>0)&&((tmp_word.charAt(0)+"").matches(punctuation)))
                    tmp_word=tmp_word.substring(1);
                while((tmp_word.length()>0)&&((tmp_word.charAt(tmp_word.length()-1)+"").matches(punctuation)))
                    tmp_word=tmp_word.substring(0,tmp_word.length()-1);
                if(!isUserMention(tmp_word)){
    //                if((!StopWords.IsStopWord(tmp_word))&&(tmp_word.length()>=Constants.MIN_WORD_LENGTH))
                    if((tmp_word.length()>=3))
                        tmp_set.add(tmp_word);
                }
//                else if(Constants.TWEET_REMOVE_USER_MENTIONS==false)
//                    tmp_set.add(tmp_word);
            }
        }
        return tmp_set;
    }
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
public void execute(Tuple input) {
	try {
		String json = input.getStringByField(inputField);
		if(json != null) {
			Item item = ItemFactory.create(json);
			_collector.emit(tuple(item));
		}
	} catch(Exception e) {
		_logger.error("Exception: "+e.getMessage());
	}
}
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
@Override
public void execute(Tuple input) {
	Item item = (Item)input.getValueByField("Item");
	if(item == null)
		return;
	
	String title = item.getTitle();
	if(title != null) {
		List<TaggedWord> taggedSentences = tag(title);
		_collector.emit(new Values(item, taggedSentences));
	}
}
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
public String toString() {
    String dyscoString = "Id: " + id + "\n";
    dyscoString += "Creation date: " + creationDate + "\n";
    dyscoString += "Score: " + score + "\n";
    dyscoString += "Items: \n";
    for (Item item : items) {
        dyscoString += item.getId() + ":" + item.getTitle() + "\n";
    }
    dyscoString += "Entities: \n";
    for (Entity entity : entities) {
        dyscoString += entity.getName() + "@@@" + entity.getType().toString() + ":" + entity.getCont() + "\n";
    }
    dyscoString += "Tags: \n";
    Iterator<Entry<String, Double>> iteratorHashtags = hashtags.entrySet().iterator();
    while (iteratorHashtags.hasNext()) {
        Entry<String, Double> hashtag = iteratorHashtags.next();
        dyscoString += hashtag.getKey() + ":" + hashtag.getValue() + "\n";
    }
    dyscoString += "Keywords: \n";
    Iterator<Entry<String, Double>> iteratorKeywords = keywords.entrySet().iterator();
    while (iteratorKeywords.hasNext()) {
        Entry<String, Double> keyword = iteratorKeywords.next();
        dyscoString += keyword.getKey() + ":" + keyword.getValue() + "\n";
    }
    dyscoString += "SolrQuery : " + solrQueryString + "\n";
    dyscoString += "Trending : " + trending + "\n";
    dyscoString += "DyscoType : " + dyscoType.toString() + "\n";
    return dyscoString;
}
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
/**
 * Function that extracts features of an Item
 * @param item Item for which features are extracted
 * @param lang String that provides the item's language 
 * @return ItemFeatures object with the features extracted
 */
public static ItemFeatures extractFeatures(Item item,String lang) {

	ItemFeatures feat = new ItemFeatures();

	feat.id = item.getId();
	feat.itemLength = itemTitle.length();
	feat.numWords = getNumItemWords();
	feat.containsQuestionMark = containsSymbol("?");
	feat.containsExclamationMark = containsSymbol("!");
	feat.numExclamationMark = getNumSymbol("!");
	feat.numQuestionMark = getNumSymbol("?");
	feat.containsHappyEmo = containsEmo(Vars.HAPPY_EMO_PATH);
	feat.containsSadEmo = containsEmo(Vars.SAD_EMO_PATH);
	feat.numUppercaseChars = getNumUppercaseChars();
	feat.containsFirstOrderPron = containsPronoun(Vars.FIRST_PRON_PATH);
	feat.containsSecondOrderPron = containsPronoun(Vars.SECOND_PRON_PATH);
	feat.containsThirdOrderPron = containsPronoun(Vars.THIRD_PRON_PATH);
	feat.numMentions = getNumMentions();
	feat.numHashtags = getNumHashtags();
	feat.numURLs = getNumURLs();
	
	if (lang.equals("en")) {
		feat.numPosSentiWords = getNumSentiWords(Vars.POS_WORDS_ENG_PATH);
		feat.numNegSentiWords = getNumSentiWords(Vars.NEG_WORDS_ENG_PATH);
	}
	else if (lang.equals("es")) {
		feat.numPosSentiWords = getNumSentiWords(Vars.POS_WORDS_ES_PATH);
		feat.numNegSentiWords = getNumSentiWords(Vars.NEG_WORDS_ES_PATH);
	}
	else if (lang.equals("de")) {
		feat.numPosSentiWords = getNumSentiWords(Vars.POS_WORDS_DE_PATH); 
		feat.numNegSentiWords = getNumSentiWords(Vars.NEG_WORDS_DE_PATH);
	}
	feat.retweetCount = getRetweetsCount(item);

	return feat;
}
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
public SolrTopicDetectionItem(Item item) {
    id = item.getId();
    title = item.getTitle();
    
    tags = item.getTags();
    
    if (item.getEntities()!=null && item.getEntities().size()!=0)
    {
    	List<Entity> e = item.getEntities();
    	entities = new String[e.size()];
    
    	for(int i=0; i<e.size(); i++) {
    		entities[i] = e.get(i).getName();
    	}
    }
    else
    	entities = null;
    
    if (item.getLinks()!=null && item.getLinks().length!=0)
    {
    	URL[] links = item.getLinks();
    	urls = new String[links.length];
    	for(int i=0; i<links.length; i++)
    		urls[i] = links[i].toString();
    }
    else
    	urls = null;
    publicationTime = item.getPublicationTime();
}
 

import eu.socialsensor.framework.common.domain.Item; //导入依赖的package包/类
public SolrNewsFeed(Item item){
	id = item.getId();
       title = item.getTitle();
       description = item.getDescription();
       publicationTime = item.getPublicationTime();
       source = item.getUrl();
       lists = item.getList();
}