def __init__(self):
        db = utils.connect_db('Two_Pick_Too_Drunk')
        collection = db['user']
        last_update_start = datetime.today()
        reviews = 'reviewer'
        clusters = 'reviewer_cluster'
        updated = (False,1)
        recommenderer = Recommender()
        while 1:
            users  = collection.find({"last_beer_update":{"$gte":last_update_start}})
            start_time = time.time()
            for user in users:
                print user['name']
                (results,result_set) = recommenderer.recommender(user["Beers_Rated"], reviews, clusters, db)
                collection.update({"_id": user["_id"]}, {"$set": {"Recommendations": results}})
                updated = (True,1)
            end_time = time.time()
            if updated[0]:
                print 'done with updation after %.3f seconds'%(end_time-start_time)

            last_update_start = datetime.now()
            if updated[0]:
                time.sleep(updated[1]*5);
                print 'Slept for '+str(updated[1]*10) + ' seconds'
                updated = (False,updated[1])
            else:
                if updated[1] < 30:
                    updated = (False,updated[1]+1)
                    time.sleep(updated[1]*10);
                    print 'Slept for '+str(updated[1]*10) + ' seconds'
                else:
                    time.sleep(updated[1]*10);
                    print 'Slept for '+str(updated[1]*10) + ' seconds'

def authentication(ckey, csecret, atoken, atokensecret):
 consumer_key = ckey
 consumer_secret = csecret
 access_token = atoken
 access_token_secret = atokensecret

 auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
 auth.set_access_token(access_token, access_token_secret)

 api = tweepy.API(auth)
 my_tweets = api.user_timeline()
 my_first_tweet = my_tweets[0].text
 following = api.friends()
 #d = defaultdict(list)
 dict_of_followed_tweets = {}
 for friend in following:
   follow_acc = api.get_user(friend.screen_name)
   dict_of_followed_tweets[friend.screen_name] = friend.timeline()
   #x = [method for method in dir(friend) if callable(getattr(friend, method))]
 
 #return template('My fi {{dict_of_followed_tweets}}', dict_of_followed_tweets=dict_of_followed_tweets)
 recommenderObj = Recommender()
 generatedTweet = recommenderObj.generate(my_tweets, 1, following, 2, dict_of_followed_tweets)

 return template('Result: {{generatedTweetHere}}',generatedTweetHere =generatedTweet)

def unlike(request, fb_id, content_id):
    _recommender = Recommender()
    visited_content = _recommender.get_user_visited_content()
    print visited_content
    visited_content[fb_id].append(content_id)
    _recommender.save_user_visited_content(visited_content)
    return HttpResponse(status=200, content=json.dumps(visited_content), content_type="application/json")

class TestRecommender(unittest.TestCase):
    def setUp(self):
        self.recommender = Recommender('../data/sample.db') #样本数据库

    def test_recommend(self):
        self.recommender.train()
        for user in self.recommender.users:
            print 'user = {0}, {1}'.format(user, self.recommender.recommend(user, 3))

class Test(unittest.TestCase):
    """ Test class for Recommender """
    
    def setUp(self):
        """ initial set up of Test cases """
        unittest.TestCase.setUp(self)
        self.recommender = Recommender()
        test_movies = [{"movie_id" : 1, "title": "Movie1", "release_date" : "01-Jan-2011", "imdb_url":"https://someurl.com", "genres":"Drama Thriller", "ratings":0},
                       {"movie_id" : 20, "title": "Movie2", "release_date" : "01-Jan-2010", "imdb_url":"https://someurl.com", "genres":"Mystery", "ratings":0},
                       {"movie_id" : 35, "title": "Movie3", "release_date" : "01-Jan-2010", "imdb_url":"https://someurl.com", "genres":"Drama", "ratings":0},
                       {"movie_id" : 3, "title": "Movie4", "release_date" : "01-Jan-2012", "imdb_url":"https://someurl.com", "genres":"Thriller Crime", "ratings":0},
                       {"movie_id" : 4, "title": "Movie5", "release_date" : "01-Jan-2012", "imdb_url":"https://someurl.com", "genres":"Crime", "ratings":0},
                       {"movie_id" : 10, "title": "Movie6", "release_date" : "01-Jan-2012", "imdb_url":"https://someurl.com", "genres":"Western", "ratings":0},
                       {"movie_id" : 11, "title": "Movie7", "release_date" : "01-Jan-2012", "imdb_url":"https://someurl.com", "genres":"War Crime", "ratings":0}]
        movies_objects = []
        for movies in test_movies:
            movies_objects.append(Movie(movies["movie_id"], movies["title"], movies["release_date"], movies["imdb_url"], movies["genres"], 0))
        
        test_ratings = [{'movie_id': 1, 'user_id': 1, 'rating': 10, 'timestamp' : datetime.datetime.fromtimestamp(891350008)},
                        {'movie_id': 20, 'user_id': 2, 'rating': 9, 'timestamp' : datetime.datetime.fromtimestamp(891350010)},
                        {'movie_id': 35, 'user_id': 3, 'rating': 7, 'timestamp' : datetime.datetime.fromtimestamp(891350012)},
                        {'movie_id': 3, 'user_id': 4, 'rating': 4, 'timestamp' : datetime.datetime.fromtimestamp(891350014)},
                        {'movie_id': 4, 'user_id': 5, 'rating': 5, 'timestamp' : datetime.datetime.fromtimestamp(891350016)},
                        {'movie_id': 10, 'user_id': 6, 'rating': 5, 'timestamp' : datetime.datetime.fromtimestamp(891350018)},
                        {'movie_id': 11, 'user_id': 7, 'rating': 6, 'timestamp' : datetime.datetime.fromtimestamp(891350020)},
                        {'movie_id': 1, 'user_id': 7, 'rating': 10, 'timestamp' : datetime.datetime.fromtimestamp(891350008)},
                        {'movie_id': 20, 'user_id': 6, 'rating': 9, 'timestamp' : datetime.datetime.fromtimestamp(891350010)},
                        {'movie_id': 35, 'user_id': 5, 'rating': 7, 'timestamp' : datetime.datetime.fromtimestamp(891350012)},
                        {'movie_id': 3, 'user_id': 4, 'rating': 4, 'timestamp' : datetime.datetime.fromtimestamp(891350014)},
                        {'movie_id': 4, 'user_id': 3, 'rating': 5, 'timestamp' : datetime.datetime.fromtimestamp(891350016)},
                        {'movie_id': 10, 'user_id': 2, 'rating': 5, 'timestamp' : datetime.datetime.fromtimestamp(891350018)},
                        {'movie_id': 11, 'user_id': 1, 'rating': 6, 'timestamp' : datetime.datetime.fromtimestamp(891350020)}]
        
        all_genres = {"unknown":0, "Action":1, "Adventure":2, "Animation":3, "Children's":4, "Comedy":5, "Crime":6, "Documentary":7, "Drama":8, "Fantasy":9, "Film-Noir":10,
                          "Horror":11, "Musical":12, "Mystery":13, "Romance":14, "Sci-Fi":15, "Thriller":16, "War":17, "Western":18}
        users_ = ["1 24 M technician 85711","2 53 F other 94043","3 23 M writer 32067","4 24 M technician 43537","5 33 F other 15213",
                  "6 42 M executive 98101","7 57 M administrator 91344","8 36 M administrator 05201","9 29 M student 01002","10 53 M lawyer 90703"]
        all_users = []
        for each_user in users_:
            each_user.split(" ")
            all_users.append(User(each_user[0], each_user[1], each_user[2], each_user[3], each_user[4]))
        
        self.recommender.all_ratings = test_ratings
        self.recommender.all_movies = movies_objects
        self.recommender.all_users = all_users
        self.recommender.all_genre = all_genres
    
        
    def test_recommend_for(self):
        """ test case for recommend_for """
        movie = self.recommender.recommend_for("h")
        self.assertIsNone(movie)
        
        movie = self.recommender.recommend_for(100)
        self.assertIsNone(movie)
        
        movie = self.recommender.recommend_for(1)
        self.assertEqual(movie[0].get_title(), "Movie2")

def run(source, target, num_topics = 100, passes = 20, lang = 'en', distance_measure = euclidean, percentage = 0.05):
	"""
	Main entry point for this package. Contains and executes the whole data pipeline. 

	Arguments:
	source -- The path string to the source file containing all reviews
	target -- The path string to the target directory where the neighbors for all users will be saved

	Keyword arguments:
	num_topics -- The number of topics LDA is supposed to discover (default 100)
	passes -- The number of iterations for the statistical inference algorithm (default 20)
	lang -- The language the reviews shall be sorted by (default 'en')
	distance_measure -- A python function that measures the distance between two vectors in a num_topics-dimensional vector space. 
				Must take two numpy arrays and return a float. (default euclidean)
	percentage -- The cutoff for being a close neighbor, i.e. two users are close if their distance is 
			within the closest percentage percent of all distances (default 0.05) 
	"""
	with open(source) as f:
		all_reviews = []
		for line in f:
			all_reviews.append(json.loads(line))

	reviews = filter_by_language(all_reviews, lang)

	rt = ReviewTokenizer(reviews)
	rt.tokenize()

	db = DictionaryBuilder(rt.tokenized_docs)
	db.build()

	dtmb = DTMBuilder(db.dictionary, db.srcTexts)
	dtmb.build()

	ldaw = LDAWrapper(dtmb.dtm, db.dictionary)
	ldaw.run(num_topics = num_topics, passes = passes)

	modelwrapper = LDAModelWrapper(ldaw.ldamodel, db.dictionary, sortByUsers(rt.tokenized_docs))
	posteriors = modelwrapper.get_all_posteriors()

	means = {}
	for key, value in posteriors.iteritems():
		means[key] = mean(value).tolist()

	x = Recommender(means)
	y = x.calc_distances(distance_measure)

	threshhold = fivePercent(y, percentage)

	for user in means.iterkeys():
		z = x.calc_neighbors(user, distance_measure, threshhold = threshhold)
		if len(target) > 0:
			fileName = target + '/' + user + '.json'
		else:
			fileName = user + '.json'
		with open(fileName, 'w') as g:
			json.dump(z, g) 

def set_keywords(request):
    _recommender = Recommender()
    _recommender.set_keywords(Keyword.objects.keyword_array())

    for program in Program.objects.all():
        keyword_map = {}
        map(lambda (k, r): keyword_map.update({k: r}), program.keyword_set.values_list('text', 'relevancy'))
        _recommender.add_content_vector(program.key, keyword_map)

    return HttpResponse(content="OK")

class RecommenderTests(unittest2.TestCase):
    @classmethod
    def setUpClass(self):
        cfg = Config()
        cfg.popcon_index = "test_data/.sample_pxi"
        cfg.popcon_dir = "test_data/popcon_dir"
        cfg.clusters_dir = "test_data/clusters_dir"
        self.rec = Recommender(cfg)

    def test_set_strategy(self):
        self.rec.set_strategy("cb")
        self.assertIsInstance(self.rec.strategy,ContentBasedStrategy)
        self.assertEqual(self.rec.strategy.content,"full")
        self.rec.set_strategy("cbt")
        self.assertIsInstance(self.rec.strategy,ContentBasedStrategy)
        self.assertEqual(self.rec.strategy.content,"tag")
        self.rec.set_strategy("cbd")
        self.assertIsInstance(self.rec.strategy,ContentBasedStrategy)
        self.assertEqual(self.rec.strategy.content,"desc")
        self.rec.set_strategy("col")
        self.assertIsInstance(self.rec.strategy,CollaborativeStrategy)

    def test_get_recommendation(self):
        user = User({"inkscape": 1, "gimp": 1, "eog":1})
        result = self.rec.get_recommendation(user)
        self.assertIsInstance(result, RecommendationResult)
        self.assertGreater(len(result.item_score),0)

def get_recom_from_input(username, input_name, data):
    '''
    generate recommendations using input from the request form
        - INPUT:
            username str
            input_name  str
            data: input from the request form
        - OUTPUT:  results_dict 
           dict(username = username,\
                input_data = data, input_name = input_name, sorted_topics = sorted_topics_for_inputs, \
                idx = range(df_recom.shape[0]), \
                df_recom = df_recom, relevant_all=relevant_all)

        - pre-requisit:

    '''
    model_name = 'v2_2'
    fname = input_name

    relevant_all = None
    # hard code process used in v2_2 model
    func_tokenizer = TfidfVectorizer(stop_words='english').build_tokenizer()
    func_stemmer = PorterStemmer()

    # load model
    t0 = time.time()
    recommender = Recommender(model_name, func_tokenizer, func_stemmer)

    # read in input text
    cleaned_slack = pre_clean_text(func_tokenizer, data)

    W, tokenized_slacks2, test_X2, top_features_list = recommender.process_input(
        cleaned_slack)
    sorted_topics = recommender.topic_model.sorted_topics_for_articles(W)

    print 'input name: %s' % input_name
    # recommendations
    print '--------------- recommendations --------------'
    df_recom = recommender.calculate_recommendations(W, test_X2, fname)
    print sorted_topics
    t1 = time.time()
    print "finished in  %4.4f min %s " % ((t1 - t0) / 60, 'finished all processing\n')

    df_recom['topics'] = df_recom['topics'].apply(format_related_topics)

    results_dict = dict(username=username,
                        input_data=data, input_name=input_name, sorted_topics=sorted_topics,
                        idx=range(df_recom.shape[0]),
                        df_recom=df_recom, relevant_all=relevant_all)

    with open(dummy_result_pkl, 'w') as out_fh:
        pickle.dump(results_dict, out_fh)
    return results_dict

def run_strategy(cfg, sample_file):
    rec = Recommender(cfg)
    repo_size = rec.items_repository.get_doccount()
    results = ExperimentResults(repo_size)
    label = get_label(cfg)
    population_sample = []
    sample_str = sample_file.split('/')[-1]
    with open(sample_file, 'r') as f:
        for line in f.readlines():
            user_id = line.strip('\n')
            population_sample.append(
                os.path.join(cfg.popcon_dir, user_id[:2], user_id))
    sample_dir = ("results/roc-sample/%s" % sample_str)
    if not os.path.exists(sample_dir):
        os.makedirs(sample_dir)
    log_file = os.path.join(sample_dir, label["values"])

    # n iterations per population user
    for submission_file in population_sample:
        user = PopconSystem(submission_file)
        user.filter_pkg_profile(cfg.pkgs_filter)
        user.maximal_pkg_profile()
        for n in range(iterations):
            # Fill sample profile
            profile_len = len(user.pkg_profile)
            item_score = {}
            for pkg in user.pkg_profile:
                item_score[pkg] = user.item_score[pkg]
            sample = {}
            sample_size = int(profile_len * 0.9)
            for i in range(sample_size):
                key = random.choice(item_score.keys())
                sample[key] = item_score.pop(key)
            iteration_user = User(item_score)
            recommendation = rec.get_recommendation(iteration_user, repo_size)
            if hasattr(recommendation, "ranking"):
                results.add_result(recommendation.ranking, sample)

    plot_roc(results, log_file)
    plot_roc(results, log_file, 1)
    with open(log_file + "-roc.jpg.comment", 'w') as f:
        f.write("# %s\n# %s\n\n" %
                (label["description"], label["values"]))
        f.write("# roc AUC\n%.4f\n\n" % results.get_auc())
        f.write(
            "# threshold\tmean_fpr\tdev_fpr\t\tmean_tpr\tdev_tpr\t\tcoverage\n")  # noqa
        for size in results.thresholds:
            f.write("%4d\t\t%.4f\t\t%.4f\t\t%.4f\t\t%.4f\t\t%.4f\n" %
                    (size, numpy.mean(results.fpr[size]),
                     numpy.std(results.fpr[size]),
                     numpy.mean(results.recall[size]),
                     numpy.std(results.recall[size]),
                     numpy.mean(results.coverage(size))))

    def show_ratings(self, _args, ObannonsBeerDict):
        args = list()
        for arg in _args:
            args.append({"BeerId": arg, "Rating": float(_args[arg][0])})
        user_ratings = args
        db = utils.connect_db("Two_Pick_Too_Drunk")

        reviews = "obannons_reviews"
        clusters = "obannons_reviews_cluster"

        recommenderer = Recommender()
        (results, result_set) = recommenderer.recommender(user_ratings, reviews, clusters, db)
        self.render("ratings.html", OBD=ObannonsBeerDict, results=results, result_set=result_set)

    def show_ratings(self,_args,ObannonsBeerDict):
        args=list()
        for arg in _args:
            args.append({'BeerId':arg,'Rating':float(_args[arg][0]) })
        user_ratings = args
        db = utils.connect_db('Two_Pick_Too_Drunk')

        reviews = 'obannons_reviews'
        clusters = 'obannons_reviews_cluster'

        recommenderer = Recommender()
        (results,result_set) = recommenderer.recommender(user_ratings, reviews, clusters, db, 0, 15)
        self.render("ratings.html",OBD=ObannonsBeerDict,results=results, result_set = result_set)

def run_strategy(cfg, user):
    for weight in weighting:
        cfg.weight = weight[0]
        cfg.bm25_k1 = weight[1]
        rec = Recommender(cfg)
        repo_size = rec.items_repository.get_doccount()
        for proportion in sample_proportions:
            results = ExperimentResults(repo_size)
            label = get_label(cfg, proportion)
            log_file = "results/strategies/" + label["values"]
            for n in range(iterations):
                # Fill sample profile
                profile_size = len(user.pkg_profile)
                item_score = {}
                for pkg in user.pkg_profile:
                    item_score[pkg] = user.item_score[pkg]
                sample = {}
                sample_size = int(profile_size * proportion)
                for i in range(sample_size):
                    key = random.choice(item_score.keys())
                    sample[key] = item_score.pop(key)
                iteration_user = User(item_score)
                recommendation = rec.get_recommendation(
                    iteration_user, repo_size)
                write_recall_log(
                    label, n, sample, recommendation, profile_size, repo_size,
                    log_file)
                if hasattr(recommendation, "ranking"):
                    results.add_result(recommendation.ranking, sample)
            with open(log_file, 'w') as f:
                precision_10 = sum(results.precision[10]) / len(
                    results.precision[10])
                f1_10 = sum(results.f1[10]) / len(results.f1[10])
                f05_10 = sum(results.f05[10]) / len(results.f05[10])
                f.write("# %s\n# %s\n\ncoverage %d\n\n" %
                        (label["description"], label["values"],
                         recommendation.size))
                f.write("# best results (recommendation size; metric)\n")
                f.write(
                    "precision (%d; %.2f)\nf1 (%d; %.2f)\nf05 (%d; %.2f)\n\n" %
                    (results.best_precision()[0], results.best_precision()[1],
                     results.best_f1()[0], results.best_f1()[1],
                     results.best_f05()[0], results.best_f05()[1]))
                f.write("# recommendation size 10\nprecision (10; %.2f)\nf1 (10; %.2f)\nf05 (10; %.2f)" %  # noqa
                        (precision_10, f1_10, f05_10))
            precision = results.get_precision_summary()
            recall = results.get_recall_summary()
            f1 = results.get_f1_summary()
            f05 = results.get_f05_summary()
            accuracy = results.get_accuracy_summary()
            plot_summary(precision, recall, f1, f05, accuracy, log_file)

 def __init__(self, data, metric=DEFAULT_METRIC, model_size=50):
     self.items = defaultdict(dict)
     for user, ratings in data.user_ratings.iteritems():
         for item, rating in ratings.iteritems():
             self.items[item][user] = rating
     Recommender.__init__(self, self.items, metric)
     self.data = data
     self.model_size = model_size
     
     self.item_similars = {}
     if exists(data.item_based_model_path):
         self.load_model(data.item_based_model_path)
     else:
         self.learn_model(data.item_based_model_path)

    def get_Recommendations(self):
        start_time = time.time()
        print "starting"
        user= self.get_current_user()

        db = utils.connect_db('Two_Pick_Too_Drunk')

        reviews = 'reviewer'
        clusters = 'reviewer_cluster'

        recommenderer = Recommender()
        (results,result_set) = recommenderer.recommender(user.Beers_Rated, reviews, clusters, db)
        end_time = time.time()
        print 'done with updation after %.3f seconds'%(end_time-start_time)

def get_recommendations(url):
    recommend_api = 'http://chat-01.heron.safaribooks.com/chat/by-popularity?start=1&topic='
    recommended_url = 'http://www.safariflow.com/library/view/_/{fpid}/{chunk}'
    #Get the tags for the given URL
    success, result = Recommender.get_url_page_contents(url)
    if success:
        tokenizedTags = Recommender.getTags(result)
        #Get the matched topic from those keywords
        matchedTopic = Recommender.matchTopic(tokenizedTags)
        #Form the recommendation url
        apiurl = recommend_api + matchedTopic
        #fetch the safari recommendation url
        result = Recommender.fetchSafariRecommendation(apiurl)
        return True, recommended_url.format(**result)
    else:
        return False, result

class GitBook():    
    
    def init(self):
        self.recommender = Recommender()
        self.rm = ResourceManager()
        self.recommender.build_project_features()
        print 'Launched GitBook instance'
        
    def get_languages(self):
        return self.recommender.get_languages()
        
    def get_areas_of_interest(self):
        return self.recommender.get_aoi()
        
    def recommend_projects(self, languages, area_interest, difficulty):
        return self.recommender.recommend_projects(languages, area_interest, difficulty) 

def retrieve_recommendations(alphas, removed_scores_file_path, users_wanted_recommendations):
    users_recommended = {}
    print "Initializing recommender..."
    recommender = Recommender("movie_matcher/new_oliwer_data.txt", removed_scores_file_path, 0.5)
    print "Recommender initialization done."

    for alpha in alphas:
        print "Current alfa: ", alpha
        recommender.set_alpha(alpha)
        users_recommended[alpha] = {}
        i = 0
        for user in users_wanted_recommendations.keys():
            print "Recommending for user: ", users_wanted_recommendations.keys()[i]
            recommendations = recommender.recommend(users_wanted_recommendations.keys()[i])
            users_recommended[alpha][user] = recommendations
            i += 1
    print "Recommendations done."
    return users_recommended

    def __init__(self):
        db = utils.connect_db('Two_Pick_Too_Drunk')
        collection = db['user']
        last_update_start = datetime(2012, 12, 6)
        reviews = 'reviewer'
        clusters = 'reviewer_cluster'

        recommenderer = Recommender()
        while 1:
            users  = collection.find({"last_beer_update":{"$gte":last_update_start}})
            start_time = time.time()
            for user in users:
                (results,result_set) = recommenderer.recommender(user["Beers_Rated"], reviews, clusters, db)
                collection.update({"_id": user["_id"]}, {"$set": {"Recommendations": results}})
            end_time = time.time()
            print 'done with updation after %.3f seconds'%(end_time-start_time)

            last_update_start = datetime.now()
            time.sleep(10)

 def __init__(self):
     logging.info("Setting up survey...")
     self.cfg = Config()
     self.rec = Recommender(self.cfg)
     self.submissions_dir = "/var/www/AppRecommender/src/web/submissions/"
     if not os.path.exists(self.submissions_dir):
         os.makedirs(self.submissions_dir)
     self.strategies = ["cbh", "cbh_eset",
                        "knn", "knn_eset", "knn_plus",
                        "knnco"]