def SVDloadData():
    svd = SVD()
    recsys.algorithm.VERBOSE = True
    dat_file = '/home/commons/RecSys/MOVIEDATA/MOVIEDATA/ml-1m/ratings.dat'
    svd.load_data(filename=dat_file, sep='::', format={'col':0, 'row':1, 'value':2, 'ids': int})
    print svd.get_matrix()
    return svd

def SVDtrain2(data,pct_train):
    train, test = data.split_train_test(percent=pct_train)                                                                                                                                                                     
    K=100
    svd = SVD()
    svd.set_data(train)
    svd.compute(k=K, min_values=5, pre_normalize=None, mean_center=True,
    post_normalize=True)
    return svd,train,test

def getSimilarityMatrix(svd_model_file):
	""" Returns similarity matrix from svd_model_file
	"""
	#Import SVD from file
	svd=SVD()
	svd.load_model(svd_model_file)

	return svd.get_matrix_similarity()

def setup():
    global users, items, svd

    print 'Reading items...'
    items = _read_items(os.path.join(MOVIELENS_DATA_PATH, 'movies.dat'))
    users = []

    svd = SVD()
    svd.load_data(filename=os.path.join(MOVIELENS_DATA_PATH, 'ratings.dat'), sep='::', format={'col':0, 'row':1, 'value':2, 'ids':int})

	def build_model(self,uids,kn):
		data = Data()
		for uid,songs in uids.items():
			for song in songs:
				data.add_tuple((1,song,uid))
		svd = SVD()
		svd.set_data(data)
		svd.compute(k=kn,min_values=1)
		self.model = svd

def train_svd(data):
    """
    This method load processed data and modelling data using Singular Value Decomposition
    :return: SVD model
    """
    svd = SVD()
    svd.set_data(get_data_model_matrix(data))
    k = 30
    svd.compute(k=k, min_values=0, pre_normalize=None, mean_center=True, post_normalize=True)
    return svd

def calculate_SVD_features():
    print "Thanks for input, calculating..."
    svd = SVD()
    recsys.algorithm.VERBOSE = True
    dat_file = 'feature_matrix.csv'
    svd.load_data(filename=dat_file, sep=',', 
                format = {'col':0, 'row':1, 'value': 2, 'ids': int})
    svd.compute(k=100, min_values=0, pre_normalize=None, 
                mean_center=False, post_normalize=True)
    return svd       

def getSVD():
    filename = "/home/udaysagar/Documents/Classes/239/recsys/model/movielens.zip"
    if os.path.exists(filename):
        return SVD("./model/movielens")
    else:
        svd = SVD()
        svd.load_data(filename='./data/movielens/ratings.dat', sep='::', format={'col':0, 'row':1, 'value':2, 'ids': int})
        k = 100
        svd.compute(k=k, min_values=10, pre_normalize=None, mean_center=True, post_normalize=True, savefile='./model/movielens')
        return svd

def get_model(model_name,datasource_name,start,end,model_params):
    if not model_name in model_data:
        model_data[model_name] = (datasource_name,start,end,model_params) 
    if not os.path.exists(model_dir+model_name):
        #initialize model with new data
        svd = SVD()
        svd.load_data(filename=data_dir+datasource_name+'.csv', sep=',', format={'col':0, 'row':1, 'value':2, 'ids': int})
        models[model_name] = svd
    else:
        if not model_name in models:
            models[model_name] = SVD(filename=model_dir+model_name)

def calculate_SVD_users():
    print "Thanks for input, calculating..."
    svd = SVD()
    recsys.algorithm.VERBOSE = True
    dat_file = 'user_data_working.csv'
    svd.load_data(filename=dat_file, sep=',', 
                format = {'col':0, 'row':1, 'value': 2, 'ids': int})
    svd.compute(k=100, min_values=2, pre_normalize=None, 
                mean_center=True, post_normalize=True)
    shutil.copy('user_data_original.csv','user_data_working.csv')
    return svd

def calculate_stats_features(pct_train):
    dat_file='feature_matrix.csv'
    data = Data()
    data.load(dat_file, sep=',', format={'col':0, 'row':1, 'value':2,'ids':int})
    train, test = data.split_train_test(percent=pct_train)               
    K=100
    svd = SVD()
    svd.set_data(train)
    svd.compute(k=K, min_values=0, pre_normalize=None, mean_center=False,
    post_normalize=False)
    return svd,train,test

 def impute_to_file(self, tastings, k=100, min_values=2, verbose=True):
     # create a data file in Movielens format with the tastings data
     self.save_tastings_to_movielens_format_file(tastings)
     # for logging/testing purposes we may like this verbose
     if verbose:
         recsys.algorithm.VERBOSE = True
     svd = SVD()
     # load source data, perform SVD, save to zip file
     source_file = self.file_location(self.tastings_movielens_format)
     svd.load_data(filename=source_file, sep='::', format={'col':0, 'row':1, 'value':2, 'ids': int})
     outfile = self.file_location(self.tastings_recsys_svd)
     svd.compute(k=k, min_values=min_values, pre_normalize=None, mean_center=True, post_normalize=True, savefile=outfile)
     return svd

def create_svd_model(train):
    """ Build SVD model
    """
    
    svd = SVD()
    svd.set_data(train)
    svd.compute(k=100,
                min_values=0,
                pre_normalize=None,
                mean_center=True,
                post_normalize=True)
    
    return svd

    def __init__(self):
        #Dataset

        data = Data()
        self.filename = "emag"
        if False and os.path.isfile(self.filename + ".zip"):
            svd = SVD(filename=self.filename)
        else:
            svd = SVD()
        svd.set_data(data)
        #svd.compute(k=K, min_values=5, pre_normalize=None, mean_center=True, post_normalize=True, savefile="svd")
        self.svd = svd
        self.iterations = 0

def process_svd(preload):
    if preload:
        svd = SVD(filename='./data/svd-all') # Loading already computed SVD model
    else:
        print "Reading data..."
        svdlibc = SVDLIBC('./data/behavior-ml-score.csv')
        svdlibc.to_sparse_matrix(sep=',', format={'col':0, 'row':1, 'value':2, 'ids': str})
        k=100
        print "Computing SVD..."
        svdlibc.compute(k)
        svd = svdlibc.export()
        svd.save_model('./data/svd-all', options={'k': k})
    #svd.predict('TV268', 9, 1, 3)
    return svd

def get_movie(movie_id):
	movie = {}
	rating = 0
	with sqlite3.connect('data/data100.db') as con:
		cur = con.cursor()
		cur.execute("SELECT * FROM movies WHERE movie_id = ?", (movie_id,))
		movie_result = cur.fetchone()
		cur.execute("SELECT director FROM movie_directors WHERE movie_id = ?", (movie_id,))
		directors = cur.fetchall()
		cur.execute("SELECT actor FROM movie_actors WHERE movie_id = ?", (movie_id,))
		actors = cur.fetchall()
		cur.execute("SELECT writer FROM movie_writers WHERE movie_id = ?", (movie_id,))
		writers = cur.fetchall()
		cur.execute("SELECT genre FROM movie_genres WHERE movie_id = ?", (movie_id,))
		genres = cur.fetchall()
		if 'session_user' in request.cookies:
			cur.execute("SELECT * FROM ratings WHERE user_id = ? AND movie_id = ?", (request.get_cookie('session_user', secret='recsys')[0], movie_id,))
			rating = cur.fetchone()
		cur.execute("SELECT * FROM ratings")
		rating_results = cur.fetchall()
		d = Data()
		d.set(rating_results)
			# with open('data/tmp.dat', 'a') as f:
			# 	for l in rating_results:
			# 		f.write('%d,%d,%d\n' % (l[0], l[1], l[2]))
		svd = SVD()
			# svd.load_data(filename='data/tmp.dat', sep=',', format={'col': 0, 'row': 1, 'value': 2, 'ids':int})
		svd.set_data(d)
		similar_list = [str(s[0]) for s in svd.similar(int(movie_id))]
		cur.execute("SELECT * FROM movies WHERE movie_id IN (%s)" % (', '.join(similar_list)))
		similar_movies = cur.fetchall()
		movie = {
			'mid': movie_result[0],
			'title': movie_result[1],
			'description': movie_result[2],
			'image': movie_result[3],
			'year': movie_result[4],
			'directors': [d[0] for d in directors],
			'writers': [w[0] for w in writers],
			'actors': [a[0] for a in actors],
			'genres': [g[0] for g in genres],
			'rating': rating,
			'similar_movies': similar_movies,
		}
	session_user = request.get_cookie('session_user', secret='recsys') if 'session_user' in request.cookies else None
	return template('static/movie.html', movie=movie, session_user=session_user)

def build_svd_item_based(user_op_item_cnt, item_op_users, user_idx, item_idx, min_nonzero):
    svd = SVD()
    data = Data()
    item_lst = []
    for ui in user_op_item_cnt:
        if len(user_op_item_cnt[ui]) < min_nonzero:
            continue
        for ti in user_op_item_cnt[ui]:
            if item_op_users[ti] < min_nonzero:
                continue
            if 1.0*user_op_item_cnt[ui][ti] < 1:
                continue
            item_lst.append(ti)
            data.add_tuple(((1.0*user_op_item_cnt[ui][ti]), item_idx[ti], user_idx[ui]))
    item_lst = list(set(item_lst))
    svd.set_data(data)
    return svd, item_lst

def get_mae_rmse(step):

    data = Data()

    format = {'col': 1, 'row': 0, 'value': 2, 'ids': 'str'}

    filename = 'second_train_test.dat.{step}'.format(step=step)

    data.load(filename, sep='::', format=format)

    train, test = data.split_train_test(percent=80)

    try:

        svd = SVD('svdn_model_{step}.zip'.format(step=step))
        print('Loading model... {step}'.format(step=step))

    except:

        return

    mae_predicted, rmse_predicted = [], []
    for rating, item_id, user_id in test:
        try:

            predicted = svd.predict(item_id, user_id)

            mae_predicted.append((rating, predicted))
            rmse_predicted.append((rating, predicted))

        except:

            pass

    mae_value, rmse_value = np.nan, np.nan

    if len(mae_predicted) > 0:
        mae = MAE(mae_predicted)
        mae_value = mae.compute()

    if len(rmse_predicted) > 0:
        rmse = RMSE(rmse_predicted)
        rmse_value = rmse.compute()

    return mae_value, rmse_value

def main():
    svd = SVD()
    train = Data()
    test = Data()
    train.load('randUser/rate1.csv', force=True, sep=',', format={'col':0, 'row':1, 'value':2, 'ids':int})
    test.load('randUser/rate1.csv', force=True, sep=',', format={'col':0, 'row':1, 'value':2, 'ids':int})
    svd.set_data(train)
    svd.compute(k=100, min_values=0.5, pre_normalize=False, mean_center=True, post_normalize=True)

    # rmse = RMSE()
    # mae = MAE()
    # for rating, item_id, user_id in test.get():
    #     try:
    #         pred_rating = svd.predict(item_id, user_id)
    #         rmse.add(rating, pred_rating)
    #         mae.add(rating, pred_rating)
    #     except KeyError:
    #         continue
    # print 'RMSE=%s' % rmse.compute()
    # print 'MAE=%s' % mae.compute()

    # test = make_test()
    # print precision_and_recall(test, svd)
    # rec_list = svd.recommend(200, n=5, only_unknowns=False, is_row=False)
    print svd.recommend(1, n=5, only_unknowns=False, is_row=False)

def compute_SVD():
	svd = SVD()
	svd.set_data(load_data())

	K=100
	svd.compute(k=K, min_values=10, pre_normalize=None, mean_center=True, post_normalize=True, savefile=None)
	svd.save_model(os.path.join(utils.get_add_dir(), 'ratings'))