def _convert_hash(self, dataset):
     data = Data()
     for key in dataset:
         record = dataset[key]
         batch = [(record[k], key, k) for k in record]
         data.set(batch, extend=True)
     return data

def train_and_save(filename):

    step = filename.split('.')[-1]

    data = Data()

    format = {'col': 1, 'row': 0, 'value': 2, 'ids': 'str'}
    data.load(filename, sep='::', format=format)

    train, test = data.split_train_test(percent=80)

    try:

        svd = SVD('svdn_model_{step}.zip'.format(step=step))
        print('Already exists: svdn_model_{step}.zip'.format(step=step))

    except:

        svd = SVD()
        svd.set_data(train)

        svd.compute(
            k=100,
            min_values=2,
            pre_normalize=False,
            mean_center=True,
            post_normalize=True,
            savefile='svdn_model_{step}'.format(step=step)
        )

        print('Saved svdn_model_{step}.zip'.format(step=step))

def ex1(dat_file='./ml-1m/ratings.dat',
        pct_train=0.5):

    data = Data()
    data.load(dat_file, sep='::', format={'col':0, 'row':1, 'value':2,'ids':int})
       

    # create train/test split
    train, test = data.split_train_test(percent=pct_train)

    # create svd
    K=100
    svd = SVD()
    svd.set_data(train)
    svd.compute(k=K, min_values=5, pre_normalize=None, mean_center=True, post_normalize=True)

    # evaluate performance
    rmse = RMSE()
    mae = MAE()
    for rating, item_id, user_id in test.get():
        try:
            pred_rating = svd.predict(item_id, user_id)
            rmse.add(rating, pred_rating)
            mae.add(rating, pred_rating)
        except KeyError:
            continue

    print 'RMSE=%s' % rmse.compute()
    print 'MAE=%s' % mae.compute()

class Algorithm(object):
    def __init__(self):
        self._data = Data()

    def __repr__(self):
        s = '%d rows.' % len(self.get_data())
        if len(self.get_data()):
            s += '\nE.g: %s' % str(self.get_data()[0])
        return s

    def __len__(self):
        return len(self.get_data())

    def get_data(self):
        return self._data

    def set_data(self, data):
        self._data = data

    def add_tuple(self, tuple):
        self.get_data().add_tuple(tuple)

    def load_data(self, filename, sep='\t', format={'value':0, 'row':1, 'col':2}):
        self._data.load_file(filename, sep, format)

    def compute(self):
        if not self._data.get():
            raise ValueError('No data set. Matrix is empty!')

def get_preference(user_List):
    #generate list of users
    
    preference_dict={}
    user_map={}
    data = Data() #saving rating data
    i=1
    for user in user_List:
        user_id=(str(user))
        url = "http://api.steampowered.com/IPlayerService/GetOwnedGames/v0001/?\
key=147CBF377C6B648EC3DC73499CE73D32&steamid="+user+"&format=json"
        response = urllib2.urlopen(url)
        owned_gameData = json.loads(response.read().decode('utf-8-sig'))
        user_Pref={}
        #print (user)
        try: 
            if owned_gameData['response']['game_count']!=0:
                user_Pref={}
                for games in owned_gameData['response']['games']:
                    if games['playtime_forever']>0:
                        user_Pref[games['appid']]= math.log(games['playtime_forever'])
                        data.add_tuple((math.log(games['playtime_forever'], 10), games['appid'], i))
                        user_map[i]=user
        except:
            continue
        i=i+1
        preference_dict[user]=user_Pref
    data.save('rating.dat')

def test_data_extend():
    dataset = [(1,2,3), (4,5,6)]
    dataset2 = [(7,8,9), (10,11,12)]
    data = Data()
    data.set(dataset)
    assert_equal(len(data), 2)

    data.set(dataset2, extend=True)
    assert_equal(len(data), 4)

def load_ratings(filename):
    """ Load ratings
    """
    
    data = Data()
    format = {'col':0, 'row':1, 'value':2, 'ids': 'int'}
    data.load(filename, sep=',', format=format)
    
    return data

	def build_model(self,uids,kn):
		data = Data()
		for uid,songs in uids.items():
			for song in songs:
				data.add_tuple((1,song,uid))
		svd = SVD()
		svd.set_data(data)
		svd.compute(k=kn,min_values=1)
		self.model = svd

def getAverageRating(ITEMID):
    averageRating = 0
    totalUsers = 0
    data = Data()
    data.load('./data/movielens/ratings.dat', sep='::', format={'col':0, 'row':1, 'value':2, 'ids':int})
    for rating, item_id, user_id in data.get():
        if(item_id == ITEMID):
            totalUsers += 1
            averageRating += rating
    print averageRating/totalUsers

def calculate_stats_features(pct_train):
    dat_file='feature_matrix.csv'
    data = Data()
    data.load(dat_file, sep=',', format={'col':0, 'row':1, 'value':2,'ids':int})
    train, test = data.split_train_test(percent=pct_train)               
    K=100
    svd = SVD()
    svd.set_data(train)
    svd.compute(k=K, min_values=0, pre_normalize=None, mean_center=False,
    post_normalize=False)
    return svd,train,test

def get_data_model_matrix(data):
    """
    This method process raw data and store rating/users/movies in a matrix <value/row/column> respectively
    using recsys library
    :return: data object (recsys.datamodel.Data()) )
    """
    processed_data = Data()
    for user, review in data.items():
        for mov, rat in review.items():
            processed_data.add_tuple((rat, user, mov))
    return processed_data

def get_friend_matrix(u_ids, raw_data):
    idata = Data()
    u_idx = 0
    for u_id in u_ids:
        u_idx += 1
        i_idx = 0
        i_ids = raw_data[u_id].keys()
        for i_id in i_ids:
            i_idx += 1
            rate, ts = raw_data[u_id][i_id]
            idata.add_tuple((float(rate),u_idx,i_idx))

    return idata

def prepare_data(raw_data):
    idata = Data()
    u_idx = 0
    for u_id in raw_data.keys():
        i_idx = 0
        u_idx += 1
        pre_u_raw_data = raw_data[u_id]
        for i_id in pre_u_raw_data.keys():
            i_idx += 1
            rate, _ = pre_u_raw_data[i_id]
            idata.add_tuple((float(rate),u_idx,i_idx))

    return idata

    def setup_svd(self, vote_list):
        if self.svd is None:
            self.cache['svd'] = SVD()
            data = Data()

            for vote in vote_list:
                user_id = vote[0].id
                item_id = vote[1]
                value = float(vote[2])
                data.add_tuple((value, item_id, user_id))  # Tuple format is: <value, row, column>
            self.cache['svd'].set_data(data)
            self.cache['svd'].compute(k=self.k, min_values=1)
        return self.svd

 def setUp(self):
     
     data = Data()
     for stars, item_id, user_id in ratings:
         data.add_tuple((stars, item_id, user_id))
     
     movies = dict()
     for mid, name, genres in movie_genres:
         movie = Item(mid)
         movie.add_data({'name': name, 'genres': genres})
         movies[mid] = movie
     
     self.ratings = data
     self.movies = movies

def test_save_n_load(percent_train,
         modelKlass = SVD,
         dataFname ='/Users/jennyyuejin/recommender/Data/movieData/u.data',
         dataFormat = {'col':0, 'row':1, 'value':2, 'ids':int}):

    data = Data()
    data.load(dataFname, sep='\t', format=dataFormat)

    print '------ evaluating original'
    train, test = data.split_train_test(percent=percent_train, shuffle_data=False)
    print len(train), 'training data points;', len(test), 'testing data points'

    #Create SVD
    K=100
    svd = modelKlass()
    svd.set_data(train)
    svd.compute(k=K, min_values=5, pre_normalize=None, mean_center=True, post_normalize=True)
    evaluate(svd, test)

    svd.save_model('./model/svd.obj.zip',
                   {'k': K, 'min_values': 5,
                    'pre_normalize': None, 'mean_center': True, 'post_normalize': True})


    print '------ evaluating copy'
    data2 = Data()
    data2.load(dataFname, sep='\t', format=dataFormat)
    _, test2 = data2.split_train_test(percent=percent_train, shuffle_data=False)   # reload data
    print len(test2), 'testing data points'

    svd_pred = modelKlass()
    svd_pred.load_model('./model/svd.obj.zip')

    evaluate(svd_pred, test2)

def read_user_data_from_ratings(data_file):
    data = Data()
    format = {'col':0, 'row':1, 'value':2, 'ids': 'int'}    
    data.load(dat_file, sep='::', format=format)
    
    userdict = {}
    for d in data.get():
        if d[2] in userdict:
            user = userdict[d[2]] 
        else:
            user = User(d[2]) 
        
        user.add_item(d[1],d[0])
        userdict[d[2]] = user
    return userdict

    def update(self, USER_ID, baseline, path, pred_items):
        print "Loading tweet occurrences pickle..."
        baseline.get_data()._load_pickle(path=path + "tweet_occurrences.p")
        tweet_occurrences = baseline.get_data().get()

        print "Loading count_dict pickle..."
        count_dict = cPickle.load(open(path + "count_dict.p"))

        print "Loading occurrences pickle..."
        occurrences = cPickle.load(open(path + "occurrences.p"))

        total_count = count_dict[USER_ID]
        upd_total_count = int(total_count) + len(pred_items)
        count_dict[USER_ID] = int(upd_total_count)

        print "Dumping count_dict pickle..."
        cPickle.dump(count_dict, open(path + "count_dict.p", "wb"), 2)

        print "Updating counts for known artists..."
        for index, (count, item_id, user_id) in enumerate(tweet_occurrences):
            if str(user_id).encode('utf-8') == USER_ID:
                item_id = str(item_id).encode('utf-8')
                count = occurrences[(item_id, USER_ID)]
                upd_count = float(count) / float(upd_total_count)

                occurrences[(item_id, USER_ID)] = float(upd_count)
                baseline._matrix.set_value(item_id, USER_ID, float(upd_count))
                tweet_occurrences[index] = (float(upd_count), item_id, user_id)

        print "Updating counts for recommended artists..."
        for item_id, relevance in pred_items:
            count = (1.0 / float(upd_total_count))
            baseline._matrix.set_value(item_id, USER_ID, float(count))
            occurrences[(item_id, USER_ID)] = float(count)
            tweet_occurrences.append((float(count), item_id, USER_ID))

        print "Dumping tweet occurrences pickle..."
        data_tweet_occurrences = Data()
        data_tweet_occurrences.set(tweet_occurrences)

        baseline.set_data(data_tweet_occurrences)
        baseline.save_data(filename=path + "tweet_occurrences.p", pickle=True)

        print "Dumping occurrence pickle..."
        cPickle.dump(occurrences, open(path + "occurrences.p", "wb"), protocol=2)

        print "Dumping sparse matrix pickle..."
        cPickle.dump(baseline._matrix.get(), open(path + "sparse_matrix.p", "w"), protocol=2)

def test_utf8_data():
    data_in = Data()

    NUM_PLAYS = 69
    ITEMID = u'Bj\xf6rk' 
    data_in.add_tuple([NUM_PLAYS, ITEMID, USERID1])

    NUM_PLAYS = 34
    ITEMID = 'Björk' 
    data_in.add_tuple([NUM_PLAYS, ITEMID, USERID2])

    data_in.save(os.path.join(MOVIELENS_DATA_PATH, 'ratings.matrix.saved.utf8'))

    data_saved = Data()
    data_saved.load(os.path.join(MOVIELENS_DATA_PATH, 'ratings.matrix.saved.utf8'))

    assert_equal(len(data_in), len(data_saved))

def build_svd_item_based(user_op_item_cnt, item_op_users, user_idx, item_idx, min_nonzero):
    svd = SVD()
    data = Data()
    item_lst = []
    for ui in user_op_item_cnt:
        if len(user_op_item_cnt[ui]) < min_nonzero:
            continue
        for ti in user_op_item_cnt[ui]:
            if item_op_users[ti] < min_nonzero:
                continue
            if 1.0*user_op_item_cnt[ui][ti] < 1:
                continue
            item_lst.append(ti)
            data.add_tuple(((1.0*user_op_item_cnt[ui][ti]), item_idx[ti], user_idx[ui]))
    item_lst = list(set(item_lst))
    svd.set_data(data)
    return svd, item_lst