def test_load_feeds__gothamist(self):
        self.client.login(username='conesus', password='test')

        management.call_command('loaddata', 'gothamist_aug_2009_1.json', verbosity=0)
        feed = Feed.objects.get(feed_link__contains='gothamist')
        stories = MStory.objects(story_feed_id=feed.pk)
        self.assertEquals(stories.count(), 0)

        feed.update(force=True)

        stories = MStory.objects(story_feed_id=feed.pk)
        self.assertEquals(stories.count(), 42)

        url = reverse('load-single-feed', kwargs=dict(feed_id=4))
        response = self.client.get(url)
        content = json.decode(response.content)
        self.assertEquals(len(content['stories']), 6)

        management.call_command('loaddata', 'gothamist_aug_2009_2.json', verbosity=0)
        feed.update(force=True)

        stories = MStory.objects(story_feed_id=feed.pk)
        self.assertEquals(stories.count(), 42)

        url = reverse('load-single-feed', kwargs=dict(feed_id=4))
        response = self.client.get(url)
        # print [c['story_title'] for c in json.decode(response.content)]
        content = json.decode(response.content)
        # Test: 1 changed char in title
        self.assertEquals(len(content['stories']), 6)

    def canonical(self, full=False):
        feed = {
            'id': self.pk,
            'feed_title': self.feed_title,
            'feed_address': self.feed_address,
            'feed_link': self.feed_link,
            'updated': relative_timesince(self.last_update),
            'subs': self.num_subscribers,
            'favicon': self.icon.data,
            'favicon_color': self.icon.color,
            'favicon_fetching': bool(not (self.icon.not_found or self.icon.data))
        }
        
        if not self.fetched_once:
            feed['not_yet_fetched'] = True
        if self.has_page_exception or self.has_feed_exception:
            feed['has_exception'] = True
            feed['exception_type'] = 'feed' if self.has_feed_exception else 'page'
            feed['exception_code'] = self.exception_code
        elif full:
            feed['has_exception'] = False
            feed['exception_type'] = None
            feed['exception_code'] = self.exception_code
        
        if full:
            feed['feed_tags'] = json.decode(self.data.popular_tags) if self.data.popular_tags else []
            feed['feed_authors'] = json.decode(self.data.popular_authors) if self.data.popular_authors else []

            
        return feed

 def test_load_feeds__gothamist(self):
     self.client.login(username='conesus', password='test')
     
     management.call_command('loaddata', 'gothamist_aug_2009_1.json', verbosity=0)
     feed = Feed.objects.get(feed_link__contains='gothamist')
     stories = MStory.objects(story_feed_id=feed.pk)
     self.assertEquals(stories.count(), 0)
     
     management.call_command('refresh_feed', force=1, feed=4, single_threaded=True, daemonize=False)
     
     stories = MStory.objects(story_feed_id=feed.pk)
     self.assertEquals(stories.count(), 42)
     
     response = self.client.post('/reader/feed', { "feed_id": 4 })
     content = json.decode(response.content)
     self.assertEquals(len(content['stories']), 30)
     
     management.call_command('loaddata', 'gothamist_aug_2009_2.json', verbosity=0)
     management.call_command('refresh_feed', force=1, feed=4, single_threaded=True, daemonize=False)
     
     stories = MStory.objects(story_feed_id=feed.pk)
     self.assertEquals(stories.count(), 42)
     
     response = self.client.get('/reader/feed', { "feed_id": 4 })
     # print [c['story_title'] for c in json.decode(response.content)]
     content = json.decode(response.content)
     # Test: 1 changed char in title
     self.assertEquals(len(content['stories']), 30)

def load_feed_statistics(request, feed_id):
    user = get_user(request)
    stats = dict()
    feed = get_object_or_404(Feed, pk=feed_id)
    feed.count_subscribers()
    feed.set_next_scheduled_update(verbose=True, skip_scheduling=True)
    feed.save_feed_story_history_statistics()
    feed.save_classifier_counts()
    
    # Dates of last and next update
    stats['active'] = feed.active
    stats['last_update'] = relative_timesince(feed.last_update)
    if feed.is_push:
        stats['next_update'] = "real-time..."
    else:
        stats['next_update'] = relative_timeuntil(feed.next_scheduled_update)

    # Minutes between updates
    update_interval_minutes, _ = feed.get_next_scheduled_update(force=True, verbose=False)
    if feed.is_push:
        stats['update_interval_minutes'] = 0
    else:
        stats['update_interval_minutes'] = update_interval_minutes
    original_active_premium_subscribers = feed.active_premium_subscribers
    original_premium_subscribers = feed.premium_subscribers
    feed.active_premium_subscribers = max(feed.active_premium_subscribers+1, 1)
    feed.premium_subscribers += 1
    premium_update_interval_minutes, _ = feed.get_next_scheduled_update(force=True, verbose=False)
    feed.active_premium_subscribers = original_active_premium_subscribers
    feed.premium_subscribers = original_premium_subscribers
    if feed.is_push:
        stats['premium_update_interval_minutes'] = 0
    else:
        stats['premium_update_interval_minutes'] = premium_update_interval_minutes
    
    # Stories per month - average and month-by-month breakout
    average_stories_per_month, story_count_history = feed.average_stories_per_month, feed.data.story_count_history
    stats['average_stories_per_month'] = average_stories_per_month
    stats['story_count_history'] = story_count_history and json.decode(story_count_history)
    
    # Subscribers
    stats['subscriber_count'] = feed.num_subscribers
    stats['stories_last_month'] = feed.stories_last_month
    stats['last_load_time'] = feed.last_load_time
    stats['premium_subscribers'] = feed.premium_subscribers
    stats['active_subscribers'] = feed.active_subscribers
    stats['active_premium_subscribers'] = feed.active_premium_subscribers
    
    # Classifier counts
    stats['classifier_counts'] = json.decode(feed.data.feed_classifier_counts)
    
    # Fetch histories
    timezone = user.profile.timezone
    stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id, timezone=timezone)
    stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id, timezone=timezone)
    stats['feed_push_history'] = MFeedPushHistory.feed_history(feed_id, timezone=timezone)
    
    logging.user(request, "~FBStatistics: ~SB%s ~FG(%s/%s/%s subs)" % (feed, feed.num_subscribers, feed.active_subscribers, feed.premium_subscribers,))

    return stats

    def test_google_reader_import(self):
        self.client.login(username="conesus", password="test")
        user = User.objects.get(username="conesus")
        f = open(os.path.join(os.path.dirname(__file__), "fixtures/google_reader.xml"))
        xml = f.read()
        f.close()

        reader_importer = GoogleReaderImporter(user, xml=xml)
        reader_importer.import_feeds()

        subs = UserSubscription.objects.filter(user=user)
        self.assertEquals(subs.count(), 66)

        usf = UserSubscriptionFolders.objects.get(user=user)
        print json.decode(usf.folders)
        self.assertEquals(
            json.decode(usf.folders),
            [
                {u"Tech": [4, 5, 2, 9, 10, 12, 13, 14, 20, 23, 24, 26, 27, 28, 31, 32, 33, 34, 48, 49, 62, 64]},
                1,
                2,
                3,
                6,
                {u"Blogs": [1, 3, 25, 29, 30, 39, 40, 41, 50, 55, 57, 58, 59, 60, 66]},
                {u"Blogs \u2014 Tumblrs": [5, 21, 37, 38, 53, 54, 63, 65]},
                {u"Blogs \u2014 The Bloglets": [6, 16, 22, 35, 51, 56]},
                {u"New York": [7, 8, 17, 18, 19, 36, 45, 47, 52, 61]},
                {u"Cooking": [11, 15, 42, 43, 46]},
                44,
            ],
        )

 def move_feed_to_folder(self, feed_id, in_folder=None, to_folder=None):
     user_sub_folders = json.decode(self.folders)
     self.delete_feed(feed_id, in_folder, commit_delete=False)
     user_sub_folders = json.decode(self.folders)
     user_sub_folders = add_object_to_folder(int(feed_id), to_folder, user_sub_folders)
     self.folders = json.encode(user_sub_folders)
     self.save()
     
     return self

 def move_folder_to_folder(self, folder_name, in_folder=None, to_folder=None):
     user_sub_folders = json.decode(self.folders)
     deleted_folder = self.delete_folder(folder_name, in_folder, [], commit_delete=False)
     user_sub_folders = json.decode(self.folders)
     user_sub_folders = add_object_to_folder(deleted_folder, to_folder, user_sub_folders)
     self.folders = json.encode(user_sub_folders)
     self.save()
     
     return self

    def move_folder_to_folder(self, folder_name, in_folder=None, to_folder=None):
        logging.user(self.user, "~FBMoving folder '~SB%s~SN' in '%s' to: ~SB%s" % (folder_name, in_folder, to_folder))
        user_sub_folders = json.decode(self.folders)
        deleted_folder = self.delete_folder(folder_name, in_folder, [], commit_delete=False)
        user_sub_folders = json.decode(self.folders)
        user_sub_folders = add_object_to_folder(deleted_folder, to_folder, user_sub_folders)
        self.folders = json.encode(user_sub_folders)
        self.save()

        return self

    def move_feed_to_folder(self, feed_id, in_folder=None, to_folder=None):
        logging.user(self.user, "~FBMoving feed '~SB%s~SN' in '%s' to: ~SB%s" % (feed_id, in_folder, to_folder))
        user_sub_folders = json.decode(self.folders)
        self.delete_feed(feed_id, in_folder, commit_delete=False)
        user_sub_folders = json.decode(self.folders)
        user_sub_folders = add_object_to_folder(int(feed_id), to_folder, user_sub_folders)
        self.folders = json.encode(user_sub_folders)
        self.save()

        return self

def load_feed_statistics(request, feed_id):
    user = get_user(request)
    stats = dict()
    feed = get_object_or_404(Feed, pk=feed_id)
    feed.update_all_statistics()
    feed.set_next_scheduled_update(verbose=True, skip_scheduling=True)
    feed.save_feed_story_history_statistics()
    feed.save_classifier_counts()

    # Dates of last and next update
    stats["active"] = feed.active
    stats["last_update"] = relative_timesince(feed.last_update)
    stats["next_update"] = relative_timeuntil(feed.next_scheduled_update)
    stats["push"] = feed.is_push

    # Minutes between updates
    update_interval_minutes = feed.get_next_scheduled_update(force=True, verbose=False)
    stats["update_interval_minutes"] = update_interval_minutes
    original_active_premium_subscribers = feed.active_premium_subscribers
    original_premium_subscribers = feed.premium_subscribers
    feed.active_premium_subscribers = max(feed.active_premium_subscribers + 1, 1)
    feed.premium_subscribers += 1
    premium_update_interval_minutes = feed.get_next_scheduled_update(force=True, verbose=False)
    feed.active_premium_subscribers = original_active_premium_subscribers
    feed.premium_subscribers = original_premium_subscribers
    stats["premium_update_interval_minutes"] = premium_update_interval_minutes
    stats["errors_since_good"] = feed.errors_since_good

    # Stories per month - average and month-by-month breakout
    average_stories_per_month, story_count_history = feed.average_stories_per_month, feed.data.story_count_history
    stats["average_stories_per_month"] = average_stories_per_month
    stats["story_count_history"] = story_count_history and json.decode(story_count_history)

    # Subscribers
    stats["subscriber_count"] = feed.num_subscribers
    stats["stories_last_month"] = feed.stories_last_month
    stats["last_load_time"] = feed.last_load_time
    stats["premium_subscribers"] = feed.premium_subscribers
    stats["active_subscribers"] = feed.active_subscribers
    stats["active_premium_subscribers"] = feed.active_premium_subscribers

    # Classifier counts
    stats["classifier_counts"] = json.decode(feed.data.feed_classifier_counts)

    # Fetch histories
    timezone = user.profile.timezone
    fetch_history = MFetchHistory.feed(feed_id, timezone=timezone)
    stats["feed_fetch_history"] = fetch_history["feed_fetch_history"]
    stats["page_fetch_history"] = fetch_history["page_fetch_history"]
    stats["feed_push_history"] = fetch_history["push_history"]

    logging.user(request, "~FBStatistics: ~SB%s" % (feed))

    return stats

 def test_move_feeds_by_folder(self):
     self.client.login(username='Dejal', password='test')
     
     response = self.client.get(reverse('load-feeds'))
     feeds = json.decode(response.content)
     self.assertEquals(feeds['folders'], [5299728, 644144, 1187026, {"Brainiacs & Opinion": [569, 38, 3581, 183139, 1186180, 15]}, {"Science & Technology": [731503, 140145, 1272495, 76, 161, 39, {"Hacker": [5985150, 3323431]}]}, {"Humor": [212379, 3530, 5994357]}, {"Videos": [3240, 5168]}])
     
     # Move feeds by folder
     response = self.client.post(reverse('move-feeds-by-folder-to-folder'), {u'feeds_by_folder': u'[\n  [\n    "5994357",\n    "Humor"\n  ],\n  [\n    "3530",\n    "Humor"\n  ]\n]', u'to_folder': u'Brainiacs & Opinion'})
     response = json.decode(response.content)
     self.assertEquals(response['code'], 1)
     
     response = self.client.get(reverse('load-feeds'))
     feeds = json.decode(response.content)
     self.assertEquals(feeds['folders'], [5299728, 644144, 1187026, {"Brainiacs & Opinion": [569, 38, 3581, 183139, 1186180, 15, 5994357, 3530]}, {"Science & Technology": [731503, 140145, 1272495, 76, 161, 39, {"Hacker": [5985150, 3323431]}]}, {"Humor": [212379]}, {"Videos": [3240, 5168]}])

 def test_delete_feed__multiple_folders(self):
     self.client.login(username='conesus', password='test')
     
     response = self.client.get(reverse('load-feeds'))
     feeds = json.decode(response.content)
     self.assertEquals(feeds['folders'], [{'Tech': [1, 4, 5, {'Deep Tech': [6, 7]}]}, 2, 3, 8, 9, {'Blogs': [8, 9]}, 1])
     
     # Delete feed
     response = self.client.post(reverse('delete-feed'), {'feed_id': 1})
     response = json.decode(response.content)
     self.assertEquals(response['code'], 1)
     
     response = self.client.get(reverse('load-feeds'))
     feeds = json.decode(response.content)
     self.assertEquals(feeds['folders'], [2, 3, 8, 9, {'Tech': [1, 4, 5, {'Deep Tech': [6, 7]}]}, {'Blogs': [8, 9]}])

def clear_view_setting(request):
    code = 1
    view_setting_type = request.POST.get('view_setting_type')
    view_settings = json.decode(request.user.profile.view_settings)
    new_view_settings = {}
    removed = 0
    for feed_id, view_setting in view_settings.items():
        if view_setting_type == 'layout' and 'l' in view_setting:
            del view_setting['l']
            removed += 1
        if view_setting_type == 'view' and 'v' in view_setting:
            del view_setting['v']
            removed += 1
        if view_setting_type == 'order' and 'o' in view_setting:
            del view_setting['o']
            removed += 1
        if view_setting_type == 'order' and 'r' in view_setting:
            del view_setting['r']
            removed += 1
        new_view_settings[feed_id] = view_setting

    request.user.profile.view_settings = json.encode(new_view_settings)
    request.user.profile.save()
    
    logging.user(request, "~FMClearing view settings: %s (found %s)" % (view_setting_type, removed))
    response = dict(code=code, view_settings=view_settings, removed=removed)
    return response

 def test_load_feeds__gawker(self):
     self.client.login(username='conesus', password='test')
     
     management.call_command('loaddata', 'gawker1.json', verbosity=0)
     
     feed = Feed.objects.get(feed_link__contains='gawker')
     stories = MStory.objects(story_feed_id=feed.pk)
     self.assertEquals(stories.count(), 0)
     
     feed.update(force=True)
     
     stories = MStory.objects(story_feed_id=feed.pk)
     self.assertEquals(stories.count(), 38)
     
     management.call_command('loaddata', 'gawker2.json', verbosity=0)
     
     feed.update(force=True)
     
     # Test: 1 changed char in content
     stories = MStory.objects(story_feed_id=feed.pk)
     self.assertEquals(stories.count(), 38)
     
     url = reverse('load-single-feed', kwargs=dict(feed_id=1))
     response = self.client.get(url)
     feed = json.decode(response.content)
     self.assertEquals(len(feed['stories']), 6)

 def test_load_feeds__slashdot(self):
     self.client.login(username='conesus', password='test')
     
     management.call_command('loaddata', 'slashdot1.json', verbosity=0)
     
     feed = Feed.objects.get(feed_link__contains='slashdot')
     stories = MStory.objects(story_feed_id=feed.pk)
     self.assertEquals(stories.count(), 0)
     
     management.call_command('refresh_feed', force=1, feed=5, single_threaded=True, daemonize=False)
     
     stories = MStory.objects(story_feed_id=feed.pk)
     self.assertEquals(stories.count(), 38)
     
     management.call_command('loaddata', 'slashdot2.json', verbosity=0)
     management.call_command('refresh_feed', force=1, feed=5, single_threaded=True, daemonize=False)
     
     stories = MStory.objects(story_feed_id=feed.pk)
     self.assertEquals(stories.count(), 38)
     
     response = self.client.post('/reader/load_single_feed', { "feed_id": 5 })
     
     # pprint([c['story_title'] for c in json.decode(response.content)])
     feed = json.decode(response.content)
     
     # Test: 1 changed char in title
     self.assertEquals(len(feed['stories']), 30)

 def collect_orphan_feeds(cls, user):
     us = cls.objects.filter(user=user)
     try:
         usf = UserSubscriptionFolders.objects.get(user=user)
     except UserSubscriptionFolders.DoesNotExist:
         return
     us_feed_ids = set([sub.feed_id for sub in us])
     folders = json.decode(usf.folders)
     
     def collect_ids(folders, found_ids):
         for item in folders:
             # print ' --> %s' % item
             if isinstance(item, int):
                 # print ' --> Adding feed: %s' % item
                 found_ids.add(item)
             elif isinstance(item, dict):
                 # print ' --> Descending folder dict: %s' % item.values()
                 found_ids.update(collect_ids(item.values(), found_ids))
             elif isinstance(item, list):
                 # print ' --> Descending folder list: %s' % len(item)
                 found_ids.update(collect_ids(item, found_ids))
         # print ' --> Returning: %s' % found_ids
         return found_ids
     found_ids = collect_ids(folders, set())
     diff = len(us_feed_ids) - len(found_ids)
     if diff > 0:
         logging.info(" ---> Collecting orphans on %s. %s feeds with %s orphans" % (user.username, len(us_feed_ids), diff))
         orphan_ids = us_feed_ids - found_ids
         folders.extend(list(orphan_ids))
         usf.folders = json.encode(folders)
         usf.save()

def set_preference(request):
    code = 1
    message = ''
    new_preferences = request.POST
    
    preferences = json.decode(request.user.profile.preferences)
    for preference_name, preference_value in new_preferences.items():
        if preference_value in ['true','false']: preference_value = True if preference_value == 'true' else False
        if preference_name in SINGLE_FIELD_PREFS:
            setattr(request.user.profile, preference_name, preference_value)
        elif preference_name in SPECIAL_PREFERENCES:
            if preference_name == 'autofollow_friends':
                social_services, _ = MSocialServices.objects.get_or_create(user_id=request.user.pk)
                social_services.autofollow = preference_value
                social_services.save()
            elif preference_name == 'dashboard_date':
                request.user.profile.dashboard_date = datetime.datetime.utcnow()
        else:
            if preference_value in ["true", "false"]:
                preference_value = True if preference_value == "true" else False
            preferences[preference_name] = preference_value
        if preference_name == 'intro_page':
            logging.user(request, "~FBAdvancing intro to page ~FM~SB%s" % preference_value)
            
    request.user.profile.preferences = json.encode(preferences)
    request.user.profile.save()
    
    logging.user(request, "~FMSaving preference: %s" % new_preferences)
    response = dict(code=code, message=message, new_preferences=new_preferences)
    return response

def set_view_setting(request):
    code = 1
    feed_id = request.POST["feed_id"]
    feed_view_setting = request.POST.get("feed_view_setting")
    feed_order_setting = request.POST.get("feed_order_setting")
    feed_read_filter_setting = request.POST.get("feed_read_filter_setting")
    feed_layout_setting = request.POST.get("feed_layout_setting")
    view_settings = json.decode(request.user.profile.view_settings)

    setting = view_settings.get(feed_id, {})
    if isinstance(setting, basestring):
        setting = {"v": setting}
    if feed_view_setting:
        setting["v"] = feed_view_setting
    if feed_order_setting:
        setting["o"] = feed_order_setting
    if feed_read_filter_setting:
        setting["r"] = feed_read_filter_setting
    if feed_layout_setting:
        setting["l"] = feed_layout_setting

    view_settings[feed_id] = setting
    request.user.profile.view_settings = json.encode(view_settings)
    request.user.profile.save()

    logging.user(
        request,
        "~FMView settings: %s/%s/%s/%s"
        % (feed_view_setting, feed_order_setting, feed_read_filter_setting, feed_layout_setting),
    )
    response = dict(code=code)
    return response

def load_feed_statistics(request):
    stats = dict()
    feed_id = request.GET['feed_id']
    feed = get_object_or_404(Feed, pk=feed_id)
    feed.save_feed_story_history_statistics()
    
    # Dates of last and next update
    stats['last_update'] = relative_timesince(feed.last_update)
    stats['next_update'] = relative_timeuntil(feed.next_scheduled_update)
    
    # Minutes between updates
    update_interval_minutes, random_factor = feed.get_next_scheduled_update()
    stats['update_interval_minutes'] = update_interval_minutes
    
    # Stories per month - average and month-by-month breakout
    average_stories_per_month, story_count_history = feed.average_stories_per_month, feed.data.story_count_history
    stats['average_stories_per_month'] = average_stories_per_month
    stats['story_count_history'] = story_count_history and json.decode(story_count_history)
    
    # Subscribers
    stats['subscriber_count'] = feed.num_subscribers
    stats['stories_last_month'] = feed.stories_last_month
    stats['last_load_time'] = feed.last_load_time
    stats['premium_subscribers'] = feed.premium_subscribers
    stats['active_subscribers'] = feed.active_subscribers
    
    # Fetch histories
    stats['feed_fetch_history'] = MFeedFetchHistory.feed_history(feed_id)
    stats['page_fetch_history'] = MPageFetchHistory.feed_history(feed_id)
    
    logging.user(request.user, "~FBStatistics: ~SB%s ~FG(%s/%s/%s subs)" % (feed, feed.num_subscribers, feed.active_subscribers, feed.premium_subscribers,))

    return stats

def set_view_setting(request):
    print 'KHong profile_views set_view_setting()'
    code = 1
    feed_id = request.POST['feed_id']
    feed_view_setting = request.POST.get('feed_view_setting')
    feed_order_setting = request.POST.get('feed_order_setting')
    feed_read_filter_setting = request.POST.get('feed_read_filter_setting')
    feed_layout_setting = request.POST.get('feed_layout_setting')
    view_settings = json.decode(request.user.profile.view_settings)
    
    setting = view_settings.get(feed_id, {})
    if isinstance(setting, basestring): setting = {'v': setting}
    if feed_view_setting: setting['v'] = feed_view_setting
    if feed_order_setting: setting['o'] = feed_order_setting
    if feed_read_filter_setting: setting['r'] = feed_read_filter_setting
    if feed_layout_setting: setting['l'] = feed_layout_setting
    
    view_settings[feed_id] = setting
    request.user.profile.view_settings = json.encode(view_settings)
    request.user.profile.save()
    
    logging.user(request, "~FMView settings: %s/%s/%s/%s" % (feed_view_setting, 
                 feed_order_setting, feed_read_filter_setting, feed_layout_setting))
    response = dict(code=code)
    return response