def process_request(self, request):
        user_agent = request.environ.get('HTTP_USER_AGENT', 'missing').lower()
        
        if 'profile' in request.path: return
        if 'haproxy' in request.path: return
        if 'dbcheck' in request.path: return
        if 'account' in request.path: return
        if 'push' in request.path: return
        if getattr(settings, 'TEST_DEBUG'): return
        
        if any(ua in user_agent for ua in BANNED_USER_AGENTS):
            data = {
                'error': 'User agent banned: %s' % user_agent,
                'code': -1
            }
            logging.user(request, "~FB~SN~BBBanned UA: ~SB%s / %s (%s)" % (user_agent, request.path, request.META))
            
            return HttpResponse(json.encode(data), status=403, mimetype='text/json')

        if request.user.is_authenticated() and any(username == request.user.username for username in BANNED_USERNAMES):
            data = {
                'error': 'User banned: %s' % request.user.username,
                'code': -1
            }
            logging.user(request, "~FB~SN~BBBanned Username: ~SB%s / %s (%s)" % (request.user, request.path, request.META))
            
            return HttpResponse(json.encode(data), status=403, mimetype='text/json')

 def collect_statistics_sites_loaded(cls, last_day=None):
     if not last_day:
         last_day = datetime.datetime.now() - datetime.timedelta(hours=24)
     now = datetime.datetime.now()
     sites_loaded = []
     avg_time_taken = []
     
     for hour in range(24):
         start_hours_ago = now - datetime.timedelta(hours=hour)
         end_hours_ago = now - datetime.timedelta(hours=hour+1)
         aggregates = dict(count=Count('loadtime'), avg=Avg('loadtime'))
         load_times = FeedLoadtime.objects.filter(
             date_accessed__lte=start_hours_ago, 
             date_accessed__gte=end_hours_ago
         ).aggregate(**aggregates)
         sites_loaded.append(load_times['count'] or 0)
         avg_time_taken.append(load_times['avg'] or 0)
     sites_loaded.reverse()
     avg_time_taken.reverse()
     
     values = (
         ('sites_loaded',            json.encode(sites_loaded)),
         ('avg_time_taken',          json.encode(avg_time_taken)),
         ('latest_sites_loaded',     sites_loaded[-1]),
         ('latest_avg_time_taken',   avg_time_taken[-1]),
         ('max_sites_loaded',        max(sites_loaded)),
         ('max_avg_time_taken',      max(1, max(avg_time_taken))),
     )
     for key, value in values:
         cls.objects(key=key).update_one(upsert=True, set__key=key, set__value=value)

def _parse_user_info(user):
    return {
        "user_info": {
            "is_anonymous": json.encode(user.is_anonymous()),
            "is_authenticated": json.encode(user.is_authenticated()),
            "username": json.encode(user.username if user.is_authenticated() else "Anonymous"),
        }
    }

def _parse_user_info(user):
    return {
        'user_info': {
            'is_anonymous': json.encode(user.is_anonymous()),
            'is_authenticated': json.encode(user.is_authenticated()),
            'username': json.encode(user.username if user.is_authenticated() else 'Anonymous')
        }
    }

    def collect_statistics_sites_loaded(cls):
        now = datetime.datetime.now()
        sites_loaded = []
        avg_time_taken = []
        
        for hour in range(24):
            start_hours_ago = now - datetime.timedelta(hours=hour)
            end_hours_ago = now - datetime.timedelta(hours=hour+1)
            
            load_times = settings.MONGOANALYTICSDB.nbanalytics.page_loads.aggregate([{
                "$match": {
                    "date": {
                        "$gte": end_hours_ago,
                        "$lte": start_hours_ago,
                    },
                    "path": {
                        "$in": [
                            "/reader/feed/",
                            "/social/stories/",
                            "/reader/river_stories/",
                            "/social/river_stories/",
                        ]
                    }
                },
            }, {
                "$group": {
                    "_id"   : 1,
                    "count" : {"$sum": 1},
                    "avg"   : {"$avg": "$duration"},
                },
            }])

            count = 0
            avg = 0
            if load_times['result']:
                count = load_times['result'][0]['count']
                avg = load_times['result'][0]['avg']
                
            sites_loaded.append(count)
            avg_time_taken.append(avg)

        sites_loaded.reverse()
        avg_time_taken.reverse()
        
        values = (
            ('sites_loaded',            json.encode(sites_loaded)),
            ('avg_time_taken',          json.encode(avg_time_taken)),
            ('latest_sites_loaded',     sites_loaded[-1]),
            ('latest_avg_time_taken',   avg_time_taken[-1]),
            ('max_sites_loaded',        max(sites_loaded)),
            ('max_avg_time_taken',      max(1, max(avg_time_taken))),
        )
        for key, value in values:
            cls.objects(key=key).update_one(upsert=True, set__key=key, set__value=value)

    def collect_statistics_for_db(cls):
        lag = db_functions.mongo_max_replication_lag(settings.MONGODB)
        cls.set('mongodb_replication_lag', lag)
        
        now = round_time(datetime.datetime.now(), round_to=60)
        r = redis.Redis(connection_pool=settings.REDIS_STATISTICS_POOL)
        db_times = {}
        latest_db_times = {}
        
        for db in ['sql', 'mongo', 'redis', 'task_sql', 'task_mongo', 'task_redis']:
            db_times[db] = []
            for hour in range(24):
                start_hours_ago = now - datetime.timedelta(hours=hour+1)
    
                pipe = r.pipeline()
                for m in range(60):
                    minute = start_hours_ago + datetime.timedelta(minutes=m)
                    key = "DB:%s:%s" % (db, minute.strftime('%s'))
                    pipe.get("%s:c" % key)
                    pipe.get("%s:t" % key)
    
                times = pipe.execute()
    
                counts = [int(c or 0) for c in times[::2]]
                avgs = [float(a or 0) for a in times[1::2]]
                if counts and avgs:
                    count = sum(counts)
                    avg = round(sum(avgs) / count, 3) if count else 0
                else:
                    count = 0
                    avg = 0
                
                if hour == 0:
                    latest_count = float(counts[-1]) if len(counts) else 0
                    latest_avg = float(avgs[-1]) if len(avgs) else 0
                    latest_db_times[db] = latest_avg / latest_count if latest_count else 0
                db_times[db].append(avg)

            db_times[db].reverse()

        values = (
            ('avg_sql_times',           json.encode(db_times['sql'])),
            ('avg_mongo_times',         json.encode(db_times['mongo'])),
            ('avg_redis_times',         json.encode(db_times['redis'])),
            ('latest_sql_avg',          latest_db_times['sql']),
            ('latest_mongo_avg',        latest_db_times['mongo']),
            ('latest_redis_avg',        latest_db_times['redis']),
            ('latest_task_sql_avg',     latest_db_times['task_sql']),
            ('latest_task_mongo_avg',   latest_db_times['task_mongo']),
            ('latest_task_redis_avg',   latest_db_times['task_redis']),
        )
        for key, value in values:
            cls.objects(key=key).update_one(upsert=True, set__key=key, set__value=value)

 def _view(request, *args, **kwargs):
     if request.user.is_anonymous():
         return HttpResponse(content=json.encode({
             "message": "You must have a valid OAuth token.",
         }), status=401)
     else:
         try:
             setattr(request, 'body_json', json.decode(request.body))
         except JSONDecodeError:
             return HttpResponse(content=json.encode({
                 "message": "Your JSON body is malformed.",
             }), status=400)
         return view_func(request, *args, **kwargs)

    def collect_statistics_sites_loaded(cls):
        now = round_time(datetime.datetime.now(), round_to=60)
        sites_loaded = []
        avg_time_taken = []
        last_5_min_time_taken = 0
        r = redis.Redis(connection_pool=settings.REDIS_STATISTICS_POOL)

        for hour in range(24):
            start_hours_ago = now - datetime.timedelta(hours=hour+1)
    
            pipe = r.pipeline()
            for m in range(60):
                minute = start_hours_ago + datetime.timedelta(minutes=m)
                key = "%s:%s" % (RStats.stats_type('page_load'), minute.strftime('%s'))
                pipe.get("%s:s" % key)
                pipe.get("%s:a" % key)
    
            times = pipe.execute()
    
            counts = [int(c) for c in times[::2] if c]
            avgs = [float(a) for a in times[1::2] if a]
            
            if hour == 0:
                last_5_min_time_taken = round(sum(avgs[:1]) / max(1, sum(counts[:1])), 2)
                
            if counts and avgs:
                count = max(1, sum(counts))
                avg = round(sum(avgs) / count, 3)
            else:
                count = 0
                avg = 0

            sites_loaded.append(count)
            avg_time_taken.append(avg)

        sites_loaded.reverse()
        avg_time_taken.reverse()

        values = (
            ('sites_loaded',            json.encode(sites_loaded)),
            ('avg_time_taken',          json.encode(avg_time_taken)),
            ('latest_sites_loaded',     sites_loaded[-1]),
            ('latest_avg_time_taken',   avg_time_taken[-1]),
            ('max_sites_loaded',        max(sites_loaded)),
            ('max_avg_time_taken',      max(1, max(avg_time_taken))),
            ('last_5_min_time_taken',   last_5_min_time_taken),
        )
        for key, value in values:
            cls.objects(key=key).update_one(upsert=True, set__key=key, set__value=value)

 def process(self):
     outline = opml.from_string(self.opml_xml)
     self.clear_feeds()
     folders = self.process_outline(outline)
     UserSubscriptionFolders.objects.create(user=self.user, folders=json.encode(folders))
     self.queue_new_feeds()
     return folders

def set_view_setting(request):
    print 'KHong profile_views set_view_setting()'
    code = 1
    feed_id = request.POST['feed_id']
    feed_view_setting = request.POST.get('feed_view_setting')
    feed_order_setting = request.POST.get('feed_order_setting')
    feed_read_filter_setting = request.POST.get('feed_read_filter_setting')
    feed_layout_setting = request.POST.get('feed_layout_setting')
    view_settings = json.decode(request.user.profile.view_settings)
    
    setting = view_settings.get(feed_id, {})
    if isinstance(setting, basestring): setting = {'v': setting}
    if feed_view_setting: setting['v'] = feed_view_setting
    if feed_order_setting: setting['o'] = feed_order_setting
    if feed_read_filter_setting: setting['r'] = feed_read_filter_setting
    if feed_layout_setting: setting['l'] = feed_layout_setting
    
    view_settings[feed_id] = setting
    request.user.profile.view_settings = json.encode(view_settings)
    request.user.profile.save()
    
    logging.user(request, "~FMView settings: %s/%s/%s/%s" % (feed_view_setting, 
                 feed_order_setting, feed_read_filter_setting, feed_layout_setting))
    response = dict(code=code)
    return response

def add_site(request, token):
    code = 0
    url = request.GET['url']
    folder = request.GET['folder']
    callback = request.GET['callback']
    
    if not url:
        code = -1
    else:
        try:
            profile = Profile.objects.get(secret_token=token)
            code, message, us = UserSubscription.add_subscription(
                user=profile.user, 
                feed_address=url,
                folder=folder,
                bookmarklet=True
            )
        except Profile.DoesNotExist:
            code = -1
    
    if code > 0:
        message = 'OK'
        
    return HttpResponse(callback + '(' + json.encode({
        'code':    code,
        'message': message,
        'usersub': us and us.feed.pk,
    }) + ')', mimetype='text/plain')

def clear_view_setting(request):
    code = 1
    view_setting_type = request.POST.get('view_setting_type')
    view_settings = json.decode(request.user.profile.view_settings)
    new_view_settings = {}
    removed = 0
    for feed_id, view_setting in view_settings.items():
        if view_setting_type == 'layout' and 'l' in view_setting:
            del view_setting['l']
            removed += 1
        if view_setting_type == 'view' and 'v' in view_setting:
            del view_setting['v']
            removed += 1
        if view_setting_type == 'order' and 'o' in view_setting:
            del view_setting['o']
            removed += 1
        if view_setting_type == 'order' and 'r' in view_setting:
            del view_setting['r']
            removed += 1
        new_view_settings[feed_id] = view_setting

    request.user.profile.view_settings = json.encode(new_view_settings)
    request.user.profile.save()
    
    logging.user(request, "~FMClearing view settings: %s (found %s)" % (view_setting_type, removed))
    response = dict(code=code, view_settings=view_settings, removed=removed)
    return response

def opml_upload(request):
    xml_opml = None
    message = "OK"
    code = 1
    payload = {}
    
    if request.method == 'POST':
        if 'file' in request.FILES:
            logging.user(request, "~FR~SBOPML upload starting...")
            file = request.FILES['file']
            xml_opml = file.read()
            opml_importer = OPMLImporter(xml_opml, request.user)
            folders = opml_importer.process()

            feeds = UserSubscription.objects.filter(user=request.user).values()
            payload = dict(folders=folders, feeds=feeds)
            logging.user(request, "~FR~SBOPML Upload: ~SK%s~SN~SB~FR feeds" % (len(feeds)))
            
            request.session['import_from_google_reader'] = False
        else:
            message = "Attach an .opml file."
            code = -1
            
    data = json.encode(dict(message=message, code=code, payload=payload))
    return HttpResponse(data, mimetype='text/plain')

    def delete_feed(self, feed_id, in_folder):
        def _find_feed_in_folders(old_folders, folder_name='', multiples_found=False, deleted=False):
            new_folders = []
            for k, folder in enumerate(old_folders):
                if isinstance(folder, int):
                    if (folder == feed_id and (
                        (folder_name != in_folder) or
                        (folder_name == in_folder and deleted))):
                        multiples_found = True
                        logging.info(" ---> [%s] Deleting feed, and a multiple has been found in '%s'" % (self.user, folder_name))
                    if folder == feed_id and folder_name == in_folder and not deleted:
                        logging.info(" ---> [%s] Delete feed: %s'th item: %s folders/feeds" % (
                            self.user, k, len(old_folders)
                        ))
                        deleted = True
                    else:
                        new_folders.append(folder)
                elif isinstance(folder, dict):
                    for f_k, f_v in folder.items():
                        nf, multiples_found, deleted = _find_feed_in_folders(f_v, f_k, multiples_found, deleted)
                        new_folders.append({f_k: nf})
    
            return new_folders, multiples_found, deleted
        
        user_sub_folders = json.decode(self.folders)
        user_sub_folders, multiples_found, deleted = _find_feed_in_folders(user_sub_folders)
        self.folders = json.encode(user_sub_folders)
        self.save()

        if not multiples_found and deleted:
            user_sub = UserSubscription.objects.get(user=self.user, feed=feed_id)
            user_sub.delete()
            MUserStory.objects(user_id=self.user.pk, feed_id=feed_id).delete()

def add_site(request, token):
    code       = 0
    url        = request.GET['url']
    folder     = request.GET['folder']
    new_folder = request.GET.get('new_folder')
    callback   = request.GET['callback']
    
    if not url:
        code = -1
    else:
        try:
            profile = Profile.objects.get(secret_token=token)
            if new_folder:
                usf, _ = UserSubscriptionFolders.objects.get_or_create(user=profile.user)
                usf.add_folder(folder, new_folder)
                folder = new_folder
            code, message, us = UserSubscription.add_subscription(
                user=profile.user, 
                feed_address=url,
                folder=folder,
                bookmarklet=True
            )
        except Profile.DoesNotExist:
            code = -1
    
    if code > 0:
        message = 'OK'
        
    logging.user(profile.user, "~FRAdding URL from site: ~SB%s (in %s)" % (url, folder))
    
    return HttpResponse(callback + '(' + json.encode({
        'code':    code,
        'message': message,
        'usersub': us and us.feed.pk,
    }) + ')', mimetype='text/plain')

def add_site_authed(request):
    code       = 0
    url        = request.GET['url']
    folder     = request.GET['folder']
    new_folder = request.GET.get('new_folder')
    callback   = request.GET['callback']
    user       = get_user(request)
    
    if not url:
        code = -1
    else:
        if new_folder:
            usf, _ = UserSubscriptionFolders.objects.get_or_create(user=user)
            usf.add_folder(folder, new_folder)
            folder = new_folder
        code, message, us = UserSubscription.add_subscription(
            user=user, 
            feed_address=url,
            folder=folder,
            bookmarklet=True
        )
    
    if code > 0:
        message = 'OK'
        
    logging.user(user, "~FRAdding authed URL from site: ~SB%s (in %s)" % (url, folder),
                 request=request)
    
    return HttpResponse(callback + '(' + json.encode({
        'code':    code,
        'message': message,
        'usersub': us and us.feed_id,
    }) + ')', mimetype='text/plain')

    def add_missing_feeds(self):
        all_feeds = self.flat()
        subs = [us.feed_id for us in
                UserSubscription.objects.filter(user=self.user).only('feed')]
        
        missing_subs = set(all_feeds) - set(subs)
        if missing_subs:
            logging.debug(" ---> %s is missing %s subs. Adding %s..." % (
                          self.user, len(missing_subs), missing_subs))
            for feed_id in missing_subs:
                feed = Feed.get_by_id(feed_id)
                if feed:
                    us, _ = UserSubscription.objects.get_or_create(user=self.user, feed=feed, defaults={
                        'needs_unread_recalc': True
                    })
                    if not us.needs_unread_recalc:
                        us.needs_unread_recalc = True
                        us.save()

        missing_folder_feeds = set(subs) - set(all_feeds)
        if missing_folder_feeds:
            user_sub_folders = json.decode(self.folders)
            logging.debug(" ---> %s is missing %s folder feeds. Adding %s..." % (
                          self.user, len(missing_folder_feeds), missing_folder_feeds))
            for feed_id in missing_folder_feeds:
                feed = Feed.get_by_id(feed_id)
                if feed and feed.pk == feed_id:
                    user_sub_folders = add_object_to_folder(feed_id, "", user_sub_folders)
            self.folders = json.encode(user_sub_folders)
            self.save()

    def delete_folder(self, folder_to_delete, in_folder, feed_ids_in_folder, commit_delete=True):
        def _find_folder_in_folders(old_folders, folder_name, feeds_to_delete, deleted_folder=None):
            new_folders = []
            for k, folder in enumerate(old_folders):
                if isinstance(folder, int):
                    new_folders.append(folder)
                    if folder in feeds_to_delete:
                        feeds_to_delete.remove(folder)
                elif isinstance(folder, dict):
                    for f_k, f_v in folder.items():
                        if f_k == folder_to_delete and (folder_name == in_folder or in_folder is None):
                            logging.user(self.user, "~FBDeleting folder '~SB%s~SN' in '%s': %s" % (f_k, folder_name, folder))
                            deleted_folder = folder
                        else:
                            nf, feeds_to_delete, deleted_folder = _find_folder_in_folders(f_v, f_k, feeds_to_delete, deleted_folder)
                            new_folders.append({f_k: nf})
    
            return new_folders, feeds_to_delete, deleted_folder
            
        user_sub_folders = json.decode(self.folders)
        user_sub_folders, feeds_to_delete, deleted_folder = _find_folder_in_folders(user_sub_folders, '', feed_ids_in_folder)
        self.folders = json.encode(user_sub_folders)
        self.save()

        if commit_delete:
            UserSubscription.objects.filter(user=self.user, feed__in=feeds_to_delete).delete()
          
        return deleted_folder

 def collect_orphan_feeds(cls, user):
     us = cls.objects.filter(user=user)
     try:
         usf = UserSubscriptionFolders.objects.get(user=user)
     except UserSubscriptionFolders.DoesNotExist:
         return
     us_feed_ids = set([sub.feed_id for sub in us])
     folders = json.decode(usf.folders)
     
     def collect_ids(folders, found_ids):
         for item in folders:
             # print ' --> %s' % item
             if isinstance(item, int):
                 # print ' --> Adding feed: %s' % item
                 found_ids.add(item)
             elif isinstance(item, dict):
                 # print ' --> Descending folder dict: %s' % item.values()
                 found_ids.update(collect_ids(item.values(), found_ids))
             elif isinstance(item, list):
                 # print ' --> Descending folder list: %s' % len(item)
                 found_ids.update(collect_ids(item, found_ids))
         # print ' --> Returning: %s' % found_ids
         return found_ids
     found_ids = collect_ids(folders, set())
     diff = len(us_feed_ids) - len(found_ids)
     if diff > 0:
         logging.info(" ---> Collecting orphans on %s. %s feeds with %s orphans" % (user.username, len(us_feed_ids), diff))
         orphan_ids = us_feed_ids - found_ids
         folders.extend(list(orphan_ids))
         usf.folders = json.encode(folders)
         usf.save()

def set_preference(request):
    code = 1
    message = ''
    new_preferences = request.POST
    
    preferences = json.decode(request.user.profile.preferences)
    for preference_name, preference_value in new_preferences.items():
        if preference_name in SINGLE_FIELD_PREFS:
            setattr(request.user.profile, preference_name, preference_value)
        elif preference_name in SPECIAL_PREFERENCES:
            if (preference_name == 'old_password' and
                (new_preferences['old_password'] or
                 new_preferences['new_password'])):
                code = change_password(request.user, new_preferences['old_password'],
                                       new_preferences['new_password'])
                if code == -1:
                    message = "Your old password is incorrect."
        else:
            if preference_value in ["true", "false"]:
                preference_value = True if preference_value == "true" else False
            preferences[preference_name] = preference_value
        
    request.user.profile.preferences = json.encode(preferences)
    request.user.profile.save()
    
    response = dict(code=code, message=message, new_preferences=new_preferences)
    return response