def fetch_request(self):
     try:
         r = requests.get(self.story.story_permalink, headers=self.headers, verify=False)
     except (AttributeError, SocketError, requests.ConnectionError, 
             requests.models.MissingSchema, requests.sessions.InvalidSchema), e:
         logging.user(self.request, "~SN~FRFailed~FY to fetch ~FGoriginal text~FY: %s" % e)
         return

 def refund_premium(self):
     refunded = False
     
     if self.stripe_id:
         stripe.api_key = settings.STRIPE_SECRET
         stripe_customer = stripe.Customer.retrieve(self.stripe_id)
         stripe_payments = stripe.Charge.all(customer=stripe_customer.id).data
         stripe_payments[0].refund()
         refunded = stripe_payments[0].amount/100
         logging.user(self.user, "~FRRefunding stripe payment: $%s" % refunded)
         self.cancel_premium()
     else:
         paypal_opts = {
             'API_ENVIRONMENT': 'PRODUCTION',
             'API_USERNAME': settings.PAYPAL_API_USERNAME,
             'API_PASSWORD': settings.PAYPAL_API_PASSWORD,
             'API_SIGNATURE': settings.PAYPAL_API_SIGNATURE,
         }
         paypal = PayPalInterface(**paypal_opts)
         transaction = PayPalIPN.objects.filter(custom=self.user.username,
                                                txn_type='subscr_payment')[0]
         refund = paypal.refund_transaction(transaction.txn_id)
         refunded = int(float(refund['raw']['TOTALREFUNDEDAMOUNT'][0]))
         logging.user(self.user, "~FRRefunding paypal payment: $%s" % refunded)
         self.cancel_premium()
     
     return refunded

    def send_first_share_to_blurblog_email(self, force=False):
        from apps.social.models import MSocialProfile, MSharedStory
        
        if not self.user.email:
            return

        sent_email, created = MSentEmail.objects.get_or_create(receiver_user_id=self.user.pk,
                                                               email_type='first_share')
        
        if not created and not force:
            return
        
        social_profile = MSocialProfile.objects.get(user_id=self.user.pk)
        params = {
            'shared_stories': MSharedStory.objects.filter(user_id=self.user.pk).count(),
            'blurblog_url': social_profile.blurblog_url,
            'blurblog_rss': social_profile.blurblog_rss
        }
        user    = self.user
        text    = render_to_string('mail/email_first_share_to_blurblog.txt', params)
        html    = render_to_string('mail/email_first_share_to_blurblog.xhtml', params)
        subject = "Your shared stories on NewsBlur are available on your Blurblog"
        msg     = EmailMultiAlternatives(subject, text, 
                                         from_email='NewsBlur <%s>' % settings.HELLO_EMAIL,
                                         to=['%s <%s>' % (user, user.email)])
        msg.attach_alternative(html, "text/html")
        msg.send(fail_silently=True)
        
        logging.user(self.user, "~BB~FM~SBSending first share to blurblog email to: %s" % self.user.email)

    def trim_user_read_stories(self, user_id):
        r = redis.Redis(connection_pool=settings.REDIS_STORY_HASH_POOL)
        subs = UserSubscription.objects.filter(user_id=user_id).only('feed')
        if not subs: return
        feeds = [f.feed_id for f in subs]
        old_rs = r.smembers("RS:%s" % user_id)
        old_count = len(old_rs)
        # new_rs = r.sunionstore("RS:%s" % user_id, *["RS:%s:%s" % (user_id, f) for f in feeds])
        new_rs = r.sunion(*["RS:%s:%s" % (user_id, f) for f in feeds])

        if not old_count: return
        
        r.sunionstore("RS:%s:backup" % user_id, "RS:%s" % user_id)
        r.expire("RS:%s:backup" % user_id, 60*60*24)
        
        missing_rs = []
        feed_re = re.compile(r'(\d+):.*?')
        for rs in old_rs:
            found = feed_re.search(rs)
            if not found:
                print " ---> Not found: %s" % rs
                continue
            rs_feed_id = found.groups()[0]
            if int(rs_feed_id) not in feeds:
                missing_rs.append(rs)
                # r.sadd("RS:%s" % user_id, *missing_rs)
        
        new_count = len(new_rs)
        missing_count = len(missing_rs)
        new_total = new_count + missing_count
        user = User.objects.get(pk=user_id)
        logging.user(user, "~FBTrimming ~FR%s~FB/%s (~SB%s~SN+~SB%s~SN saved) user read stories..." %
                     (old_count - new_total, old_count, new_count, missing_count))

def opml_upload(request):
    xml_opml = None
    message = "OK"
    code = 1
    payload = {}
    
    if request.method == 'POST':
        if 'file' in request.FILES:
            logging.user(request, "~FR~SBOPML upload starting...")
            file = request.FILES['file']
            xml_opml = file.read()
            opml_importer = OPMLImporter(xml_opml, request.user)
            folders = opml_importer.process()

            feeds = UserSubscription.objects.filter(user=request.user).values()
            payload = dict(folders=folders, feeds=feeds)
            logging.user(request, "~FR~SBOPML Upload: ~SK%s~SN~SB~FR feeds" % (len(feeds)))
            
            request.session['import_from_google_reader'] = False
        else:
            message = "Attach an .opml file."
            code = -1
            
    data = json.encode(dict(message=message, code=code, payload=payload))
    return HttpResponse(data, mimetype='text/plain')

def mark_feed_as_read(request):
    feed_ids = [int(f) for f in request.REQUEST.getlist('feed_id') if f]
    feed_count = len(feed_ids)
    multiple = feed_count > 1
    code = 0
    for feed_id in feed_ids:
        try:
            feed = Feed.objects.get(id=feed_id)
        except Feed.DoesNotExist:
            continue
        code = 0
    
        us = UserSubscription.objects.get(feed=feed, user=request.user)
        try:
            us.mark_feed_read()
        except IntegrityError:
            code = -1
        else:
            code = 1
        
        if not multiple:
            logging.user(request, "~FMMarking feed as read: ~SB%s" % (feed,))
            
    if multiple:
        logging.user(request, "~FMMarking ~SB%s~SN feeds as read" % (feed_count,))
        
    return dict(code=code)

def paypal_form(request):
    domain = Site.objects.get_current().domain
    
    paypal_dict = {
        "cmd": "_xclick-subscriptions",
        "business": "[email protected]",
        "a3": "12.00",                     # price 
        "p3": 1,                           # duration of each unit (depends on unit)
        "t3": "Y",                         # duration unit ("M for Month")
        "src": "1",                        # make payments recur
        "sra": "1",                        # reattempt payment on payment error
        "no_note": "1",                    # remove extra notes (optional)
        "item_name": "NewsBlur Premium Account",
        "notify_url": "http://%s%s" % (domain, reverse('paypal-ipn')),
        "return_url": "http://%s%s" % (domain, reverse('paypal-return')),
        "cancel_return": "http://%s%s" % (domain, reverse('index')),
        "custom": request.user.username,
    }

    # Create the instance.
    form = PayPalPaymentsForm(initial=paypal_dict, button_type="subscribe")

    logging.user(request, "~FBLoading paypal/feedchooser")

    # Output the button.
    return HttpResponse(form.render(), mimetype='text/html')

 def fetch_request(self, use_mercury=True):
     headers = self.headers
     url = self.story_url
     if self.story and not url:
         url = self.story.story_permalink
     
     if use_mercury:
         mercury_api_key = getattr(settings, 'MERCURY_PARSER_API_KEY', 'abc123')
         headers["content-type"] = "application/json"
         headers["x-api-key"] = mercury_api_key
         if settings.DEBUG:
             url = "http://nb.local.com:4040/rss_feeds/original_text_fetcher?url=%s" % url
         else:
             url = "https://www.newsblur.com/rss_feeds/original_text_fetcher?url=%s" % url
         
     try:
         r = requests.get(url, headers=headers, verify=False)
         r.connection.close()
     except (AttributeError, SocketError, requests.ConnectionError,
             requests.models.MissingSchema, requests.sessions.InvalidSchema,
             requests.sessions.TooManyRedirects,
             requests.models.InvalidURL,
             requests.models.ChunkedEncodingError,
             requests.models.ContentDecodingError,
             urllib3.exceptions.LocationValueError,
             LocationParseError, OpenSSLError, PyAsn1Error), e:
         logging.user(self.request, "~SN~FRFailed~FY to fetch ~FGoriginal text~FY: %s" % e)
         return

    def import_starred_items(self, count=10):

        continuation = ""

        while True:
            if continuation:
                sub_url = "%s/0/stream/contents/user/-/state/com.google/starred?n=%s&c=%s" % (
                    self.scope,
                    count,
                    continuation,
                )
            else:
                sub_url = "%s/0/stream/contents/user/-/state/com.google/starred?n=%s" % (self.scope, count)

            stories_str = self.send_request(sub_url)

            try:
                stories = json.decode(stories_str)
                continuation = stories.get("continuation")
            except:
                logging.user(self.user, "~BB~FW~SBGoogle Reader starred stories: ~BT~FWNo stories")
                stories = None
            if stories:
                logging.user(
                    self.user, "~BB~FW~SBGoogle Reader starred stories: ~BT~FW%s stories" % (len(stories["items"]))
                )
                self.process_starred_items(stories["items"])

            if not continuation or count < 1000:
                break

        starred_count = MStarredStory.objects.filter(user_id=self.user.pk).count()
        return starred_count

def load_recommended_feed(request):
    user        = get_user(request)
    page        = int(request.REQUEST.get('page', 0))
    usersub     = None
    refresh     = request.REQUEST.get('refresh')
    now         = datetime.datetime.now
    unmoderated = request.REQUEST.get('unmoderated', False) == 'true'
    
    if unmoderated:
        recommended_feeds = RecommendedFeed.objects.filter(is_public=False, declined_date__isnull=True)[page:page+2]
    else:
        recommended_feeds = RecommendedFeed.objects.filter(is_public=True, approved_date__lte=now)[page:page+2]
    if recommended_feeds and request.user.is_authenticated():
        usersub = UserSubscription.objects.filter(user=user, feed=recommended_feeds[0].feed)
    if refresh != 'true' and page > 0:
        logging.user(request, "~FBBrowse recommended feed: ~SBPage #%s" % (page+1))
    
    recommended_feed = recommended_feeds and recommended_feeds[0]
    feed_icon = MFeedIcon.objects(feed_id=recommended_feed.feed.pk)
    
    if recommended_feed:
        return render_to_response('recommendations/render_recommended_feed.xhtml', {
            'recommended_feed'  : recommended_feed,
            'description'       : recommended_feed.description or recommended_feed.feed.data.feed_tagline,
            'usersub'           : usersub,
            'feed_icon'         : feed_icon and feed_icon[0],
            'has_next_page'     : len(recommended_feeds) > 1,
            'has_previous_page' : page != 0,
            'unmoderated'       : unmoderated,
            'today'             : datetime.datetime.now(),
        }, context_instance=RequestContext(request))
    else:
        return HttpResponse("")

    def _check_if_first_newsletter(self, user, force=False):
        if not user.email:
            return

        subs = UserSubscription.objects.filter(user=user)
        found_newsletter = False
        for sub in subs:
            if sub.feed.is_newsletter:
                found_newsletter = True
                break
        if not found_newsletter and not force: 
            return        
        
        params = dict(receiver_user_id=user.pk, email_type='first_newsletter')
        try:
            sent_email = MSentEmail.objects.get(**params)
            if not force:
                # Return if email already sent
                return
        except MSentEmail.DoesNotExist:
            sent_email = MSentEmail.objects.create(**params)
                
        text    = render_to_string('mail/email_first_newsletter.txt', {})
        html    = render_to_string('mail/email_first_newsletter.xhtml', {})
        subject = "Your email newsletters are now being sent to NewsBlur"
        msg     = EmailMultiAlternatives(subject, text, 
                                         from_email='NewsBlur <%s>' % settings.HELLO_EMAIL,
                                         to=['%s <%s>' % (user, user.email)])
        msg.attach_alternative(html, "text/html")
        msg.send(fail_silently=True)
        
        logging.user(user, "~BB~FM~SBSending first newsletter email to: %s" % user.email)

    def send_opml_export_email(self, reason=None, force=False):
        if not self.user.email:
            return
        
        emails_sent = MSentEmail.objects.filter(receiver_user_id=self.user.pk,
                                                email_type='opml_export')
        day_ago = datetime.datetime.now() - datetime.timedelta(days=1)
        for email in emails_sent:
            if email.date_sent > day_ago and not force:
                logging.user(self.user, "~SN~FMNot sending opml export email, already sent today.")
                return

        MSentEmail.record(receiver_user_id=self.user.pk, email_type='opml_export')
        
        exporter = OPMLExporter(self.user)
        opml     = exporter.process()

        params = {
            'feed_count': UserSubscription.objects.filter(user=self.user).count(),
            'reason': reason,
        }
        user    = self.user
        text    = render_to_string('mail/email_opml_export.txt', params)
        html    = render_to_string('mail/email_opml_export.xhtml', params)
        subject = "Backup OPML file of your NewsBlur sites"
        filename= 'NewsBlur Subscriptions - %s.xml' % datetime.datetime.now().strftime('%Y-%m-%d')
        msg     = EmailMultiAlternatives(subject, text, 
                                         from_email='NewsBlur <%s>' % settings.HELLO_EMAIL,
                                         to=['%s <%s>' % (user, user.email)])
        msg.attach_alternative(html, "text/html")
        msg.attach(filename, opml, 'text/xml')
        msg.send(fail_silently=True)
        
        logging.user(self.user, "~BB~FM~SBSending OPML backup email to: %s" % self.user.email)

def add_site(request, token):
    code = 0
    url = request.GET["url"]
    folder = request.GET["folder"]
    new_folder = request.GET.get("new_folder")
    callback = request.GET["callback"]

    if not url:
        code = -1
    else:
        try:
            profile = Profile.objects.get(secret_token=token)
            if new_folder:
                usf, _ = UserSubscriptionFolders.objects.get_or_create(user=profile.user)
                usf.add_folder(folder, new_folder)
                folder = new_folder
            code, message, us = UserSubscription.add_subscription(
                user=profile.user, feed_address=url, folder=folder, bookmarklet=True
            )
        except Profile.DoesNotExist:
            code = -1

    if code > 0:
        message = "OK"

    logging.user(profile.user, "~FRAdding URL from site: ~SB%s (in %s)" % (url, folder), request=request)

    return HttpResponse(
        callback + "(" + json.encode({"code": code, "message": message, "usersub": us and us.feed_id}) + ")",
        mimetype="text/plain",
    )

def set_view_setting(request):
    code = 1
    feed_id = request.POST["feed_id"]
    feed_view_setting = request.POST.get("feed_view_setting")
    feed_order_setting = request.POST.get("feed_order_setting")
    feed_read_filter_setting = request.POST.get("feed_read_filter_setting")
    feed_layout_setting = request.POST.get("feed_layout_setting")
    view_settings = json.decode(request.user.profile.view_settings)

    setting = view_settings.get(feed_id, {})
    if isinstance(setting, basestring):
        setting = {"v": setting}
    if feed_view_setting:
        setting["v"] = feed_view_setting
    if feed_order_setting:
        setting["o"] = feed_order_setting
    if feed_read_filter_setting:
        setting["r"] = feed_read_filter_setting
    if feed_layout_setting:
        setting["l"] = feed_layout_setting

    view_settings[feed_id] = setting
    request.user.profile.view_settings = json.encode(view_settings)
    request.user.profile.save()

    logging.user(
        request,
        "~FMView settings: %s/%s/%s/%s"
        % (feed_view_setting, feed_order_setting, feed_read_filter_setting, feed_layout_setting),
    )
    response = dict(code=code)
    return response

def load_starred_stories(request):
    user = get_user(request)
    offset = int(request.REQUEST.get('offset', 0))
    limit = int(request.REQUEST.get('limit', 10))
    page = int(request.REQUEST.get('page', 0))
    if page: offset = limit * (page - 1)
        
    mstories = MStarredStory.objects(user_id=user.pk).order_by('-starred_date')[offset:offset+limit]
    stories = Feed.format_stories(mstories)
    
    for story in stories:
        story_date = localtime_for_timezone(story['story_date'], user.profile.timezone)
        now = localtime_for_timezone(datetime.datetime.now(), user.profile.timezone)
        story['short_parsed_date'] = format_story_link_date__short(story_date, now)
        story['long_parsed_date'] = format_story_link_date__long(story_date, now)
        starred_date = localtime_for_timezone(story['starred_date'], user.profile.timezone)
        story['starred_date'] = format_story_link_date__long(starred_date, now)
        story['read_status'] = 1
        story['starred'] = True
        story['intelligence'] = {
            'feed': 0,
            'author': 0,
            'tags': 0,
            'title': 0,
        }
    
    logging.user(request, "~FCLoading starred stories: ~SB%s stories" % (len(stories)))
    
    return dict(stories=stories)

 def mark_story_ids_as_read(self, story_ids, request=None):
     data = dict(code=0, payload=story_ids)
     
     if not request:
         request = self.user
 
     if not self.needs_unread_recalc:
         self.needs_unread_recalc = True
         self.save()
 
     if len(story_ids) > 1:
         logging.user(request, "~FYRead %s stories in feed: %s" % (len(story_ids), self.feed))
     else:
         logging.user(request, "~FYRead story in feed: %s" % (self.feed))
     
     for story_id in set(story_ids):
         try:
             story = MStory.objects.get(story_feed_id=self.feed_id, story_guid=story_id)
         except MStory.DoesNotExist:
             # Story has been deleted, probably by feed_fetcher.
             continue
         except MStory.MultipleObjectsReturned:
             story = MStory.objects.filter(story_feed_id=self.feed_id, story_guid=story_id)[0]
         now = datetime.datetime.utcnow()
         date = now if now > story.story_date else story.story_date # For handling future stories
         m, _ = MUserStory.objects.get_or_create(story_id=story_id, user_id=self.user_id, 
                                                 feed_id=self.feed_id, defaults={
             'read_date': date, 
             'story': story, 
             'story_date': story.story_date,
         })
             
     return data

def mark_story_as_unread(request):
    story_id = request.POST['story_id']
    feed_id = int(request.POST['feed_id'])

    try:
        usersub = UserSubscription.objects.select_related('feed').get(user=request.user, feed=feed_id)
    except Feed.DoesNotExist:
        duplicate_feed = DuplicateFeed.objects.filter(duplicate_feed_id=feed_id)
        if duplicate_feed:
            try:
                usersub = UserSubscription.objects.get(user=request.user, 
                                                       feed=duplicate_feed[0].feed)
            except Feed.DoesNotExist:
                return dict(code=-1)
                
    if not usersub.needs_unread_recalc:
        usersub.needs_unread_recalc = True
        usersub.save()
        
    data = dict(code=0, payload=dict(story_id=story_id))
    logging.user(request, "~FY~SBUnread~SN story in feed: %s" % (usersub.feed))
        
    story = MStory.objects(story_feed_id=feed_id, story_guid=story_id)[0]
    m = MUserStory.objects(story=story, user_id=request.user.pk, feed_id=feed_id)
    m.delete()
    
    return data

def import_from_google_reader(request):
    code = 0
    feed_count = 0
    starred_count = 0
    delayed = False
    
    if request.user.is_authenticated():
        reader_importer = GoogleReaderImporter(request.user)
        auto_active = bool(request.REQUEST.get('auto_active') or False)
        
        try:
            code = reader_importer.try_import_feeds(auto_active=auto_active)
        except TimeoutError:
            ProcessReaderImport.delay(request.user.pk, auto_active=auto_active)
            feed_count = UserSubscription.objects.filter(user=request.user).count()
            logging.user(request, "~FR~SBGoogle Reader import took too long, found %s feeds. Tasking..." % feed_count)
            delayed = True
            code = 2
            
        if 'import_from_google_reader' in request.session:
            del request.session['import_from_google_reader']
    
        feed_count = UserSubscription.objects.filter(user=request.user).count()
        
    return dict(code=code, delayed=delayed, feed_count=feed_count, starred_count=starred_count)

def save_feed_chooser(request):
    approved_feeds = [int(feed_id) for feed_id in request.POST.getlist('approved_feeds') if feed_id][:64]
    activated = 0
    usersubs = UserSubscription.objects.filter(user=request.user)
    
    for sub in usersubs:
        try:
            if sub.feed.pk in approved_feeds:
                sub.active = True
                activated += 1
                sub.save()
                sub.feed.count_subscribers()
            elif sub.active:
                sub.active = False
                sub.save()
        except Feed.DoesNotExist:
            pass
            
    
    logging.user(request, "~BB~FW~SBActivated standard account: ~FC%s~SN/~SB%s" % (
        activated, 
        usersubs.count()
    ))
    request.user.profile.queue_new_feeds()
    request.user.profile.refresh_stale_feeds(exclude_new=True)

    return {'activated': activated}

def original_text(request):
    story_id = request.REQUEST.get('story_id')
    feed_id = request.REQUEST.get('feed_id')
    story_hash = request.REQUEST.get('story_hash', None)
    force = request.REQUEST.get('force', False)
    debug = request.REQUEST.get('debug', False)

    if story_hash:
        story, _ = MStory.find_story(story_hash=story_hash)
    else:
        story, _ = MStory.find_story(story_id=story_id, story_feed_id=feed_id)

    if not story:
        logging.user(request, "~FYFetching ~FGoriginal~FY story text: ~FRstory not found")
        return {'code': -1, 'message': 'Story not found.', 'original_text': None, 'failed': True}
    
    original_text = story.fetch_original_text(force=force, request=request, debug=debug)

    return {
        'feed_id': story.story_feed_id,
        'story_hash': story.story_hash,
        'story_id': story.story_guid,
        'image_urls': story.image_urls,
        'secure_image_urls': Feed.secure_image_urls(story.image_urls),
        'original_text': original_text,
        'failed': not original_text or len(original_text) < 100,
    }