def fetch_request(self):
     try:
         proxies = {"http": settings.COW_PROXY_HANDLER, "https": settings.COW_PROXY_HANDLER}
         r = requests.get(self.story.story_permalink, headers=self.headers, verify=False, proxies=proxies)
         # r = requests.get(self.story.story_permalink, headers=self.headers, verify=False)
     except (
         AttributeError,
         SocketError,
         requests.ConnectionError,
         requests.models.MissingSchema,
         requests.sessions.InvalidSchema,
     ), e:
         logging.user(self.request, "~SN~FRFailed~FY to fetch ~FGoriginal text~FY: %s" % e)
         logging.error(
             "fetch_request with:ERROR \n"
             + str(e)
             + "  feed_id:"
             + str(self.story.story_feed_id)
             + "  stroy_link:"
             + str(self.story.story_permalink)
         )
         if settings.SEND_ERROR_MAILS:
             mail_admins(
                 "Error in text_importer fetch_request",
                 str(e)
                 + "\n"
                 + "  feed_id:"
                 + str(self.story.story_feed_id)
                 + "\n"
                 + "  stroy_link:"
                 + str(self.story.story_permalink)
                 + "\n"
                 + traceback.format_exc(),
             )
         return

def create_zip(archive, files):
    '''Creates a zip file containing the files being backed up.'''
    import zipfile
    from utils.misc import add_file_hash

    try:
        # zipfile always follows links
        with zipfile.ZipFile(archive, 'w') as zipf:
            zipf.comment = 'Created by s3-backup'
            for f in files:
                f = f.strip()
                if os.path.exists(f):
                    zipf.write(f)
                    add_file_hash(archive, f)
                    log.debug('Added %s.' % f)
                else:
                    log.error('%s does not exist.' % f)
                
                if zipf.testzip() != None:
                    log.error('An error occured creating the zip archive.')
    except zipfile.BadZipfile:
        # I assume this only happens on reads? Just in case...
        log.critical('The zip file is corrupt.')
    except zipfile.LargeZipFile:
        log.critical('The zip file is greater than 2 GB.'
                ' Enable zip64 functionality.')

    def push(self):
        """Push todo to gist.github.com"""
        github = Github()
        gist_id = GistId().get()
        token = GithubToken().get()

        github.login(token)

        if not self.todo.name:
            name = "Todo"
        else:
            name = self.todo.name

        files = {
            name: {
                "content": self.todo_content
            }
        }

        log.info(
            "Pushing '%s' to https://gist.github.com/%s .." % (name, gist_id)
        )
        response = github.edit_gist(gist_id, files=files)

        if response.status_code == 200:
            log.ok("Pushed success.")
        elif response.status_code == 401:
            log.warning("Github token out of date, empty the old token")
            GithubToken().save('')  # empty the token!
            self.push()  # and repush
        else:
            log.error("Pushed failed. %d" % response.status_code)

    def get(self):
        """
          call this method to get a token::

            token = GithubToken().get()

          what the get() does:
            1) read from "~/.todo/token"
            2) check if the token read is empty.
               (yes)-> 1) if empty,
                          ask user for user&passwd to access api.github.com
                        2) fetch the token, set to this instance and store it.
            3) return the token (a string)
        """
        if self.is_empty:
            user = ask_input.text("Github user:")
            password = ask_input.password("Password for %s:" % user)

            log.info("Authorize to github.com..")
            response = Github().authorize(user, password)

            if response.status_code == 201:
                # 201 created
                log.ok("Authorized success.")
                # get token from dict
                token = response.json()["token"]
                self.save(token)
            else:
                log.error("Authorization failed. %d" % response.status_code)
        return self.content

 def get_task_by_id(self, index):
     """return task object by its index, if not found, fatal error."""
     index -= 1
     tasks = self.todo.tasks
     if index >= len(tasks):
         log.error("Task not found.")
     return tasks[index]

  def parse(self, file_name):
    errors = []

    patterns = {}
    patterns['type'] = r'^(?P<name>\w+),\*,\*,\s*(?P<type>(?:struct |unsigned |const )?[*\w]+(?: \*| const)?),\*,\*,?$'
    patterns['comment'] = r'^#.*$'

    with open(file_name, 'r') as dottm_file:
      for line in [ l.strip() for l in dottm_file if len(l.strip()) > 0 ]:
        for k, regex in patterns.items():
          match = re.match(regex, line)
          if match:
            if k == 'type':
              type_name = match.group('type')
              if type_name == '*':
                type_name = match.group('name')

              type_mappings[match.group('name')] = type_name

            break
        else:
          errors.append('U: ' + line)

    for e in errors:
      log.error(file_name + ': ' + e)

def load_from_rawxls(file_path):
    """从 xlsx 文件恢复数据到 lang.csv 的格式

    xlsx 文件可能是 <name, desc> 模式，也可能是 list<text> 模式。

    Args:
        file_path (str): xlsx 文件路径

    Returns:
        category (str): category from lang_def
        csv_data (list[str]): list of [file_id, unknown, index, text]，不带前导0
    """

    data = load_xls(file_path)[1:]
    # 判断文件模式
    id_split = data[0][1].split('-')
    if len(id_split) > 3 and id_split[-1].isdigit() and id_split[-2].isdigit() and id_split[-3].isdigit():
        # list of text
        return load_from_list_category(data)
    elif len(id_split) > 1 and id_split[-1].isdigit():
        # name_desc
        return load_from_pair_category(data)
    else:
        log.error('load %s failed.' % file_path)
        return '', []

    def _is_valid_link(self, link):
        """
        Return True if given link is non document, Since this is not a perfect way to check
        but it avoids a call to server. 
        """

        # Check ONLY_ROOTDOMAIN

        scheme, netloc, path, params, query, fragment = urlparse(link)

        try:
            if get_tld(self.base_url) == get_tld(link) and not ONLY_ROOTDOMAIN:
            # if get_tld(self.base_url) == get_tld(link):
                return False
        except Exception as e:
            log.error(str(e), self.base_url, link)


        # Need to add more
        DOC_EXT = [".pdf", ".xmls", ".docx", ".odt"]

        try:

            urlPath = [i for i in (path.split('/')) if i]

            file_name = urlPath[-1]
            ext = file_name.split(".")[-1]
        except IndexError:
            # Its just a root URL
            return True
        return ext not in DOC_EXT

def load_from_langxls(file_path, lang, need_check=False, load_ui=False):
    """从 xlsx 文件中读取 lang.csv 的翻译

    xlsx 文件可能是 <name, desc> 模式，也可能是 list<text> 模式。

    Args:
        file_path (str): xlsx 文件路径
        lang (str): "zh"/"en", 读中文还是英文
        need_check (bool): 是否检查
        load_ui (bool): 是否读取ui汉化文件的内容

    Returns:
        category (str): category from lang_def
        translated_data (list[str]): list of [file_id, unknown, index, text]，不带前导0
    """

    data = load_xls(file_path)[1:]
    # 判断文件模式
    id_split = data[0][1].split('-')
    if len(id_split) > 3 and id_split[-1].isdigit() and id_split[-2].isdigit() and id_split[-3].isdigit():
        # list of text
        return load_from_list_category(data, lang, need_check)
    elif len(id_split) > 1 and id_split[-1].isdigit():
        # name_desc
        return load_from_pair_category(data, lang, need_check)
    elif len(id_split) == 1 and '_' in id_split[0] \
            and id_split[0][0].isalpha() and id_split[0][-1].isalpha():
        if load_ui:
            return load_from_ui_fake(data, lang, need_check)
        else:
            log.info('skip ui file %s.' % file_path)
            return '', []
    else:
        log.error('load %s failed.' % file_path)
        return '', []

 def fetch(self, skip_save=False):
     # =====================
     # =Modified by SongJun=
     # =====================
     try:
         request = urllib2.Request(self.story.story_permalink, headers=self.headers)
         opener = urllib2.build_opener(
             urllib2.ProxyHandler({"http": settings.COW_PROXY_HANDLER, "https": settings.COW_PROXY_HANDLER})
         )
         # opener = urllib2.build_opener()
         text = opener.open(request).read()
         # logging.info(text)
         # Add by Xinyan Lu: some websites always return gzip files
         if text[:6] == "\x1f\x8b\x08\x00\x00\x00":
             text = gzip.GzipFile(fileobj=cStringIO.StringIO(text)).read()
     except httplib.IncompleteRead as e:
         text = e.partial
     except (Exception), e:
         logging.user(self.request, "~SN~FRFailed~FY to fetch ~FGoriginal text~FY: %s" % e)
         logging.error(
             "error fetch_request"
             + str(e)
             +
             # '  feed_id:'+str(self.story.story_feed_id)+\
             "  stroy_link:"
             + str(self.story.story_permalink)
         )
         return

def current(roster, cli, config):
    '''Print information on the current shift in the roster.'''
    # roster.current return a *list* of all the people on duty
    shifts = roster.current
    if len(shifts) == 1:
        [current] = shifts
    elif len(shifts) == 0:
        log.error('Nobody is on duty.')
        now = datetime.datetime.now(tz=pytz.UTC)
        current = Shift(now, now, None, None, None)
    else:
        log.error('Several people where on duty, picking a random one.')
        for counter, shift in enumerate(shifts, 1):
            log.error('On duty #{}: {}'.format(counter, shift))
        current = choice(shifts)
    # Replace missing fields with fallback ones
    if not current.email:
        current.email = config['fallback.email']
        if current.name is not None:
            log.error('Missing email address for "{}"'.format(current.name))
    if not current.phone:
        current.phone = config['fallback.phone']
        if current.name is not None:
            log.error('Missing phone number for "{}"'.format(current.name))
    current.name = current.name or 'Fallback Contact Details'
    # Compute what fields to output
    fields = ('start', 'end', 'name', 'email', 'phone')
    mask = []
    for attr_name in fields:
        mask.append(cli[attr_name])
    if not any(mask):
        mask = [True] * 5  # No explicit field, means all fields
    bits = [val for val, flag in zip(current.as_string_tuple, mask) if flag]
    print('\t'.join(bits))

 def on_receive(self, msg):
     try:
         return getattr(self, msg['func'])(msg['msg'])
     except Exception, e:
         log.error('Get error: %s', e, exc_info=True)
         # TODO fix return
         return None

    def handle(self, *args, **options):
        
        if options['daemonize']:
            daemonize()
        
        settings.LOG_TO_STREAM = True        
            
        r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL)
        
        if options['initialize']:
            feeds = Feed.objects.filter(num_subscribers__gte=1).order_by('?')
            print 'Query feeds done with num of feeds',len(feeds)
            r.ltrim('freeze_feeds',1,0)

            pipeline = r.pipeline()
            for feed in feeds:
                pipeline.rpush('freeze_feeds',feed.pk)
            pipeline.execute()
            print 'Initialize freeze_feeds done'

        feed_id = r.lpop('freeze_feeds')
        while feed_id:
            try:
                frozen_num = MStory.freeze_feed(int(feed_id))
                if frozen_num > 0:
                    r.rpush('freeze_feeds',feed_id)
            except Exception, e:
                logging.error(str(e)+\
                            traceback.format_exc()+'\n'+\
                            'Error from:  freeze_feeds\n')
            feed_id = r.lpop('freeze_feeds')

    def run(self, target=None, tid=None):
        if target is None:
            log.critical("Please set --target param")
            sys.exit()
        if tid is None:
            log.critical("Please set --tid param")
            sys.exit()

        # Statistic Code
        p = subprocess.Popen(
            ['cloc', target], stdout=subprocess.PIPE)
        (output, err) = p.communicate()
        rs = output.split("\n")
        for r in rs:
            r_e = r.split()
            if len(r_e) > 3 and r_e[0] == 'SUM:':
                t = CobraTaskInfo.query.filter_by(id=tid).first()
                if t is not None:
                    t.code_number = r_e[4]
                    try:
                        db.session.add(t)
                        db.session.commit()
                        log.info("Statistic code number done")
                    except Exception as e:
                        log.error("Statistic code number failed" + str(e.message))

def required(a, group=None, name=None):
    k = key(a, group)
    if k.required == True:
        if name:
            log.error("Missing '"+a+"' key in "+group+" '"+name+"'")
        else:
            log.error("Missing key '"+a+"' in group '"+group+"'")

    def get_links(self, url, page_depth=0):
        """
        Get the links from url
        """
        if (self._is_url(url) and 
            page_depth <= MAX_DEPTH and 
            not self._is_crawled(url)):

            try:
                redis.hset(URLS_CRAWLLING, base64.b64encode(url.encode('UTF-8')), "")

                response = requests.get(url)

                if (response.status_code//100 == 2 and
                    self._is_html(response)):

                    redis.hset(URLS_CRAWLLING, base64.b64encode(url.encode('UTF-8')), response.content)

                    status, links = Parser(self._base_url(url)).parse_href(response.content)

                    log.debug("Page Depth {0} URL {2} Total count  {1}".format(page_depth, len(links), url))
                    for link in links:
                        self.get_links(link, page_depth+1)

            except Exception as e:
                log.error(str(e))
                raise

def isType(key, value, expected):
    try:
        assert isinstance(value, expected)
    except AssertionError:
        log.error("Key '"+key+"' is of type '"+type(value).__name__+"';",
                   "expected '"+expected.__name__+"'.",
                   "(value: "+str(value)+")")

    def run(self, feed_pks, **kwargs):
        try:
            from apps.rss_feeds.models import Feed
            #from apps.statistics.models import MStatistics
            r = redis.Redis(connection_pool=settings.REDIS_FEED_POOL)
            #mongodb_replication_lag = int(MStatistics.get('mongodb_replication_lag', 0))
            #compute_scores = bool(mongodb_replication_lag < 10)

            options = {
            #    'quick': float(MStatistics.get('quick_fetch', 0)),
            #    'compute_scores': compute_scores,
            #    'mongodb_replication_lag': mongodb_replication_lag,
            }

            if not isinstance(feed_pks, list):
                feed_pks = [feed_pks]

            for feed_pk in feed_pks:
                feed = Feed.get_by_id(feed_pk)
                if not feed or feed.pk != int(feed_pk):
                    logging.info(" ---> ~FRRemoving feed_id %s from tasked_feeds queue, points to %s..." % (feed_pk, feed and feed.pk))
                    r.zrem('tasked_feeds', feed_pk)
                if feed:
                    feed.update(**options)
        except Exception, e:
            logging.error(str(e)+\
                traceback.format_exc()+'\n'+\
                'error from:  UpdateFeeds\n')
            if settings.SEND_ERROR_MAILS:
                mail_admins("Error in UpdateFeeds",str(e)+'\n'+traceback.format_exc())

def isMissingKeys(keylist):
    if len(_missingkeys):
        msg = "The following keys are missing from the packfile!\n"
        for k in _missingkeys:
            msg += "\n- %-16s %-50s (e.g. %s)" % (k,
                    keylist.description(k),
                    keylist.example(k))
        log.error(msg)

def module(name, parent=None):
    if parent == None:
        return importlib.import_module(name)
    else:
        if not type(parent) == ModuleType:
            log.error("Module '"+str(parent)+"' is not a module!")

        return importlib.import_module(parent.__name__+'.'+name)