def removeDynamicContent(self, page, dynamicMarks):
        """
        Removing dynamic content from supplied page basing removal on
        precalculated dynamic markings
        """
        if page and len(dynamicMarks) > 0:
            encoding = chardet.detect(page)['encoding']
            page = page.decode(encoding, errors='replace')
            for item in dynamicMarks:
                prefix, suffix = item
                if prefix is not None:
                    prefix = prefix.decode(encoding, errors='replace')
                if suffix is not None:
                    suffix = suffix.decode(encoding, errors='replace')

                if prefix is None and suffix is None:
                    continue
                elif prefix is None:
                    page = re.sub(r'(?s)^.+{0}'.format(re.escape(suffix)), suffix.replace('\\', r'\\'), page)
                elif suffix is None:
                    page = re.sub(r'(?s){0}.+$'.format(re.escape(prefix)), prefix.replace('\\', r'\\'), page)
                else:
                    page = re.sub(r'(?s){0}.+{1}'.format(re.escape(prefix), re.escape(suffix)), "{0}{1}".format(prefix.replace('\\', r'\\'), suffix.replace('\\', r'\\')), page)

            page = page.encode()

        return page

def getHeuristicCharEncoding(page):
    """
    Returns page encoding charset detected by usage of heuristics
    Reference: http://chardet.feedparser.org/docs/
    """
    retVal = detect(page)["encoding"]

    infoMsg = "heuristics detected web page charset '%s'" % retVal
    singleTimeLogMessage(infoMsg, logging.INFO, retVal)

    return retVal

    def _detectEncodeType(self, content):
        result = {}

        for key,value in self._bomList.iteritems():
            if content.startswith(value):
                result['encoding'] = key + "-bom"
                result['confidence'] = 0.80
                break
        else:
            result = chardet.detect(content)

        return result

def getHeuristicCharEncoding(page):
    """
    Returns page encoding charset detected by usage of heuristics
    Reference: http://chardet.feedparser.org/docs/
    """

    key = hash(page)
    retVal = kb.cache.encoding.get(key) or detect(page)["encoding"]
    kb.cache.encoding[key] = retVal

    if retVal:
        infoMsg = "heuristics detected web page charset '%s'" % retVal
        singleTimeLogMessage(infoMsg, logging.INFO, retVal)

    return retVal

    def detect(self, size=2048):
        '''
        文件编码类型推断
        '''
        content = open(self.fileName,"rb").read(size)
        result = dict()
        for key,value in self._bomList.iteritems():
            if content.startswith(value):
                result['encoding'] = key + "-bom"
                result['confidence'] = 0.80
                break
        else:
            result = chardet.detect(content)

        return result

 def detect(self):
     '''
     非ASCII字符串编码类型推断
     '''
     rawstr = "".join([x[1] for x in self._autoPreDecode()])
     return chardet.detect(rawstr)