def encode(self, string):
     clean_sentence_unwantedchars= '["\t\n ]+'
     string = string.encode('utf8')
     string = string.decode('utf-8')    
     string = re.sub(clean_sentence_unwantedchars, ' ', string)
     string = string.encode('ascii', 'replace').encode('utf-8')
     string = string.decode('utf-8')
     return str(string)    

def ensure_unicode(string):
    if isinstance(string, str):
        try:
            string = string.decode("utf-8")
        except UnicodeDecodeError:
            string = string.decode("unicode-escape")

    return string

def scrubstring(string):
	from scrubber import Scrubber
	scrubber = Scrubber(autolink=True)
	try:
		string = string.decode('ascii')
	except UnicodeDecodeError:
		string = string.decode('utf-8')
	string = scrubber.scrub(string)
	return string.encode('utf-8')

def ensure_unicode(stuff, encoding = 'utf8', encoding2 = 'latin-1'):
    if type(stuff) is not str and type(stuff) is not np.string_:
        return stuff
    else:
        string = stuff
    try:
        string = string.decode(encoding)
    except:
        string = string.decode(encoding2, errors = 'ignore')
    return string

def unicode_cleaner(string):
    if isinstance(string, unicode):
        return string

    try:
        return string.decode('utf-8')
    except UnicodeError:
        try:
            return string.decode('latin-1')
        except UnicodeError:
            return string.decode('utf-8', 'ignore')

    def decode_guess(self, string, encoding):
        # text is not valid utf-8, try to make sense of it
        if encoding:
            try:
                return string.decode(encoding).encode('utf-8')
            except UnicodeDecodeError:
                pass

        try:
            return string.decode('latin-1').encode('utf-8')
        except UnicodeDecodeError:
            return string.decode('ascii', 'replace').encode('utf-8')

def tryDecode(string):
	try:
		string_d = string.decode('utf-8','strict')
		enc = 'utf-8'
	except:
		try:
			string_d = string.decode(config.encoding,'strict')
			enc = config.encoding
		except:
			enc = 'iso-8859-1'
			try:
				string_d = string.decode('iso-8859-1','strict')
			except:
				string_d = string.decode('iso-8859-1','replace')
	return [string_d, enc]

	def _decode_string(self, string):
		for encoding in ['ascii', 'UTF8', 'latin-1']:
			try:
				return string.decode(encoding)
			except:
				pass
		return 'INVALID ENCODING'

def htmlenties2txt(string, encoding="latin-1"):
    """
    Resolves all the HTML entities in the input string.
    Returns a Unicode string with the entities resolved.
    """
    try:
        string = string.decode(encoding)
    except:
        pass

    i = 0
    while i < len(string):
        amp = string.find("&", i) # find & as start of entity
        if amp == -1: # not found
            break
        i = amp + 1

        semicolon = string.find(";", amp) # find ; as end of entity
        if string[amp + 1] == "#": # numerical entity like "&#039;"
            entity = string[amp:semicolon+1]
            replacement = unichr(int(entity[2:-1]))
        else:
            entity = string[amp:semicolon + 1]
            if semicolon - amp > 7:
                continue
            try:
                # the array has mappings like "Uuml" -> "�"
                replacement = unichr(htmlentitydefs.name2codepoint[entity[1:-1]])
            except KeyError:
                continue        
        string = string.replace(entity, replacement)
    return string

def string_decode(string):
    '''
    For cross compatibility between Python 2 and Python 3 strings.
    '''
    if PY_MAJOR_VERSION > 2:
        return bytes(string, 'utf-8').decode('unicode_escape')
    else:
        return string.decode('string_escape')

def tryDecode(string):
    codec = ('ascii', 'latin-1', 'utf-8')
    for c in codec:
        try:
            return string.decode(c)
        except UnicodeDecodeError:
            continue
    raise UnicodeDecodeError(string)

def string_decode(string):
    """
    For cross compatibility between Python 2 and Python 3 strings.
    """
    if PY_MAJOR_VERSION > 2:
        return bytes(string, "utf-8").decode("unicode_escape")
    else:
        return string.decode("string_escape")

def string_decode(string):
	'''
	For cross compatibility between Python 2 and Python 3 strings.
	'''
	if sys.version_info.major > 2:
		return bytes(string, 'utf-8').decode('unicode_escape')
	else:
		return string.decode('string_escape')

def mb_code(string, coding="utf-8"):
    if isinstance(string, unicode):
        return string.encode(coding)
    for c in ('utf-8', 'gb2312', 'gbk', 'gb18030', 'big5'):
        try:
            return string.decode(c).encode(coding)
        except:
            pass
    return string

def encoded(string, encoding='utf8'):
    """Cast string to binary_type.

    :param string: six.binary_type or six.text_type
    :param encoding: encoding which the object is forced to
    :return: six.binary_type
    """
    assert isinstance(string, string_types) or isinstance(string, binary_type)
    if isinstance(string, text_type):
        return string.encode(encoding)
    try:
        # make sure the string can be decoded in the specified encoding ...
        string.decode(encoding)
        return string
    except UnicodeDecodeError:
        # ... if not use latin1 as best guess to decode the string before encoding as
        # specified.
        return string.decode('latin1').encode(encoding)

def mapUTF8toXML(string):
	uString = string.decode('utf8')
	string = ""
	for uChar in uString:
		i = ord(uChar)
		if (i < 0x80) and (i > 0x1F):
			string = string + chr(i)
		else:
			string = string + "&#x" + hex(i)[2:] + ";"
	return string

def dewindows(string):
    h = HTMLParser.HTMLParser()
    try:
        string = string.decode("windows-1252")
    except:
        try:
            string = string.decode("windows-1251")
        except:
            try:
                string = string.decode("ISO-8859-1")
            except:
                try:
                    string = string.decode("utf-8")
                except:
                    pass
    try:
        string = h.unescape(string)
    except:
        pass
    return string

def to_unicode(string):
    """
        Converts a 'string' to unicode
    """
    if not isinstance(string, unicode):
        if not isinstance(string,str):
            raise TypeError('You are required to pass either unicode or string here, not: %r (%s)' % (type(string), string))
        try:
            string = string.decode("UTF-8")
        except UnicodeDecodeError, exc:
            raise TypeError(str(exc))

 def assertContained(self, values, string, additional_info=''):
   if type(values) not in [list, tuple]: values = [values]
   for value in values:
     if type(value) is unicode: string = string.decode('UTF-8') # If we have any non-ASCII chars in the expected string, treat the test string from ASCII as UTF8 as well.
     if type(string) is not str and type(string) is not unicode: string = string()
     if value in string: return # success
   raise Exception("Expected to find '%s' in '%s', diff:\n\n%s\n%s" % (
     limit_size(values[0]), limit_size(string),
     limit_size(''.join([a.rstrip()+'\n' for a in difflib.unified_diff(values[0].split('\n'), string.split('\n'), fromfile='expected', tofile='actual')])),
     additional_info
   ))

def fsdecode(string):
    u"""Decode byte strings to unicode with file system encoding.
    
    This function is modelled after its namesake in the Python 3 os.path module.
    """
    if isinstance(string, str):
        return string.decode( sys.getfilesystemencoding() )
    elif isinstance(string, unicode):
        return string
    else:
        raise TypeError("argument is not of string type: {!r}".format(string))