def get_text(self, mess):
        text =[]
        html =[]
        char_set = mess.get_charsets()

        for part in mess.walk():
            type    = part.get_content_type()
            charset = part.get_content_charset()
            if type == 'text/plain':
                body = part.get_payload()

                # Determine what character set they use and convert to utf8
                body = quopri.decodestring(body)
                body = decode_heuristically(body, enc = charset, denc = "utf-8")
                if body != None:
                    text.append(body[0])
   
            elif type == 'multipart/alternative':
                body = part.get_payload()
                if isinstance(body,ListType):
                    for b in body:
                        tx,ht = self.get_text(b)
                        text.append(tx)
                        html.append(ht)
                        
                else:
                    body = quopri.decodestring(body)
                    body = quopri.decodestring(body)
                    body = decode_heuristically(body, enc = charset, denc = "utf-8")
                    if body != None:
                        htlm.append(body[0])
                
        return text,html

 def get_first_text_part(self, msg):
     maintype = msg.get_content_maintype()
     if maintype == 'multipart':
         for part in msg.get_payload():
             print part.get_content_charset(part.get_payload())
             if part.get_content_maintype() == 'text':
                 resp= ' '
                 if part['Content-Transfer-Encoding'] == 'quoted-printable':
                     resp= quopri.decodestring(part.get_payload())
                 if part.get_content_charset(False):
                     resp = part.get_payload().decode(part.get_content_charset())
                     print resp
                 return resp
         for part in msg.get_payload():             
             return self.get_first_text_part(part)
     elif maintype == 'text':
         resp= ''
         print msg.get_content_charset(msg.get_payload())
         if msg['Content-Transfer-Encoding'] == 'quoted-printable':
             resp= quopri.decodestring(msg.get_payload())
         if msg.get_content_charset(False):
             resp = msg.get_payload().decode(msg.get_content_charset())
         print resp
         return resp
     else:
         return ' '

def handleOutgoingMail (ctx, mail):
	#imprime(mail)
	uri = __aps__['uri']
	if uri:
		found = None
		for line in mail.head:
			if line.lower ().startswith ('list-unsubscribe:'):
				found = line
				break
		if found is None:	
			# Tentando extrair link embutido na newsletter
			if mail.body is not None:
				soup = BeautifulSoup(quopri.decodestring(mail.body), "html.parser")
				linkSair = soup.find('a',id='linkUnsubscribe')
				
				if linkSair is not None:
					if linkSair['href'].lower().find("form.do") != -1:
						novoLink = linkSair['href']
					else:
					# Substituindo link pelo mnemônico, a fim de permitir reconhecimento em alguns leitores de e-mails
						novoLink = (uri % linkSair['href'][linkSair['href'].lower().find("uid=")+4:])
						er = re.compile(r"<a[^<]*linkUnsubscribe.*?>",re.IGNORECASE|re.DOTALL)
						
						linkInserido = quopri.decodestring(re.search(er,mail.body).group())
						erStyle = re.compile(r"(style=.*?)[a-z].=",re.IGNORECASE|re.DOTALL)
						styleAdd = re.search(erStyle,linkInserido).group(1) if re.search(erStyle,linkInserido) else ""

						mail.body = re.sub(er,quopri.encodestring(("<a %s href=%s>" % (styleAdd,novoLink))),mail.body)
						
					mail.head.append ('List-Unsubscribe: <%s>' % novoLink)
					#imprime(mail)
					return

 def hashLoginPassword(username, password, sessionid):
     """ Hashes login password """
     username = quopri.decodestring(username)
     username = username.lower()
     password = quopri.decodestring(password)
     sha = hashlib.sha1(username + password).hexdigest()
     sha = hashlib.sha1(sha + sessionid).hexdigest()
     return sha

def headerUnicode(mimebytestring):
    h = email.header.decode_header(mimebytestring)
    res = []
    for hh in h:
        if hh[1] is None:
            res.append(unicode(quopri.decodestring(hh[0])))
        else:
            res.append(unicode(quopri.decodestring(hh[0]).decode(hh[1])))
    return u" ".join(res)

def maintain_rfc_parse(message):
    """
    This function parses an email and returns an array with different parts of the message
    but leaves the email still RFC compliant so that it works with Mail-Parser Plus app.
    Attachment headers are left in tact.
    :param message: This represents the email to be checked for attached email.
    :type message: email message object
    :return: Returns a email message formatted as a string
      :rtype: str
    """
    if not message.is_multipart():
        reformatted_message = quopri.decodestring(
                                message.as_string().encode('ascii', 'ignore')
                            ).decode("utf-8",'ignore')
        return reformatted_message
    boundary = message.get_boundary()
    new_payload = '--' + boundary
    for i in message.get_payload():
        content_type = i.get_content_type()
        extension = str(os.path.splitext(i.get_filename() or '')[1]).lower()
        if extension in TEXT_FILE_EXTENSIONS or content_type in SUPPORTED_CONTENT_TYPES or \
           i.get_content_maintype() == 'text':
            text_content = i.as_string().encode('ascii', 'ignore')
            text_content = quopri.decodestring(text_content).decode("utf-8",'ignore')
            new_payload += '\n' + text_content
        else:
            replace = re.sub(r'(?:\n\n)[\s\S]+',r'\n\n#UNSUPPORTED_ATTACHMENT:',i.as_string())
            filename = i.get_filename()
            charset = i.get_content_charset()
            try:
                md5 = hashlib.md5(i.get_payload(None,True)).hexdigest()
                sha256 = hashlib.sha256(i.get_payload(None,True)).hexdigest()
            except:
                md5 = ''
                sha256 = ''
            replace_string = """
file_name = %(filename)s
type = %(content_type)s
charset = %(charset)s
md5 = %(md5)s
sha256 = %(sha256)s
"""
            metadata = replace_string % dict(
                content_type=content_type, 
                filename=filename, 
                charset=charset,
                md5=md5,
                sha256=sha256,
            )
            new_payload += '\n' \
                + replace \
                + metadata
        new_payload += '\n--' + boundary
    new_payload += '--'
    message.set_payload(new_payload)
    return message.as_string()

 def set_message(self, message):
     self.sender = get_sender(message['Subject'])
     self.to = message['To']
     if message.is_multipart():
         src = unicode(quopri.decodestring(message.get_payload()[0]))
     else:
         src = unicode(quopri.decodestring(message.get_payload()))
     self.body, rest = split_body(src)
     self.loc = get_loc(rest)
     self.url = get_link(rest)
     self.raw = message.as_string()
     self.sent = datetime.datetime(*rfc822.parsedate(message['Date'])[:6])

def decodeContentType(part):
    tmp = re.search('(?si)Content-Type: +(.*?)(?: |\n|;)', part)
    if tmp!=None:
        content_type = tmp.group(1)
    else:
        content_type = ''
    tmp = re.search('charset="{0,1}([A-Za-z0-9\-]+)', part)
    if tmp!=None:
        enc = tmp.group(1)
    else:
        enc = ''
    tmp = re.search('(?s).*?(?:\n\n|\r\n\r\n)(.*)', part)
    if tmp!=None:
        body = tmp.group(1)
    else:
        body = ''
    tmp = re.search('(?si)(content-disposition: *attachment)', part)
    if tmp == None:
        tmp = re.search('(?si)(content-disposition: *inline)', part)
    if tmp!=None:
        attachment = 1
        tmp = re.search('(?si)content-type:.*?(?:\n\n|\r\n\r\n)(.*)', body)
        if tmp!=None:
            body = tmp.group(1)
    else:
        attachment = 0
    tmp = re.search('Content-Transfer-Encoding: +([a-zA-Z0-9\-]*)', part)
    if tmp!=None:
        cte = tmp.group(1)
        if cte.lower() == 'base64' and (content_type=='text/plain' or content_type=='text/html') and attachment==0:
            if (enc==''):
                body = base64.b64decode(body.encode('utf-8')).decode('utf-8', 'replace')
            else:
                body = base64.b64decode(body.encode('utf-8')).decode(enc, 'replace')
        if cte.lower() == 'quoted-printable' and (content_type=='text/plain' or content_type=='text/html') and attachment==0:
            if enc=='':
                body = quopri.decodestring(body.encode('utf-8')).decode('utf-8')
            else:
                body = quopri.decodestring(body.encode('utf-8')).decode(enc)

    tmp = re.search('(?si)filename="{0,1}(.*?)(?:"|\n)', part)
    if tmp!=None:
        filename = decodeHeader(tmp.group(1))
    else:
        filename = ''
    tmp = re.search('(?si)Content-Id: *<([^>]*)>', part)
    if tmp!=None:
        content_id = tmp.group(1)
    else:
        content_id = ''
    return content_type, enc, body, attachment, filename, content_id

def get_msg_content(msg):
    from BeautifulSoup import BeautifulSoup
    text=''
    last=msg['id']
    if msg.is_multipart():
        text=get_from_mp(msg)
    else:
        text=msg.get_payload(decode=True)

    text=quopri.decodestring(text)
    enc=BeautifulSoup(text).originalEncoding
    msg['enc']=enc
    subre="<b><span style='font-weight:bold'>Subject:</span></b> Re:"
    print msg['subj_e']


    if enc and enc in ['ISO-8859-2']:
        try:
            msg['text']=text.decode('cp1251')
        except:
            msg['text']=text.replace('\x98','').decode('cp1251')
    elif enc and enc not in ['utf8','utf-8']:
        msg['text']=text.decode(enc)
    else:
        msg['text']=text

    return msg['text']

    def test_notifications_for_new_post(self):
        ws = self.portal['workspaces']['workspace-1']
        people = self.portal['people']
        self.mailhost.messages = []
        login(self.portal, 'test_user_1')

        post = helpers.create_post(ws, 'new-post', title=u"New Post",
            text=richtextify(u"<p>test</p><p>test</p>123"))

        self.assertEqual(len(self.mailhost.messages), 0)

        helpers.publish_post(post)

        self.assertEqual(len(self.mailhost.messages), 2)

        recipients = [unicode(m['To']) for m in self.mailhost.messages]
        expected_recipients = []
        for uid in ws.getMemberIds():
            name = getattr(people.get(uid, None), 'title', uid)
            email = getattr(people.get(uid, None), 'emailAddress', None)
            if not email:
                continue
            expected_recipients.append('{0} <{1}>'.format(name, email))
        self.assertEqual(recipients, expected_recipients)

        for message in self.mailhost.messages:
            self.assertEqual(unicode(message['Subject']), u"[Plone site] " +
                u"New post: “New Post” by Test User in Workspace 1")

            msg_text = quopri.decodestring(message.get_payload())
            expected_msg_text = u"test\n\ntest\n123"
            self.assertEqual(msg_text, expected_msg_text)

def decode_transfer_encoding(encoding, body):
    if encoding == 'base64':
        return _base64_decode(body)
    elif encoding == 'quoted-printable':
        return quopri.decodestring(body)
    else:
        return body

 def _infer_text_fragment_inner(self, title, body, post_id):
     # dead code? If not needs to be refactored with langstrings
     body = sanitize_html(body, [])
     quote = self.body.replace("\r", "")
     try:
         # for historical reasons
         quote = quopri.decodestring(quote)
     except:
         pass
     quote = sanitize_html(quote, [])
     if quote != self.body:
         self.body = quote
     quote = quote.replace("\n", "")
     start = body.find(quote)
     lookin = 'message-body'
     if start < 0:
         xpath = "//div[@id='%s']/div[class='post_title']" % (post_id)
         start = title.find(quote)
         if start < 0:
             return None
         lookin = 'message-subject'
     xpath = "//div[@id='message-%s']//div[@class='%s']" % (
         Post.uri_generic(post_id), lookin)
     tfi = self.db.query(TextFragmentIdentifier).filter_by(
         extract=self).first()
     if not tfi:
         tfi = TextFragmentIdentifier(extract=self)
     tfi.xpath_start = tfi.xpath_end = xpath
     tfi.offset_start = start
     tfi.offset_end = start + len(quote)
     return tfi

    def test_notifications_for_new_reply(self):
        ws = self.portal['workspaces']['workspace-1']
        people = self.portal['people']
        post = ws['post-2']
        self.mailhost.messages = []
        login(self.portal, 'test_user_1')
        helpers.create_reply(post, 'new-reply',
            text=richtextify(u"<p>test</p><p>test</p>123"))

        self.assertEqual(len(self.mailhost.messages), 2)

        recipients = [unicode(m['To']) for m in self.mailhost.messages]
        expected_recipients = []
        for uid in ['test_user_1', 'test_user_2']:
            name = getattr(people.get(uid, None), 'title', uid)
            email = getattr(people.get(uid, None), 'emailAddress', None)
            expected_recipients.append('{0} <{1}>'.format(name, email))
        self.assertEqual(recipients, expected_recipients)

        for message in self.mailhost.messages:
            self.assertEqual(unicode(message['Subject']), u"[Plone site] " +
                u"Test User replied to “Post 2” in Workspace 1")

            msg_text = quopri.decodestring(message.get_payload())
            expected_msg_text = u"test\n\ntest\n123"
            self.assertEqual(msg_text, expected_msg_text)

    def __init__(self, message, message_id=None, gpg=None):
        """
        :arg str message: raw text of the email
        :arg str message_id: IMAP message ID or maildir message path
        :arg gpg: :class:`GnuPG` instance or None (which will create one)
        """

        Message.__init__(self, message)
        self._message_id = message_id
        if not gpg:
            self._gpg = GnuPG()
        else:
            self._gpg = gpg

        self._content_types = ["text/plain", "text/html",
            "application/pgp-signature", "application/pgp-keys",
            "application/octet-stream"]

        self._parts = []
        for part in self.walk():
            content_type = part.get_content_type()
            if content_type in self._content_types:
                payload = quopri.decodestring(part.get_payload().strip())
                self._parts.append( (content_type, payload) )

        self._parse_for_openpgp()

    def _get_verification_data(self, email_address):
        """A private helper for public helpers below.

        Note: We have two different public helpers here for verification
        code and link so that functional tests don't need to deal with
        idioms like:
            vcode, ignored = get_verification_for_address(email_address).
        """
        email_msg = mail.get_latest_email_sent_to(email_address)
        vcode = link = None
        if email_msg:
            # The body is encoded as quoted-printable. This affects any line
            # longer than a certain length.  Decode now to not have to worry
            # about it in the regexen.
            body = quopri.decodestring(email_msg.get_payload())
            # get code
            match = re.search(
                '(Here is your confirmation code:|Copy and paste the '
                'confirmation code below into the desktop application.)'
                '(.*?)(Enter|If you made|If you don)',
                body, re.S)
            if match:
                vcode = match.group(2).strip()
            else:
                vcode = None
            # get link
            match = re.search(
                'confirm your (?:account|email address|reset):(.*)If',
                body, re.S)
            link = None
            if match:
                link = match.group(1).strip()
        return vcode, link

def linksToEpisodesSY(msg):
    """extract the links to episodes in a seriesyonkis email"""
    # TESTED
    if msg is None:
        raise TypeError

    # TODO testing how to encode an email right
    #logging.info(repr(str_msg))

    #syMsg = f.parsestr(str)
    #syMsg = f.parsestr(str_msg)
    #syMsg = msg
    # TODO: aclarar si esto es asi, o no. mientras tanto, esta funcion
    # falla con errores de encoding, asi que no me vale.
    # con nosetes funciona, con normal falla  -> con encode
    # syMsg = quopri.decodestring(msg.encode("UTF-8"))
    # con nosetest falla, con normal funcion -> sin encode
    syMsg = quopri.decodestring(msg)
    soup = bs4.BeautifulSoup(syMsg)

    episodes = set()

    for link in soup.find_all("a"):
        cleanLink = link.get('href', '')
        if "capitulo" in cleanLink:
            episodes.add(cleanLink.decode())

    lst = list(episodes)
    lst.sort()

    return lst

    def detachFile(self, att_name, filecontent, date=None):
        """ Writes the file and return the corresponding html

            The html file will store a link to the detached file
            If the file already exists, we compute a new name and create a link to the computed name file
            and to the file that has the same name...
            date is the date to use for the detached file
        """
        att_name=os.path.split(att_name)[1] # sometimes, the name of the attached file contains the whole path which is not wanted
        if att_name.find('=?') != -1: # If the filename is encoded, we decode it
            filename=quopri.decodestring(self.decode_Q(att_name).encode('latin1')) # and encode it in latin1 (you may change this to utf-8)
        else:
            filename=att_name
        filelist=[filename] # 
        # we write the attached file
        # Sometimes, due to formating, we find \n, \r or \t characters in the filenames. So we cut them off
        ofname=os.path.join(prm.attachedFilesDir, filename.strip().replace('\n','').replace('\r','').replace('?','').expandtabs(0))
        if os.path.exists(ofname):
            i=1
            fname=ofname
            sfname=os.path.splitext(fname)
            while os.path.exists(fname):
                fname="%s[%d]%s" % (sfname[0], i, sfname[1])
                i+=1
            filelist.append(os.path.split(fname)[-1])
            ofname=fname # the outfile name is the new one
        of=open(ofname,'wb')
        of.write(filecontent)
        of.close()
        if date:
            os.utime(ofname, (date, date))
        return self.getHtmlLinksFileContent(filelist)

def parse_part(part):
	part = part.strip();
	# parse the part description (first three lines)
	# get Content-Type
	pat1 = 'Content-Type: (.*)';
	pat1_res = re.search(pat1, part, re.I);
	ctype = pat1_res.groups()[0].strip() if pat1_res else '';
	# get Content-Transfer-Encoding
	pat2 = 'Content-Transfer-Encoding: (.*)';
	pat2_res = re.search(pat2, part, re.I);
	cenc = pat2_res.groups()[0].strip() if pat2_res else '';
	# get Content-Location
	pat3 = 'Content-Location: (.*)';
	pat3_res = re.search(pat3, part, re.I);
	cloc = pat3_res.groups()[0].strip() if pat3_res else '';
	# check part description
	if cenc == '':
		return (-1, ctype, cenc, cloc, '');
	# parse the contents
	try:
		contents = part.split('\n\n', 1)[1];
	except:
		contents = part.split('\n\r\n', 1)[1];
	if cenc == 'base64':
		s = base64.b64decode(contents);
	elif cenc == 'quoted-printable':
		s = quopri.decodestring(contents);
	return (0, ctype, cenc, cloc, s);

def parse_email_set_var(context, address, expression):
    assert context.persona is not None, u'no persona is setup'
    msgs = context.mail.user_messages(address)
    assert msgs, u'no email received'
    mail = email.message_from_string(msgs[-1])
    text = quopri.decodestring(mail.get_payload()).decode("utf-8")
    parse_text(context, text, expression)

def decode_transfer_encoding(encoding, body):
    if encoding == "base64":
        return base64.b64decode(body)
    elif encoding == "quoted-printable":
        return quopri.decodestring(body)
    else:
        return body