def test_unicode(self):
        """Title can contain unicode characters."""
        if 'utf-8' != sys.getdefaultencoding():
            raise SkipTest("encoding '%s' can't deal with snowmen"
                    % sys.getdefaultencoding())

        rv = self.run_script(r"""
            snowman = u'\u2603'

            import setproctitle
            setproctitle.setproctitle("Hello, " + snowman + "!")

            import os
            print os.getpid()
            print os.popen("ps -o pid,command 2> /dev/null").read()
        """)
        lines = filter(None, rv.splitlines())
        pid = lines.pop(0)
        pids = dict([r.strip().split(None, 1) for r in lines])

        snowmen = [
            u'\u2603',          # ps supports unicode
            r'\M-b\M^X\M^C',    # ps output on BSD
            r'M-bM^XM^C',       # ps output on OS-X
        ]
        title = self._clean_up_title(pids[pid])
        for snowman in snowmen:
            if title == "Hello, " + snowman + "!":
                break
        else:
            self.fail("unexpected ps output: %r" % title)

def exec_process(cmdline, silent=True, catch_enoent=True, input=None, **kwargs):
    """Execute a subprocess and returns the returncode, stdout buffer and stderr buffer.
    Optionally prints stdout and stderr while running."""
    try:
        sub = subprocess.Popen(args=cmdline, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
        stdout, stderr = sub.communicate(input=input)

        if type(stdout) != type(""):
            # decode on Python 3
            # do nothing on Python 2 (it just doesn't care about encoding anyway)
            stdout = stdout.decode(sys.getdefaultencoding(), "replace")
            stderr = stderr.decode(sys.getdefaultencoding(), "replace")

        returncode = sub.returncode
        if not silent:
            sys.stdout.write(stdout)
            sys.stderr.write(stderr)
    except OSError as e:
        if e.errno == errno.ENOENT and catch_enoent:
            raise DistutilsError('"%s" is not present on this system' % cmdline[0])
        else:
            raise
    if returncode != 0:
        raise DistutilsError('Got return value %d while executing "%s", stderr output was:\n%s' % (returncode, " ".join(cmdline), stderr.rstrip("\n")))
    return stdout

def bug_1771_with_user_config(var):
    # sys.getdefaultencoding is considered as a never
    # returning function in the inconsistent_returns.rc file.
    if var == 1:
        sys.getdefaultencoding()
    else:
        return var * 2

def encodeASCII(string, language=None): #from Unicodize and plex scanner and other sources
  if string=="": return ""
  ranges = [ {"from": ord(u"\u3300"), "to": ord(u"\u33ff")}, {"from": ord(u"\ufe30"), "to": ord(u"\ufe4f")}, {"from": ord(u"\uf900"), "to": ord(u"\ufaff")},  # compatibility ideographs
             {"from": ord(u"\u30a0"), "to": ord(u"\u30ff")}, {"from": ord(u"\u2e80"), "to": ord(u"\u2eff")},                                                  # Japanese Kana    # cjk radicals supplement
             {"from": ord(u"\u4e00"), "to": ord(u"\u9fff")}, {"from": ord(u"\u3400"), "to": ord(u"\u4dbf")}]                                                  # windows: TypeError: ord() expected a character, but string of length 2 found #{"from": ord(u"\U00020000"), "to": ord(u"\U0002a6df")}, #{"from": ord(u"\U0002a700"), "to": ord(u"\U0002b73f")}, #{"from": ord(u"\U0002b740"), "to": ord(u"\U0002b81f")}, #{"from": ord(u"\U0002b820"), "to": ord(u"\U0002ceaf")}, # included as of Unicode 8.0                             #{"from": ord(u"\U0002F800"), "to": ord(u"\U0002fa1f")}  # compatibility ideographs
  encodings, encoding = ['iso8859-1', 'utf-16', 'utf-16be', 'utf-8'], ord(string[0])                                                                          #
  if 0 <= encoding < len(encodings):  string = string[1:].decode('cp949') if encoding == 0 and language == 'ko' else string[1:].decode(encodings[encoding])   # If we're dealing with a particular language, we might want to try another code page.
  if sys.getdefaultencoding() not in encodings:
    try:     string = string.decode(sys.getdefaultencoding())
    except:  pass
  if not sys.getfilesystemencoding()==sys.getdefaultencoding():
    try:     string = string.decode(sys.getfilesystemencoding())
    except:  pass
  string = string.strip('\0')
  try:       string = unicodedata.normalize('NFKD', string)    # Unicode  to ascii conversion to corect most characters automatically
  except:    pass
  try:       string = re.sub(RE_UNICODE_CONTROL, '', string)   # Strip control characters.
  except:    pass
  try:       string = string.encode('ascii', 'replace')        # Encode into Ascii
  except:    pass
  original_string, string, i = string, list(string), 0
  while i < len(string):                                       ### loop through unicode and replace special chars with spaces then map if found ###
    if ord(string[i])<128:  i = i+1
    else: #non ascii char
      char, char2, char3, char_len = 0, "", [], unicodeLen(string[i])
      for x in range(0, char_len):
        char = 256*char + ord(string[i+x]); char2 += string[i+x]; char3.append(string[i+x])
        if not x==0: string[i] += string[i+x]; string[i+x]=''
      try:    asian_language = any([mapping["from"] <= ord("".join(char3).decode('utf8')) <= mapping["to"] for mapping in ranges])
      except: asian_language = False
      if char in CHARACTERS_MAP:  string[i]=CHARACTERS_MAP.get( char )
      elif not asian_language:    Log("*Character missing in CHARACTERS_MAP: %d:'%s'  , #'%s' %s, string: '%s'" % (char, char2, char2, char3, original_string))
      i += char_len
  return ''.join(string)

def detect_console_encoding():
    """
    Try to find the most capable encoding supported by the console.
    slightly modified from the way IPython handles the same issue.
    """
    global _initial_defencoding

    encoding = None
    try:
        encoding = sys.stdout.encoding or sys.stdin.encoding
    except AttributeError:
        pass

    # try again for something better
    if not encoding or 'ascii' in encoding.lower():
        try:
            encoding = locale.getpreferredencoding()
        except Exception:
            pass

    # when all else fails. this will usually be "ascii"
    if not encoding or 'ascii' in encoding.lower():
        encoding = sys.getdefaultencoding()

    # GH3360, save the reported defencoding at import time
    # MPL backends may change it. Make available for debugging.
    if not _initial_defencoding:
        _initial_defencoding = sys.getdefaultencoding()

    return encoding

 def _get_user_identity(self):
     """Determine the identity to use for new commits.
     """
     user = os.environ.get("GIT_COMMITTER_NAME")
     email = os.environ.get("GIT_COMMITTER_EMAIL")
     config = self.get_config_stack()
     if user is None:
         try:
             user = config.get(("user", ), "name")
         except KeyError:
             user = None
     if email is None:
         try:
             email = config.get(("user", ), "email")
         except KeyError:
             email = None
     if user is None:
         import getpass
         user = getpass.getuser().encode(sys.getdefaultencoding())
     if email is None:
         import getpass
         import socket
         email = ("{}@{}".format(getpass.getuser(), socket.gethostname())
                  .encode(sys.getdefaultencoding()))
     return (user + b" <" + email + b">")

def upload2(pyew, doprint=True):
    """ upload file to virscan.org"""
    print sys.getdefaultencoding()
    register_openers()
    url = r'http://up.virscan.org/up.php'


    datagen, headers = multipart_encode({"upfile": open(pyew.filename, "rb"),
        'UPLOAD_IDENTIFIER' : 'KEYbc7cf6642fc84bf67747f1bbdce597f0',
        'langkey' : '1',
        'setcookie' : '0',
        'tempvar' : '',
        'fpath' : 'C:\\fakepath\\'+pyew.filename
        })


    request = urllib2.Request(url, datagen, headers)
    request.add_header('Host', 'up.virscan.org')
    request.add_header('Cache-Control', 'max-age=0')
    request.add_header('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.104 Safari/537.36')
    request.add_header('Host', 'up.virscan.org')
    request.add_header('Accept-Encoding', 'gzip, deflate')
    request.add_header('Accept-Language', 'zh-CN,zh;q=0.8,en;q=0.6')

    resp = urllib2.urlopen(request).read()
    if re.findall("innerHTML='(.*?) file upload", resp):
        print "Upload File Failed"
    else:
        print "Upload Success"

    def send_email(self, to_, from_=None, subject=None, body=None,
                   subtype="plain", charset="utf-8"):

        message = MIMEText(body, subtype, charset)

        if subject:
            subject_header = Header()
            subject = (codecs.decode(bytearray(subject, sys.getdefaultencoding()), charset)
                       if isinstance(subject, str) else subject)
            subject_header.append(subject.strip())
            message["Subject"] = subject_header

        from_ = from_ or self.default_sender
        from_ = (codecs.decode(bytearray(from_, sys.getdefaultencoding()), charset)
                 if isinstance(from_, str) else from_)
        from_realname, from_addr = parseaddr(from_)
        from_header = Header()
        from_header.append(formataddr((from_realname, from_addr)))
        message['From'] = from_header

        to_ = (codecs.decode(bytearray(to_, sys.getdefaultencoding()), charset)
               if isinstance(to_, str) else to_)
        to_realname, to_addr = parseaddr(to_)
        to_header = Header()
        to_header.append(formataddr((to_realname, to_addr)))
        message['To'] = to_header

        self._send(message, from_addr, to_addr)

 def __init__(self, msg, stderr=None, stdout=None):
     # type: (str, bytes, bytes) -> None
     if stderr:
         msg += '\n[stderr]\n' + stderr.decode(sys.getdefaultencoding(), 'replace')
     if stdout:
         msg += '\n[stdout]\n' + stdout.decode(sys.getdefaultencoding(), 'replace')
     super().__init__(msg)

def runCommand(cmd, instream = None, msg = '', upon_succ = None, show_stderr = False, return_zero = True):
    if isinstance(cmd, str):
        cmd = shlex.split(cmd)
    popen_env = os.environ.copy()
    popen_env.update(env.path)
    try:
        tc = subprocess.Popen(cmd, stdin = subprocess.PIPE,
                              stdout = subprocess.PIPE, stderr = subprocess.PIPE,
                              env=popen_env)
        if instream:
            instream = instream.encode(sys.getdefaultencoding())
            out, error = tc.communicate(instream)
        else:
            out, error = tc.communicate()
        out = out.decode(sys.getdefaultencoding())
        error = error.decode(sys.getdefaultencoding())
        if return_zero:
            if tc.returncode < 0:
                raise ValueError ("Command '{0}' was terminated by signal {1}".format(cmd, -tc.returncode))
            elif tc.returncode > 0:
                raise ValueError ("{0}".format(error))
        if error.strip() and show_stderr:
            env.logger.error(error)
    except OSError as e:
        raise OSError ("Execution of command '{0}' failed: {1}".format(cmd, e))
    # everything is OK
    if upon_succ:
        # call the function (upon_succ) using others as parameters.
        upon_succ[0](*(upon_succ[1:]))
    return out.strip(), error.strip()

    def send_html_email(self, to_, from_=None, subject=None, text=None,
                        html=None, charset="utf-8"):

        message = MIMEMultipart("alternative")

        if subject:
            subject_header = Header()
            subject = (codecs.decode(bytearray(subject, sys.getdefaultencoding()), charset)
                       if isinstance(subject, str) else subject)
            subject_header.append(subject.strip())
            message["Subject"] = subject_header

        from_ = from_ or self.default_sender
        from_ = (codecs.decode(bytearray(from_, sys.getdefaultencoding()), charset)
                 if isinstance(from_, str) else from_)
        from_realname, from_addr = parseaddr(from_)
        from_header = Header()
        from_header.append(formataddr((from_realname, from_addr)))
        message['From'] = from_header

        to_ = (codecs.decode(bytearray(to_, sys.getdefaultencoding()), charset)
               if isinstance(to_, str) else to_)
        to_realname, to_addr = parseaddr(to_)
        to_header = Header()
        to_header.append(formataddr((to_realname, to_addr)))
        message['To'] = to_header

        message.attach(MIMEText(text, "plain", charset))
        message.attach(MIMEText(html, "html", charset))

        self._send(message, from_addr, to_addr)

    def body(self):
        r'''
        Extract the plain text body of the message with signatures
        stripped off.

        :param message: the message to extract the body from
        :type  message: :class:`notmuch.Message`
        :returns: the extracted text body
        :rtype:   :class:`list` of :class:`str`o
        '''
        if self._body is None:
            content = []
            for part in self.mail.walk():
                if part.get_content_type() == 'text/plain':
                    raw_payload = part.get_payload(decode=True)
                    encoding = part.get_content_charset()
                    if encoding:
                        try:
                            raw_payload = raw_payload.decode(encoding,
                                                             'replace')
                        except LookupError:
                            raw_payload = raw_payload.decode(
                                sys.getdefaultencoding(), 'replace')
                    else:
                        raw_payload = raw_payload.decode(
                            sys.getdefaultencoding(), 'replace')

                    lines = raw_payload.split('\n')
                    lines = self._strip_signatures(lines)

                    content.append('\n'.join(lines))
            self._body = '\n'.join(content)

        return self._body

def main(query, exclude):
    print sys.getdefaultencoding()
    createWorkingDirectories(query)
    getGooogleResults(query, exclude)
    file = open("researchResults/" + query + "/data/searchResults.txt")
    line = file.readline()
    while line:

 def DetectEncodingAndRead(self, fd):
     encodings = ["utf-8", "utf-16"]
     if locale.getpreferredencoding() not in encodings:
         encodings.append(locale.getpreferredencoding())
     if sys.getdefaultencoding() not in encodings:
         encodings.append(sys.getdefaultencoding())
     if locale.getdefaultlocale()[1] not in encodings:
         encodings.append(locale.getdefaultlocale()[1])
     if sys.getfilesystemencoding() not in encodings:
         encodings.append(sys.getfilesystemencoding())
     if 'latin-1' not in encodings:
         encodings.append('latin-1')
         
     for enc in encodings:
         fd.seek(0)
         try:
             reader = codecs.getreader(enc)(fd)
             content = reader.read()
         except:
             continue
         else:
             self._encoding = enc
             logger.info("Detect file %s 's encoding is %s" % (self.GetFilename(), self._encoding))
             return content
         
     logger.error("Fail to detect the encoding for file %s" % self.GetFilename())
     return None

	def unzipDataFile(self, dataFileName, unzipFileName):
		print sys.getdefaultencoding()
		file_object_Z = open(dataFileName, "rb")
		file_object_W = open(unzipFileName, "wb")
		try:
			nDictCnt = 0
			while True:
				urlLen = struct.unpack("L",file_object_Z.read(4))
				srcUrl = file_object_Z.read(urlLen[0])

				htmlLen = struct.unpack("L",file_object_Z.read(4))
				zipHtml = file_object_Z.read(htmlLen[0])
				srcHtml = zlib.decompress(zipHtml)

				htmlLen = struct.pack("L",len(srcHtml))

				urlLen = struct.pack("L",urlLen[0])
				file_object_W.write(urlLen)
				file_object_W.write(srcUrl)

				file_object_W.write(htmlLen)
				file_object_W.write(srcHtml)
				#print srcHtml
				#break

		except IOError, e:
			print "IOError:", e

    def execute_shell(cmd, input='', timeout=None):
        """
        :param cmd:
        :param input: sent to stdin
        :return: returncode, stdout, stderr.
        :raise: subprocess.TimeoutExpired (I kill the process!)
        """

        proc_stdin = subprocess.PIPE if input != '' else None
        proc_input = input if input != '' else None

        p = subprocess.Popen(cmd,
                             stdin=proc_stdin,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             shell=True,
                             preexec_fn=os.setsid if timeout else None)  # http://stackoverflow.com/a/4791612/801203
        try:
            if proc_input:
                out, err = p.communicate(bytes(proc_input, encoding=sys.getdefaultencoding()), timeout=timeout)
            else:
                out, err = p.communicate(timeout=timeout)
        except subprocess.TimeoutExpired:
            logging.debug('Timeout expired, killing the process')
            os.killpg(os.getpgid(p.pid), signal.SIGTERM)
            p.communicate()
            raise

        return p.returncode,\
               str(out, sys.getdefaultencoding()),\
               str(err, sys.getdefaultencoding())

def main():
    args = sys.argv
    cur_sys = system()
    file_name = GLOBALS["db_name"]+"-"+str(options.port)+".log"
    # if cur_sys == "Darwin":
    #     f = "/Users/"+os.getlogin()+"/Desktop/"+ file_name
    # elif cur_sys == "Linux":
    #     f = os.getcwd() + "/" + file_name
    # else:
    #     raise NotImplementedError
    # args.append("--log_file_prefix=" + f)
    # logging.basicConfig(filename=f, level=logging.DEBUG)

    tornado.options.parse_command_line()
    applicaton = Application()
    http_server = tornado.httpserver.HTTPServer(applicaton, xheaders=True)

    http_server.listen(options.port)
    print("="*50)
    print("initializing program with port : ", options.port)
    print("="*50)
    print("my ip is : ", socket.gethostbyname(socket.gethostname()))
    print("="*50)
    print("File system DEFAULT Encoding-type is : ", sys.getdefaultencoding())
    print("File system Encoding-type is : ", sys.getfilesystemencoding())
    print("="*50)
    logging.info("File system DEFAULT Encoding-type is : " + str(sys.getdefaultencoding()))
    logging.info("File system Encoding-type is : " +  str(sys.getfilesystemencoding()))

    ioloop = tornado.ioloop.IOLoop.instance()
    ioloop.start()

 def test_startLogging(self):
     """
     startLogging() installs FileLogObserver and overrides sys.stdout and
     sys.stderr.
     """
     origStdout, origStderr = sys.stdout, sys.stderr
     self._startLoggingCleanup()
     # When done with test, reset stdout and stderr to current values:
     fakeFile = StringIO()
     observer = log.startLogging(fakeFile)
     self.addCleanup(observer.stop)
     log.msg("Hello!")
     self.assertIn("Hello!", fakeFile.getvalue())
     self.assertIsInstance(sys.stdout, log.StdioOnnaStick)
     self.assertEqual(sys.stdout.isError, False)
     encoding = getattr(origStdout, "encoding", None)
     if not encoding:
         encoding = sys.getdefaultencoding()
     self.assertEqual(sys.stdout.encoding, encoding)
     self.assertIsInstance(sys.stderr, log.StdioOnnaStick)
     self.assertEqual(sys.stderr.isError, True)
     encoding = getattr(origStderr, "encoding", None)
     if not encoding:
         encoding = sys.getdefaultencoding()
     self.assertEqual(sys.stderr.encoding, encoding)

def new_dataset(model):
    """Initializes the dataset."""
    # print model.data_path
    dataset_id = model.id
    dataset_path = model.data_path
    print dataset_path
    for root, dirs, files in os.walk(dataset_path):
        first = 1
        print files

        for image in files:
            print image
            print sys.getdefaultencoding()
            image = image.encode("utf8")
            if image.find(".jpg") > 0:
                print image
                if image.find("thumb") > 0:
                    continue
                classname = root.split('/')[-1]
                print classname
                if first == 1:
                    new = Class(model.id,classname)
                    db.session.add(new)
                    first = 0
                this_class = Class.query.filter(and_(Class.class_name == classname,Class.dataset_id == dataset_id)).first()
                print dataset_id
                print image

                img = Image(dataset_id,this_class.id,os.path.join(root,image),"unlabelled")
                db.session.add(img)
            db.session.commit()

def get_build_message(message, outputs_url, logs_urls=None,
                      applied_reviews=None,
                      failed_command=None):
    """Retrieves build message."""
    if logs_urls is None:
        logs_urls = []
    if applied_reviews is None:
        applied_reviews = []
    build_msg = "%s\n\n" % message
    if applied_reviews:
        build_msg += "Reviews applied: `%s`\n\n" % applied_reviews
    if failed_command:
        build_msg += "Failed command: `%s`\n\n" % failed_command
    build_msg += ("All the build artifacts available"
                  " at: %s\n\n" % (outputs_url))
    logs_msg = ''
    for url in logs_urls:
        response = urllib2.urlopen(url)
        log_content = response.read().decode(sys.getdefaultencoding())
        if log_content == '':
            continue
        file_name = url.split('/')[-1]
        logs_msg += "- [%s](%s):\n\n" % (file_name, url)
        logs_msg += "```\n"
        log_tail = log_content.split("\n")[-LOG_TAIL_LIMIT:]
        logs_msg += "\n".join(log_tail)
        logs_msg += "```\n\n"
    if logs_msg == '':
        return build_msg
    build_msg += "Relevant logs:\n\n%s" % (logs_msg)
    return build_msg.encode(sys.getdefaultencoding())