def test_url_unescape_unicode(self):
     tests = [
         ('%C3%A9', u('\u00e9'), 'utf8'),
         ('%C3%A9', u('\u00c3\u00a9'), 'latin1'),
         ('%C3%A9', utf8(u('\u00e9')), None),
     ]
     for escaped, unescaped, encoding in tests:
         # input strings to url_unescape should only contain ascii
         # characters, but make sure the function accepts both byte
         # and unicode strings.
         self.assertEqual(url_unescape(to_unicode(escaped), encoding), unescaped)
         self.assertEqual(url_unescape(utf8(escaped), encoding), unescaped)

 def test_url_escape_quote_plus(self):
     unescaped = '+ #%'
     plus_escaped = '%2B+%23%25'
     escaped = '%2B%20%23%25'
     self.assertEqual(url_escape(unescaped), plus_escaped)
     self.assertEqual(url_escape(unescaped, plus=False), escaped)
     self.assertEqual(url_unescape(plus_escaped), unescaped)
     self.assertEqual(url_unescape(escaped, plus=False), unescaped)
     self.assertEqual(url_unescape(plus_escaped, encoding=None),
                      utf8(unescaped))
     self.assertEqual(url_unescape(escaped, encoding=None, plus=False),
                      utf8(unescaped))

	def query(self,option,q):
		url_unescape(q,'utf-8')
		if(option == 'option1'):
			info = self.db.query('SELECT U.UID,U.USERNAME,U.CHINAME,D.DETECTTIME,D.STATUS FROM USER U LEFT OUTER JOIN \
								DETECT D ON U.UID = D.OWNER WHERE U.CHINAME=\'%s\'' % (q))
		elif(option == 'option2'):
			info = self.db.query('SELECT U.UID,U.USERNAME,U.CHINAME,D.DETECTTIME,D.STATUS FROM USER U LEFT OUTER JOIN \
								DETECT D ON U.UID = D.OWNER WHERE U.UID=%s' % (q))
		elif(option == 'option3'):
			info = self.db.query('SELECT U.UID,U.USERNAME,U.CHINAME,D.DETECTTIME,D.STATUS FROM USER U LEFT OUTER JOIN \
								DETECT D ON U.UID = D.OWNER WHERE U.USERNAME=\'%s\'' % (q))
		for item in info:
			item['DETECTTIME'] = datetime_handler(item['DETECTTIME'])

		return info

	def get_link(self, link):
		try:
			data 		= yield http_client(link, c_try=5, c_delay=self.delay)
			try:
				data 		= escape.url_unescape(data.split('proxy.link=',1)[1].split('"',1)[0].split('&',1)[0].strip())
			except:
				data 	= data.split('<param name="movie" value="',1)[1].split('"',1)[0]
				data 	= escape.url_unescape(data)
				if 'youtube' in data:
					data = data.split('?',1)[0].strip()
			print(link, data)
			return data
		except Exception as e:
			traceback.print_exc(file=sys.stdout)
			return None

  def get(self, leaderboard_type, leaderboard_object="", slash_separated_players=None, conjunction=None):
    if slash_separated_players:
      players = filter(None, slash_separated_players.split("/"))
    else:
      players = []
    if leaderboard_object in ["", "players/"]:
      unique_players = leaderboard_object == "players/"
      all_high_scores = get_all_high_scores(10, leaderboard_type, players, conjunction, unique_players=unique_players)
    elif leaderboard_object in ["games/"]:
      all_high_scores = get_all_high_games(10, leaderboard_type, players, conjunction)

    try:
      time_offset = int(url_unescape(self.get_cookie("time_offset")))
    except:
      time_offset = 0
    
    self.render(
        "leaderboard.html",
        players=players,
        all_high_scores=all_high_scores,
        leaderboard_types=[('alltime', 'All Time'), ('thisweek', 'This Week'), ('today', 'Today')],
        selected_leaderboard_type=leaderboard_type,
        leaderboard_object=leaderboard_object,
        time_offset=time_offset,
        conjunction=conjunction,
        game_type_info=GAME_TYPE_INFO)

    def get_blink_cookie(self, name):
        """Gets a blink cookie value"""
        value = self.get_cookie(name)

        if value != None:
            self.clear_cookie(name)
            return escape.url_unescape(value)

 def post(self,path=None):
     try:
         result = {}
         root = os.path.join(self.root,path) if path else self.root
         self._check_dir_(root)
         for key in self.request.files:
             for fileinfo in self.request.files[key]:
                 fname = url_unescape(fileinfo['filename'])
                 fNo = 0
                 fActual,fExt = os.path.splitext(fname)
                 files = os.listdir(root)
                 while fname in files:
                     fNo = fNo + 1
                     fname = "%s_%s%s" % (fActual,fNo,fExt)
                 
                 file_path = os.path.join(root,fname)
                 logging.info("uploading %s", file_path)
                 with open(file_path,'wb') as fh:
                     fh.write(fileinfo['body'])
                 info = {}
                 result[key]={
                              "name":fname,
                              "info":info
                              }
         self.write({"result":result})
     except Exception as ex:
         logging.exception(path)
         self.write({"error":str(ex)})

 def get(self, task, status_list):
     status_list = escape.url_unescape(status_list.lower()).split(',')
     self.write(pd.io.json.dumps(
         {status: list(map(tansform_bson_id,
                           self.application.mongo.tasks.find({'status': status,
                                                              'task': task})))
          for status in status_list}))

 def environ(request):
     """Converts a `tornado.httputil.HTTPServerRequest` to a WSGI environment.
     """
     hostport = request.host.split(":")
     if len(hostport) == 2:
         host = hostport[0]
         port = int(hostport[1])
     else:
         host = request.host
         port = 443 if request.protocol == "https" else 80
     environ = {
         "REQUEST_METHOD": request.method,
         "SCRIPT_NAME": "",
         "PATH_INFO": to_wsgi_str(escape.url_unescape(
             request.path, encoding=None, plus=False)),
         "QUERY_STRING": request.query,
         "REMOTE_ADDR": request.remote_ip,
         "SERVER_NAME": host,
         "SERVER_PORT": str(port),
         "SERVER_PROTOCOL": request.version,
         "wsgi.version": (1, 0),
         "wsgi.url_scheme": request.protocol,
         "wsgi.input": BytesIO(escape.utf8(request.body)),
         "wsgi.errors": sys.stderr,
         "wsgi.multithread": False,
         "wsgi.multiprocess": True,
         "wsgi.run_once": False,
     }
     if "Content-Type" in request.headers:
         environ["CONTENT_TYPE"] = request.headers.pop("Content-Type")
     if "Content-Length" in request.headers:
         environ["CONTENT_LENGTH"] = request.headers.pop("Content-Length")
     for key, value in request.headers.items():
         environ["HTTP_" + key.replace("-", "_").upper()] = value
     return environ

    def post(self):
        next_page = escape.url_unescape(self.get_argument("next"))
        id = int(self.get_argument("id"))
        type = self.get_argument("type")
        if type == "entry":
            meidodb.delete_entry(id)
            meidodb.delete_comment_by_entry(id)
        elif type == "comment":
            meidodb.delete_comment_by_id(id)

        # if next page is the blog just removed, jump to /

        if type == "entry" and next_page.find("/blog/") != -1:
            self.redirect("/")
        else:
            self.redirect(escape.url_unescape(next_page))

    def parse_request(self, request):
        self._log.debug("Path: %s", request.path)
        self._log.debug("Query: %s", request.query)
        # If a request object exists then it contains the method type
        self._command_type = request.method

        # Parse the command name
        self.parse_path(request.path)

        # If a request object exists then it should contain useful trace information
        # Check for the remote IP
        if request.remote_ip:
            self._trace[CommandTrace.origin_address] = request.remote_ip
        # Check for the username of the client
        if 'User' in request.headers:
            self._trace[CommandTrace.user] = request.headers['User']
        # Check for the creation time of the request
        if 'Creation-Time' in request.headers:
            self._trace[CommandTrace.creation_time] = request.headers['Creation-Time']
        # Check for the user agent (client application)
        if 'User-Agent' in request.headers:
            self._trace[CommandTrace.origin_type] = request.headers['User-Agent']

        self._log.debug("Parsed request [%s], trace: %s", self._command_type, self._trace)

        # Parse any parameters
        self.parse_parameters(request.query)

        # Check request body to see if we can parse it
        if request.body:
            try:
                self._parameters.update(json.loads(request.body))
            except:
                #self.parse_parameters(str(request.body.encode('ascii')))
                self.parse_parameters(str(escape.url_unescape(request.body))) #.decode("utf-8")))

    def post(self, pathname):
        rhs = self.get_argument('rhs', default=None)
        vtype = self.get_argument('type', default=None)
        if (rhs and vtype):
            obj, dot, var = escape.url_unescape(pathname).partition('.')
            if vtype == 'str':
                command = '%s.set(%r, %r)' % (obj, var, rhs)
            else:
                command = '%s.set(%r, %s)' % (obj, var, rhs)

            result = ''
            try:
                cserver = self.get_server()
                result = cserver.onecmd(command)
            except Exception as exc:
                print >>sys.stderr, "VariableHandler: Error issuing command %s: %s" \
                                    % (command, str(exc) or repr(exc))
                result += str(sys.exc_info())
            if result:
                result += '\n'
                self.content_type = 'text/html'
                self.write(result)

        else:
            self.send_error(400)  # bad request

    def initialize(self):
        super(RequestHandler, self).initialize()

        self._force_rollback = False
        self._session = None
        self._force_redirect = None
        flash_cookie = self.get_cookie('flash')
        if flash_cookie:
            self.flash = Flash.load(url_unescape(flash_cookie))
        else:
            self.flash = Flash()

        host = self.request.host.lower()
        self.production = (host == 'graffspotting.com' or host.endswith('.graffspotting.com'))

        mobile = None
        mobile_cookie = self.get_cookie('m', None)
        if mobile_cookie is None:
            if detect_mobile(self.request.headers.get('User-Agent')):
                self.set_cookie('m', '1')
                mobile = True
                if host == 'graffspotting.com':
                    self._force_redirect = 'm.graffspotting.com'
                    return
            else:
                mobile = False
                self.set_cookie('m', '0')
        elif host == 'm.graffspotting.com' and mobile_cookie == '0':
            self._force_redirect = 'graffspotting.com'
            return
        elif host == 'graffspotting.com' and mobile_cookie == '1':
            self._force_redirect = 'm.graffspotting.com'
            return

        if mobile is None:
            mobile = mobile_cookie == '1'

        user_id = self.get_secure_cookie('s')
        if user_id:
            user_id, = struct.unpack('<I', user_id)
            self.user = db.User.by_id(self.session, user_id)
        else:
            self.user = None

        # ensure the user has a unique visitor cookie
        if self.get_secure_cookie('v', None) is None:
            self.set_secure_cookie('v', os.urandom(10))

        self.env = {
            'config': config,
            'debug': self.settings['debug'],
            'esc': url_escape,
            'flash': self.flash,
            'gmaps_api_key': config.get('gmaps_api_key', 'AIzaSyCTd_7j6ZeXATLOfTvpAqaqCkxM0zFP5Oc'),
            'is_error': False,
            'mobile': mobile,
            'today': datetime.date.today(),
            'user': self.user
            }

 def get(self):
     wechat_uuid = None
     cli = AsyncHTTPClient()
     uri = url_unescape(self.get_argument('redirect_uri', None))
     with db_session:
         user = User.get(uuid=self.current_user)
         wechat_uuid = user.wechat_uuid
     yield cli.fetch(uri, self.callback)

def parse(data):
    """Parse url-encoded data from jQuery.param()"""
    ret = {}
    for part in data.split('&'):
        if part:
            key, value = url_unescape(part).split('=',1)
            value = get_type(value)
            parse_one(key, value, ret)
    return ret

 def get(self, *args):
     realm = url_unescape(self.request.query)
     if self.request.header["PHP_AUTH_USER"]:
         self.set_status(401)
         self.set_header("WWW-Authenticate", "basic realm=\"%s\"" % realm)
         self.write(
             "If you are seeing this text, authentication did not work.")
     else:
         self.write("If you are seeing this text, authentication worked.")

	def get_link(self, link):
		try:
			data 	= yield http_client(link, c_try=5, c_delay=self.delay)
			data 	= escape.url_unescape(data.split('&proxy.link=',1)[1].split('&',1)[0].strip())
			print(link, data)
			return data
		except Exception as e:
			traceback.print_exc(file=sys.stdout)
			return None

    def post(self):
        global data, columns_name
        # data = read_csv('data.csv')
        val = dict(map(lambda x : x.split('='), self.get_argument('val').split('&')))
        logging.info(val)
        # print(data)
        if val['action'] == 'change_col':
            j = int(val['j'])
            columns_name[j] = url_unescape(val['ch'])
            # old_name = data.columns[j]
            # new_name = url_unescape(val['ch'])
            # data.rename(columns={old_name:new_name}, inplace=True)

        if val['action'] == 'change':
            i = int(val['i'])
            j = int(val['j'])
            data.iloc[i,j] = url_unescape(val['ch'])

        if val['action'] == 'new_column':
            columns_name.append('New column')
            data[str(len(data.columns))] = ''

        if val['action'] == 'del_column':
            j = int(val['j'])
            del columns_name[j]
            data = data.drop(data.columns[j], axis=1)

        if val['action'] == 'new_row':
            data.loc[len(data)] = ['']*len(data.columns)         

        if val['action'] == 'del_row':
            i = int(val['i'])
            data.drop(i, inplace=True,)
            data.reset_index(drop=True) 

        if val['action'] == 'save_csv':
            data2 = data.copy()
            data2.columns = columns_name
            data2.to_csv('data.csv',index=False)

        if val['action'] == 'reload_csv':
            read_data()

        self.finish("{}")

    def get(self, secure, netloc, url):
        proto = 'http' + secure
        netloc = url_unescape(netloc)

        if '/?' in url:
            url, query = url.rsplit('/?', 1)
        else:
            query = None

        remote_url = u"{}://{}/{}".format(proto, netloc, quote(url))

        if query:
            remote_url = remote_url + '?' + query
        if not url.endswith('.ipynb'):
            # this is how we handle relative links (files/ URLs) in notebooks
            # if it's not a .ipynb URL and it is a link from a notebook,
            # redirect to the original URL rather than trying to render it as a notebook
            refer_url = self.request.headers.get('Referer', '').split('://')[-1]
            if refer_url.startswith(self.request.host + '/url'):
                self.redirect(remote_url)
                return

        parse_result = urlparse(remote_url)

        robots_url = parse_result.scheme + "://" + parse_result.netloc + "/robots.txt"

        public = False # Assume non-public

        try:
            robots_response = yield self.fetch(robots_url)
            robotstxt = response_text(robots_response)
            rfp = robotparser.RobotFileParser()
            rfp.set_url(robots_url)
            rfp.parse(robotstxt.splitlines())
            public = rfp.can_fetch('*', remote_url)
        except httpclient.HTTPError as e:
            app_log.debug("Robots.txt not available for {}".format(remote_url),
                    exc_info=True)
            public = True
        except Exception as e:
            app_log.error(e)


        response = yield self.fetch(remote_url)

        try:
            nbjson = response_text(response, encoding='utf-8')
        except UnicodeDecodeError:
            app_log.error("Notebook is not utf8: %s", remote_url, exc_info=True)
            raise web.HTTPError(400)

        yield self.finish_notebook(nbjson, download_url=remote_url,
                                   msg="file from url: %s" % remote_url,
                                   public=public,
                                   request=self.request,
                                   format=self.format)

def get_previous_queries(self):
    previous_queries = self.get_cookie("queries", None)
    if previous_queries:
        try:
            previous_queries = json.loads(url_unescape(previous_queries))
        except Exception:
            previous_queries = []
    else:
        previous_queries = []
    return previous_queries