def configure(path, uid=None):
    """Configures the tornado logging streams with application specific
    customizatons, including configuring the application to log to
    the specified directory.

    Throws:
        OSError -- if the given directory doesn't exist and cannot be
                   created, or if it exists but cannot be written to

    Args:
        path -- a directory to create and write logs to

    Keyword Args:
        uid -- If provided, the uid that should own the current, non-rotated
               version of each log is owned by the given system user.
               This is useful if we plan on dropping down to a less
               privilaged user on application run
    """
    # First, create the specified logging directory if it does not already
    # exist.  If we don't have permissions to create the directory,
    # then OSError will be thrown
    if not os.path.isdir(path):
        os.mkdir(path)

    # Next, make sure that the current process has the needed permissions
    # to write to the specified logging directory. If not, throw an
    # exception, to prevent log-less execution
    if not os.access(path, os.W_OK | os.X_OK):
        error = "Unable to write to logging directory {0}".format("path")
        raise OSError(error)

    # Otherwise, if we're sure we can write to the specified logging
    # directory, configure the built in tornado loggers to use that
    # directory instead of the system wide one
    format = "%(created)f|%(message)s"

    tornado_logs = (('access.log', access_log), ('application.log', app_log),
                    ('general.log', gen_log))

    for log_name, logger in tornado_logs:
        log_path = os.path.join(path, log_name)
        handler = TimedRotatingFileHandler(log_path, when="midnight")
        formatter = logging.Formatter(format)
        handler.setFormatter(formatter)
        logger.addHandler(handler)

        # Allow application errors to propogate up, so that serious errors
        # can wind up on STDERR or other useful places
        if logger is not app_log:
            logger.propagate = False

        if uid:
            os.chown(log_path, uid, -1)

    tornado.log.enable_pretty_logging()

    # Finally, write a simple start up message, both to test that we're
    # able to write as expected, and to get a start time in the logs
    gen_log.setLevel(logging.INFO)
    gen_log.info("Starting webserver (pid:{0}).".format(os.getpid()))

    def post(self):
        msg = 'success'
        save_flag = self.get_argument('flag', '').lower()
        cp = self.get_argument('cp', '').strip()
        cpid = self.get_argument('cpid', '').strip()

        if 'deep' == save_flag:
            poiid = self.get_argument('poiid', '').strip()
            deep = self.get_argument('deep', '')
            if not poiid or 'null' == poiid.lower():
                deep_queue.put((cp, poiid, cpid, deep, -3, -3))
            deep_queue.put((cp, poiid, cpid, deep, 1, 1))
            logger.info("queue.size:%s", len(deep_queue))
            self.do_flush(deep_sql, deep_queue)

        elif 'rti' == save_flag:
            rti = self.get_argument('rti', '')
            value = (cp, cpid, rti, flag_value, flag_value)
            rti_queue.put(value)
            self.do_flush(rti_sql, rti_queue)

        elif 'newpoi' == save_flag:
            new_poi = self.get_argument('newpoiid', '')
            value = (cp, cpid, new_poi)
            newpoi_queue.put(value)
            self.do_flush(newpoi_sql, newpoi_queue)

        else:
            msg = {'failure': MESSAGE.BAD_FLAG}
            logger.info("cp=%s, cpid=%s, msg=%s\nReuqest=%s",
                        cp, cpid, MESSAGE.BAD_FLAG, self.request.arguments)
        self.write(msg)

 def callback():
     try:
         while True:
             gen_log.info(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
             time.sleep(1)
     except:
         pass

    def get_authenticated_user(self, callback, http_client=None):
        """Gets the OAuth authorized user and access token on callback.

        This method should be called from the handler for your registered
        OAuth Callback URL to complete the registration process. We call
        callback with the authenticated user, which in addition to standard
        attributes like 'name' includes the 'access_key' attribute, which
        contains the OAuth access you can use to make authorized requests
        to this service on behalf of the user.

        """
        request_key = escape.utf8(self.get_argument("oauth_token"))
        oauth_verifier = self.get_argument("oauth_verifier", None)
        request_cookie = self.get_cookie("_oauth_request_token")
        if not request_cookie:
            gen_log.warning("Missing OAuth request token cookie")
            callback(None)
            return
        self.clear_cookie("_oauth_request_token")
        cookie_key, cookie_secret = [base64.b64decode(escape.utf8(i)) for i in request_cookie.split("|")]
        if cookie_key != request_key:
            gen_log.info((cookie_key, request_key, request_cookie))
            gen_log.warning("Request token does not match cookie")
            callback(None)
            return
        token = dict(key=cookie_key, secret=cookie_secret)
        if oauth_verifier:
            token["verifier"] = oauth_verifier
        if http_client is None:
            http_client = self.get_auth_http_client()
        http_client.fetch(self._oauth_access_token_url(token),
                          self.async_callback(self._on_access_token, callback))

def start_notebook(url, port, user):
    hub_url = 'https://%s:%s/hub' % (url, port)
    user_url = 'https://%s:%s/user/%s' % (url, port, user)

    cookies = login(hub_url, user, user)
    api = NBAPI(url=user_url, cookies=cookies)

    path = 'Hello.ipynb'
    for i in itertools.count():
        gen_log.info("loading %s (%s)", user, i)
        nb = api.get_notebook(path)

        gen_log.info("starting %s (%s)", user, i)
        session = Session()
        kernel = yield api.new_kernel(session.session)

        try:
            for j in range(20):
                gen_log.info("running %s (%s:%s)", user, j, i)
                yield run_notebook(nb, kernel, session)
                yield sleep(0.05)

            gen_log.info("saving %s (%s)", user, i)
            api.save_notebook(nb, path)

        finally:
            api.kill_kernel(kernel['id'])


    gen_log.info("history: %s", response.history)

def run_notebook(nb, kernel, session):
    """Run all the cells of a notebook"""
    ncells = sum(cell['cell_type'] == 'code' for cell in nb.cells)
    i = 0
    for cell in nb.cells:
        if cell['cell_type'] == 'code':
            i += 1
            gen_log.info("Executing cell %i/%i", i, ncells)
            yield execute(cell, kernel, session)

def saveFile(files, key, path):
    fl = files[key][0]
    req_name = fl["filename"]
    body = fl["body"]
    timestamp = int(time.time() + 300)
    fileName = "%d_%s.%s" % (timestamp, str(uuid.uuid1()), req_name.split(".").pop())
    gen_log.info(fileName)
    with open(path + fileName, "w") as f:
        f.write(body)
    return fileName

 def api(self, path, **kwargs):
     try:
         import time
         s_time = time.time()
         data = yield self._make_request(path, **kwargs)
         e_time = time.time()
         gen_log.info("=====Time request wio api, {}".format(float(e_time)-float(s_time)))
     except Exception as e:
         gen_log.error(e)
         raise
     raise gen.Return(data)

def _check_file(modify_times, path):
    try:
        modified = os.stat(path).st_mtime
    except Exception:
        return
    if path not in modify_times:
        modify_times[path] = modified
        return
    if modify_times[path] != modified:
        gen_log.info("%s modified; restarting server", path)
        _reload()

    def add(self, req):
        if self.get_style(req):
            req.reply_error("The style already exist")
            gen_log.info('Style %s already exists' % req.content['name'])
            return

        style = create_style(req.content['name'])
        req.client.styles.append(style)
        req.content['style'] = style

        req.send_to_all(style)
        return True

    def _make_request(self, path, query=None, method="GET", body=None, headers=None):
        """
        Makes request on `path` in the graph.

        path -- endpoint to the facebook graph api
        query -- A dictionary that becomes a query string to be appended to the path
        method -- GET, POST, etc
        body -- message body
        headers -- Like "Content-Type"
        """
        if not query:
            query = {}

        if self.access_token:
            query["access_token"] = self.access_token

        query_string = urllib.urlencode(query) if query else ""
        if method == "GET":
            body = None
        else:
            if headers and "json" in headers.get('Content-Type'):
                body = json.dumps(body) if body else ""
            else:
                body = urllib.urlencode(body) if body else ""

        url = BASE_URL + path
        if query_string:
            url += "?" + query_string

        # url = "https://wio.temp-io.life/v1/nodes/create?access_token=123"
        gen_log.info("URL=====> {}".format(url))
        gen_log.info("method=====> {}".format(method))
        gen_log.info("body=====> {}".format(body))

        client = AsyncHTTPClient()
        request = HTTPRequest(url, method=method, body=body, headers=headers)
        try:
            response = yield client.fetch(request)
        except HTTPError as e:
            raise WioAPIError(e)
        except Exception as e:
            gen_log.error(e)
            raise

        content_type = response.headers.get('Content-Type')
        gen_log.info("#### content_type: {}".format(content_type))
        gen_log.info("#### body: {}".format(response.body))
        if 'json' in content_type:
            data = json.loads(response.body.decode())
        else:
            raise WioAPIError('Maintype was not json')

        raise gen.Return(data)

    def _on_headers(self, data):
        self._old_request = None
        self._end_notified = False
        self._please_notify_end_of_request = False
        try:
            data = native_str(data.decode('latin1'))
            eol = data.find("\r\n")
            start_line = data[:eol]
            try:
                method, uri, version = start_line.split(" ")
            except ValueError:
                raise _BadRequestException("Malformed HTTP request line")
            if not version.startswith("HTTP/"):
                raise _BadRequestException("Malformed HTTP version in HTTP Request-Line")
            headers = httputil.HTTPHeaders.parse(data[eol:])

            # HTTPRequest wants an IP, not a full socket address
            if getattr(self.stream.socket, 'family', socket.AF_INET) in (
                socket.AF_INET, socket.AF_INET6):
                # Jython 2.5.2 doesn't have the socket.family attribute,
                # so just assume IP in that case.
                remote_ip = self.address[0]
            else:
                # Unix (or other) socket; fake the remote address
                remote_ip = '0.0.0.0'

            self._request = HTTPRequest(
                connection=self, method=method, uri=uri, version=version,
                headers=headers, remote_ip=remote_ip)

            if self._events["connect"]:
                self._events["connect"](self._request)

            for name,handler in self._events.items():
                if handler:
                    self._request.on(name, handler)

            content_length = headers.get("Content-Length")
            if content_length:
                content_length = int(content_length)
                if content_length > self.stream.max_buffer_size:
                    raise _BadRequestException("Content-Length too long")
                if headers.get("Expect") == "100-continue":
                    self.stream.write(b("HTTP/1.1 100 (Continue)\r\n\r\n"))
                self.stream.read_bytes(content_length, self._on_request_body)
                return

            self.request_callback(self._request)
        except _BadRequestException, e:
            gen_log.info("Malformed HTTP request from %s: %s",
                         self.address[0], e)
            self.close()
            return

def parse_request(data):
    try:
        req = ast.literal_eval(data)
        path, sep, query = req['uri'].partition('?')
        get_arguments = parse_qs_bytes(query, keep_blank_values=True)
        post_arguments = parse_qs_bytes(req['body'], keep_blank_values=True)
        host = req['headers']['Host']
        headers = req['headers']
        return headers, host, path, get_arguments, post_arguments
    except _BadRequestException as e:
        gen_log.info("Malformed HTTP request:%s", e)
        return

  def _on_headers(self, data):
    try:
      data = data.decode('latin1')
      eol = data.find("\r\n")
      start_line = data[:eol]
      try:
        method, uri, version = start_line.split(" ")
      except ValueError:
        raise tornado.httpserver._BadRequestException("Malformed HTTP request line")
      if not version.startswith("HTTP/"):
        raise tornado.httpserver._BadRequestException("Malformed HTTP version in HTTP Request-Line")
      headers = tornado.httputil.HTTPHeaders.parse(data[eol:])

      # HTTPRequest wants an IP, not a full socket address
      if self.address_family in (socket.AF_INET, socket.AF_INET6):
        remote_ip = self.address[0]
      else:
        # Unix (or other) socket; fake the remote address
        remote_ip = '0.0.0.0'

      self._request = tornado.httpserver.HTTPRequest(
        connection=self, method=method, uri=uri, version=version,
        headers=headers, remote_ip=remote_ip, protocol=self.protocol)

      content_length = headers.get("Content-Length")
      if content_length:
        content_length = int(content_length)
        use_tmp_files = self._get_handler_info()
        if not use_tmp_files and content_length > self.stream.max_buffer_size:
          raise _BadRequestException("Content-Length too long")
        if headers.get("Expect") == "100-continue":
          self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
        if use_tmp_files:
          gen_log.debug('using temporary files for uploading')

          # avoid raising
          # IOError("Reached maximum read buffer size")
          # in tornado.iostream.BaseIOStream._read_to_buffer
          self.stream.max_buffer_size = maxint

          self._receive_content(content_length)
        else:
          gen_log.debug('using memory for uploading')
          self.stream.read_bytes(content_length, self._on_request_body)
        return

      self.request_callback(self._request)
    except tornado.httpserver._BadRequestException as e:
      gen_log.info("Malformed HTTP request from %s: %s",
             self.address[0], e)
      self.close()
      return

	def __init__(self) : 

		gen_log.info ("\n/// GenericSpiderMix / init ")

		# Default fields for mixin class
		
		# self.name = ""  # The name of the spider to use when executing the spider

		self.error_array = []
		self.item_count = 0  # will be incremented each time a new item is created
		self.item_count_depth_1 = 0  # will be incremented each time an item is completed in detailed page
		self.LIMIT = 5  # The number of pages where the spider will stop
		self.page_count = 1  # The number of pages already scraped
		self.download_delay = 0  # The delay in seconds between each request. some website will block too many requests

	def get(self,*args,**kwargs):
		pageIndex=int(self.get_argument('pageIndex',1))
		company=self.get_argument('type','wechat')
		querys={'wechat':{'wechat':{'$exists':True}},
				'1xinxi':{'sendid':{'$exists':True}},
				'chanzor':{'sendid':{'$exists':False},'wechat':{'$exists':False}}}

		db = self.application.db
		gen_log.info(company)
		gen_log.info(pageIndex)
		record_list = yield db.sendrecord.find(querys.get(company)).sort([("_id",-1)]).skip((pageIndex-1)*20).limit(20).to_list(length=None)
		self.set_header('content-type','application/json')
		self.write(json_encode({"data":record_list}))
		pass

def open_run_save(api, path, legacy=False):
    """open a notebook, run it, and save.
    
    Only the original notebook is saved, the output is not recorded.
    """
    nb = api.get_notebook(path)
    session = Session()
    kernel = yield api.new_kernel(session.session, legacy=legacy)
    try:
        yield run_notebook(nb, kernel, session)
    finally:
        api.kill_kernel(kernel['id'])
    gen_log.info("Saving %s/notebooks/%s", api.url, path)
    api.save_notebook(nb, path)

    def _on_headers(self, data):
        try:
            data = native_str(data.decode('latin1'))
            eol = data.find("\r\n")
            start_line = data[:eol]
            try:
                method, uri, version = start_line.split(" ")
            except ValueError:
                raise _BadRequestException("Malformed HTTP request line")
            if not version.startswith("HTTP/"):
                raise _BadRequestException("Malformed HTTP version in HTTP Request-Line")
            try:
                headers = httputil.HTTPHeaders.parse(data[eol:])
            except ValueError:
                # Probably from split() if there was no ':' in the line
                raise _BadRequestException("Malformed HTTP headers")

            # HTTPRequest wants an IP, not a full socket address
            if self.address_family in (socket.AF_INET, socket.AF_INET6):
                remote_ip = self.address[0]
            else:
                # Unix (or other) socket; fake the remote address
                remote_ip = '0.0.0.0'


            # 构造一个httpRequest对象
            self._request = HTTPRequest(
                connection=self, method=method, uri=uri, version=version,
                headers=headers, remote_ip=remote_ip, protocol=self.protocol)

            # 如果头部带有content-length就继续解包，然后回调我们request_body函数
            # 用回调的方式，估计也是因为多路复用，导致非阻塞的情况
            content_length = headers.get("Content-Length")
            if content_length:
                content_length = int(content_length)
                if content_length > self.stream.max_buffer_size:
                    raise _BadRequestException("Content-Length too long")
                if headers.get("Expect") == "100-continue":
                    self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
                self.stream.read_bytes(content_length, self._on_request_body) # 原来都是写到内存里面的
                return

            # 如果请求不带content-length，那很简单，直接开始处理的具体逻辑
            self.request_callback(self._request) # 这里是调用app内部的那个 __call__魔术方法了
        except _BadRequestException as e:
            gen_log.info("Malformed HTTP request from %r: %s",
                         self.address, e)
            self.close()
            return

    def _on_headers(self, data):
        try:
            data = native_str(data.decode("latin1"))
            eol = data.find("\r\n")
            start_line = data[:eol]
            try:
                method, uri, version = start_line.split(" ")
            except ValueError:
                raise _BadRequestException("Malformed HTTP request line")
            if not version.startswith("HTTP/"):
                raise _BadRequestException("Malformed HTTP version in HTTP Request-Line")
            try:
                headers = httputil.HTTPHeaders.parse(data[eol:])
            except ValueError:
                # Probably from split() if there was no ':' in the line
                raise _BadRequestException("Malformed HTTP headers")

            # HTTPRequest wants an IP, not a full socket address
            if self.address_family in (socket.AF_INET, socket.AF_INET6):
                remote_ip = self.address[0]
            else:
                # Unix (or other) socket; fake the remote address
                remote_ip = "0.0.0.0"

            self._request = HTTPRequest(
                connection=self,
                method=method,
                uri=uri,
                version=version,
                headers=headers,
                remote_ip=remote_ip,
                protocol=self.protocol,
            )

            content_length = headers.get("Content-Length")
            if content_length:
                content_length = int(content_length)
                if content_length > self.stream.max_buffer_size:
                    raise _BadRequestException("Content-Length too long")
                if headers.get("Expect") == "100-continue":
                    self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
                self.stream.read_bytes(content_length, self._on_request_body)
                return

            self.request_callback(self._request)
        except _BadRequestException as e:
            gen_log.info("Malformed HTTP request from %s: %s", self.address[0], e)
            self.close()
            return

    def _handle_events(self, fd, events):
        if self.closed():
            gen_log.warning("Got events for closed stream %s", fd)
            return
        try:
            if self._connecting:
                self._handle_connect()
            if self.closed():
                return
            if events & self.io_loop.READ:
                # NOTE: We use explict read instead of implicit.
                # The reason IOStream is not idle is that when an event happened,
                # tornado iostream will still try to read them into buffer.
                # Our approach is that when someone is trying to read the iostream,
                # we will read it.
                if self._should_socket_close() or self.reading():
                    self._handle_read()

            if self.closed():
                return
            if events & self.io_loop.WRITE:
                self._handle_write()
            if self.closed():
                return
            if events & self.io_loop.ERROR:
                self.error = self.get_fd_error()
                self.io_loop.add_callback(self.close)
                return
            state = self.io_loop.ERROR
            if self.reading():
                state |= self.io_loop.READ
            if self.writing():
                state |= self.io_loop.WRITE
            if state == self.io_loop.ERROR and self._read_buffer_size == 0:
                state |= self.io_loop.READ
            if state != self._state:
                assert self._state is not None, \
                    "shouldn't happen: _handle_events without self._state"
                self._state = state
                self.io_loop.update_handler(self.fileno(), self._state)
        except UnsatisfiableReadError as e:
            gen_log.info("Unsatisfiable read, closing connection: %s" % e)
            self.close(exc_info=True)
        except Exception:
            gen_log.error("Uncaught exception, closing connection.",
                          exc_info=True)
            self.close(exc_info=True)
            raise