def get(self):
     request_client = HTTPRequest(url = self.request.uri, 
                                  method = self.request.method,
                                  body = self.request.body or None,
                                  headers = self.request.headers)
     http_client = CurlAsyncHTTPClient()
     response = http_client.fetch(request_client, self.response_client)

def get_media_content(local_config, filename, media_type, callback):
    scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
    
    logging.debug('downloading file %(filename)s of remote camera %(id)s on %(url)s' % {
            'filename': filename,
            'id': camera_id,
            'url': pretty_camera_url(local_config)})
    
    path += '/%(media_type)s/%(id)s/download/%(filename)s' % {
            'media_type': media_type,
            'id': camera_id,
            'filename': filename}
    
    # timeout here is 10 times larger than usual - we expect a big delay when fetching the media list
    request = _make_request(scheme, host, port, username, password,
            path, timeout=10 * settings.REMOTE_REQUEST_TIMEOUT)
    
    def on_response(response):
        if response.error:
            logging.error('failed to download file %(filename)s of remote camera %(id)s on %(url)s: %(msg)s' % {
                    'filename': filename,
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'msg': utils.pretty_http_error(response)})
            
            return callback(error=utils.pretty_http_error(response))
        
        return callback(response.body)

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

class Scour(object):
    "The base class of the scraper"
    
    def __init__(self, seed_urls=[], url_fetch_timeout=25):
        
        ioloop.IOLoop.instance() #instanceiate IOLoop
        self.client = CurlAsyncHTTPClient()
        
        logging.basicConfig(level=logging.DEBUG, filename="m_scrape.log", filemode='w')
        self.log = logging.getLogger(name="mod_scrape")
        
        self.__extractors = []
        
        #Queue Argumet urls
        for url in seed_urls:
            self.get(url)
            
    def get(self, url):
        self.client.fetch(str(url), self.process_page)
            
    def process_page(self, response):
        print "processing page"
        try:
            soup = bSoup(response.body)
            for x in self.__extractors:             #Run the extractors
                x(self, soup, response)
        except Exception, e:                           #Log any errors
            import traceback
            self.log.error(str(traceback.format_exc(limit=10)))
            self.log.error(str(e))

def del_media_group(local_config, group, media_type, callback):
    scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
    
    logging.debug('deleting group "%(group)s" of remote camera %(id)s on %(url)s' % {
            'group': group or 'ungrouped',
            'id': camera_id,
            'url': pretty_camera_url(local_config)})
    
    path += '/%(media_type)s/%(id)s/delete_all/%(group)s/' % {
            'media_type': media_type,
            'id': camera_id,
            'group': group}

    request = _make_request(scheme, host, port, username, password, path,
            method='POST', data='{}',
            timeout=settings.REMOTE_REQUEST_TIMEOUT, content_type='application/json')

    def on_response(response):
        if response.error:
            logging.error('failed to delete group "%(group)s" of remote camera %(id)s on %(url)s: %(msg)s' % {
                    'group': group or 'ungrouped',
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'msg': utils.pretty_http_error(response)})
            
            return callback(error=utils.pretty_http_error(response))
        
        callback()

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

def get_timelapse_movie(local_config, key, group, callback):
    scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
    
    logging.debug('downloading timelapse movie for remote camera %(id)s on %(url)s' % {
            'id': camera_id,
            'url': pretty_camera_url(local_config)})
    
    request = _make_request(scheme, host, port, username, password,
            path + '/picture/%(id)s/timelapse/%(group)s/?key=%(key)s' % {
                'id': camera_id,
                'group': group,
                'key': key},
            timeout=10 * settings.REMOTE_REQUEST_TIMEOUT)

    def on_response(response):
        if response.error:
            logging.error('failed to download timelapse movie for remote camera %(id)s on %(url)s: %(msg)s' % {
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'msg': utils.pretty_http_error(response)})

            return callback(error=utils.pretty_http_error(response))

        callback({
            'data': response.body,
            'content_type': response.headers.get('Content-Type'),
            'content_disposition': response.headers.get('Content-Disposition')
        })

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

 def get_real_ip(self, geolocate=True):
     try:
         self.real_ip = self.request.headers.get(
             'X-Real-Ip',
             self.request.headers.get('X-Forwarded-For', None))
         logging.info(
             "Request from " + str(self.real_ip) + str(self.__class__))
         if geolocate:
             geo_key = "geo_%s" % self.real_ip
             cached_geo = memcache_get(geo_key)
             if cached_geo:
                 logging.info(cached_geo)
             else:
                 def handle_request(responses):
                     geo = load_json(responses.body)
                     memcache_set(geo_key, geo)
                     logging.info(geo)
                 http_client = CurlAsyncHTTPClient()
                 # need to make this a external configuration
                 http_client.fetch("http://freegeoip.net/json/%s" %
                                   self.real_ip,
                                   callback=handle_request,
                                   request_timeout=2,
                                   connect_timeout=2)
     except Exception as e:
         self.error(e)

def check_timelapse_movie(local_config, group, callback):
    scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
    
    logging.debug('checking timelapse movie status for remote camera %(id)s on %(url)s' % {
            'id': camera_id,
            'url': pretty_camera_url(local_config)})
    
    request = _make_request(scheme, host, port, username, password,
            path + '/picture/%(id)s/timelapse/%(group)s/?check=true' % {
                    'id': camera_id,
                    'group': group})
    
    def on_response(response):
        if response.error:
            logging.error('failed to check timelapse movie status for remote camera %(id)s on %(url)s: %(msg)s' % {
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'msg': utils.pretty_http_error(response)})

            return callback(error=utils.pretty_http_error(response))
        
        try:
            response = json.loads(response.body)

        except Exception as e:
            logging.error('failed to decode json answer from %(url)s: %(msg)s' % {
                    'url': pretty_camera_url(local_config),
                    'msg': unicode(e)})

            return callback(error=unicode(e))
        
        callback(response)

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

    def fetch(self, url, **kwargs):
        # init HTTPRequest
        http_client = CurlAsyncHTTPClient()
        session = HTTPRequest('', follow_redirects=False)
        instance_parameters = copy.deepcopy(self._req_params) # 参数

        self.init_request(session, url, **kwargs)

        while True:

            self.pre_request(session, url=url, **kwargs)
            try:
                fetch_logger.info('{method} {url}'.format(method=session.method, url=session.url))
                response = yield http_client.fetch(session)
                fetch_logger.log_green('{code} {url}'.format(code=response.code, url=session.url))
                break
            except HTTPError as httperr:
                # redirects handler
                if httperr.code > 300 and httperr.code < 400:
                    fetch_logger.warning('{code} {url}'.format(code=httperr.code, url=session.url))
                    self.post_request(session, httperr.response, url, **kwargs)
                    if not kwargs.get('disabled_redirect'):
                        url = httperr.response.headers.get('Location')
                    else:
                        fetch_logger.debug(httperr)
                        raise gen.Return(httperr.response)
                else:
                    fetch_logger.error(httperr)
                    raise httperr


        del instance_parameters
        self.post_request(session, response, url, **kwargs)

        raise gen.Return(response)

class jarvis_testcase(tornado.testing.AsyncHTTPTestCase):

    def setUp(self):
        super(jarvis_testcase, self).setUp()
        self.http_client = CurlAsyncHTTPClient(self.io_loop)
        self.headers = {'secret': config.config['secret']}

    def get_app(self):
        self.jarvis = kernel.init(config.config)
        return self.jarvis._application

    @tornado.testing.gen_test
    def http_request(self, request, headers = None):
        command = urllib.quote(request, '')
        uri = command.replace(urllib.quote(' '), '/', 2)
        url = '/api/' + uri

        if headers == None:
            headers = self.headers

        request = HTTPRequest(self.get_url(url), headers=headers)
        response = yield self.http_client.fetch(request, raise_error=False)
        raise tornado.gen.Return(json.loads(response.body))

    @tornado.testing.gen_test
    def raw_http_request(self, url, headers = None):
        if headers == None:
            headers = self.headers

        request = HTTPRequest(self.get_url(url), headers=headers)
        response = yield self.http_client.fetch(request, raise_error=False)
        raise tornado.gen.Return(response)

def test(local_config, data, callback):
    scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
    what = data['what']
    logging.debug('testing %(what)s on remote camera %(id)s, on %(url)s' % {
            'what': what,
            'id': camera_id,
            'url': pretty_camera_url(local_config)})

    data = json.dumps(data)

    request = _make_request(scheme, host, port, username, password,
            path + '/config/%(id)s/test/' % {'id': camera_id},
            method='POST', data=data, content_type='application/json')

    def on_response(response):
        if response.error:
            logging.error('failed to test %(what)s on remote camera %(id)s, on %(url)s: %(msg)s' % {
                    'what': what,
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'msg': utils.pretty_http_error(response)})

            return callback(error=utils.pretty_http_error(response))
        
        callback()

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

def set_preview(local_config, controls, callback):
    scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
    
    logging.debug('setting preview for remote camera %(id)s on %(url)s' % {
            'id': camera_id,
            'url': pretty_camera_url(local_config)})
    
    data = json.dumps(controls)
    
    request = _make_request(scheme, host, port, username, password,
            path + '/config/%(id)s/set_preview/' % {'id': camera_id},
            method='POST', data=data, content_type='application/json')

    def on_response(response):
        if response.error:
            logging.error('failed to set preview for remote camera %(id)s on %(url)s: %(msg)s' % {
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'msg': utils.pretty_http_error(response)})
        
            return callback(error=utils.pretty_http_error(response))
        
        callback()

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

def set_config(local_config, ui_config, callback):
    scheme = local_config.get('@scheme', local_config.get('scheme'))
    host = local_config.get('@host', local_config.get('host')) 
    port = local_config.get('@port', local_config.get('port'))
    username = local_config.get('@username', local_config.get('username'))
    password = local_config.get('@password', local_config.get('password'))
    path = local_config.get('@path', local_config.get('path')) or ''
    camera_id = local_config.get('@remote_camera_id', local_config.get('remote_camera_id'))

    logging.debug('setting config for remote camera %(id)s on %(url)s' % {
            'id': camera_id,
            'url': pretty_camera_url(local_config)})
    
    ui_config = json.dumps(ui_config)
    
    request = _make_request(scheme, host, port, username, password,
            path + '/config/%(id)s/set/' % {'id': camera_id},
            method='POST', data=ui_config, content_type='application/json')
    
    def on_response(response):
        if response.error:
            logging.error('failed to set config for remote camera %(id)s on %(url)s: %(msg)s' % {
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'msg': utils.pretty_http_error(response)})
            
            return callback(error=utils.pretty_http_error(response))
    
        callback()

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

def exec_action(local_config, action, callback):
    scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
    
    logging.debug('executing action "%(action)s" of remote camera %(id)s on %(url)s' % {
            'action': action,
            'id': camera_id,
            'url': pretty_camera_url(local_config)})

    path += '/action/%(id)s/%(action)s/' % {
            'action': action,
            'id': camera_id}

    request = _make_request(scheme, host, port, username, password, path,
            method='POST', data='{}',
            timeout=settings.REMOTE_REQUEST_TIMEOUT, content_type='application/json')

    def on_response(response):
        if response.error:
            logging.error('failed to execute action "%(action)s" of remote camera %(id)s on %(url)s: %(msg)s' % {
                    'action': action,
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'msg': utils.pretty_http_error(response)})

            return callback(error=utils.pretty_http_error(response))
        
        callback()

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

class CurlHTTPClientTestCase(AsyncHTTPTestCase):
	def setUp(self):
		super(CurlHTTPClientTestCase, self).setUp()
		self.http_client = CurlAsyncHTTPClient(self.io_loop)

	def get_app(self):
		return Application([
			('/digest', DigestAuthHandler),
		])

	def test_prepare_curl_callback_stack_context(self):
		exc_info = []

		def error_handler(typ, value, tb):
			exc_info.append((typ, value, tb))
			self.stop()
			return True

		with ExceptionStackContext(error_handler):
			request = HTTPRequest(self.get_url('/'),
								  prepare_curl_callback=lambda curl: 1 / 0)
		self.http_client.fetch(request, callback=self.stop)
		self.wait()
		self.assertEqual(1, len(exc_info))
		self.assertIs(exc_info[0][0], ZeroDivisionError)

	def test_digest_auth(self):
		response = self.fetch('/digest', auth_mode='digest',
							  auth_username='foo', auth_password='bar')
		self.assertEqual(response.body, b'ok')

 def get(self):
     request_client = HTTPRequest(url = self.request.uri, 
                                 method = self.request.method, 
                                 body = self.request.body or None,
                                 headers = self.request.headers, 
                                 proxy_host = options.host_proxy, 
                                 proxy_port = options.port_proxy)
     http_client = CurlAsyncHTTPClient()
     response = http_client.fetch(request_client, self.response_client)

def make_timelapse_movie(local_config, framerate, interval, group, callback):
    scheme, host, port, username, password, path, camera_id = _remote_params(local_config)

    logging.debug('making timelapse movie for group "%(group)s" of remote camera %(id)s with rate %(framerate)s/%(int)s on %(url)s' % {
            'group': group or 'ungrouped',
            'id': camera_id,
            'framerate': framerate,
            'int': interval,
            'url': pretty_camera_url(local_config)})

    path += '/picture/%(id)s/timelapse/%(group)s/?interval=%(int)s&framerate=%(framerate)s' % {
            'id': camera_id,
            'int': interval,
            'framerate': framerate,
            'group': group}
    
    request = _make_request(scheme, host, port, username, password,
            path, timeout=100 * settings.REMOTE_REQUEST_TIMEOUT)

    def on_response(response):
        if response.error:
            logging.error('failed to make timelapse movie for group "%(group)s" of remote camera %(id)s with rate %(framerate)s/%(int)s on %(url)s: %(msg)s' % {
                    'group': group or 'ungrouped',
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'int': interval,
                    'framerate': framerate,
                    'msg': utils.pretty_http_error(response)})

            return callback(error=utils.pretty_http_error(response))
        
        try:
            response = json.loads(response.body)

        except Exception as e:
            logging.error('failed to decode json answer from %(url)s: %(msg)s' % {
                    'url': pretty_camera_url(local_config),
                    'msg': unicode(e)})

            return callback(error=unicode(e))
        
        callback(response)

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

def list(local_config, callback):
    scheme, host, port, username, password, path, _ = _remote_params(local_config)
    
    logging.debug('listing remote cameras on %(url)s' % {
            'url': pretty_camera_url(local_config, camera=False)})
    
    request = _make_request(scheme, host, port, username, password,
            path + '/config/list/')
    
    def on_response(response):
        def make_camera_response(c):
            return {
                'id': c['id'],
                'name': c['name']
            }
        
        if response.error:
            logging.error('failed to list remote cameras on %(url)s: %(msg)s' % {
                    'url': pretty_camera_url(local_config, camera=False),
                    'msg': utils.pretty_http_error(response)})
            
            return callback(error=utils.pretty_http_error(response))
        
        try:
            response = json.loads(response.body)
            
        except Exception as e:
            logging.error('failed to decode json answer from %(url)s: %(msg)s' % {
                    'url': pretty_camera_url(local_config, camera=False),
                    'msg': unicode(e)})
            
            return callback(error=unicode(e))
        
        cameras = response['cameras']
        
        # filter out simple mjpeg cameras
        cameras = [make_camera_response(c) for c in cameras
                if c['proto'] != 'mjpeg' and c.get('enabled')]
        
        callback(cameras)
    
    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

def make_zipped_content(local_config, media_type, group, callback):
    scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
    
    logging.debug('preparing zip file for group "%(group)s" of remote camera %(id)s on %(url)s' % {
            'group': group or 'ungrouped',
            'id': camera_id,
            'url': pretty_camera_url(local_config)})

    prepare_path = path + '/%(media_type)s/%(id)s/zipped/%(group)s/' % {
            'media_type': media_type,
            'id': camera_id,
            'group': group}
 
    # timeout here is 100 times larger than usual - we expect a big delay
    request = _make_request(scheme, host, port, username, password,
            prepare_path, timeout=100 * settings.REMOTE_REQUEST_TIMEOUT)

    def on_response(response):
        if response.error:
            logging.error('failed to prepare zip file for group "%(group)s" of remote camera %(id)s on %(url)s: %(msg)s' % {
                    'group': group or 'ungrouped',
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'msg': utils.pretty_http_error(response)})

            return callback(error=utils.pretty_http_error(response))
        
        try:
            key = json.loads(response.body)['key']

        except Exception as e:
            logging.error('failed to decode json answer from %(url)s: %(msg)s' % {
                    'url': pretty_camera_url(local_config),
                    'msg': unicode(e)})

            return callback(error=unicode(e))

        callback({'key': key})

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

def main():
    parse_command_line()
    app = Application([('/', ChunkHandler)])
    app.listen(options.port, address='127.0.0.1')
    def callback(response):
        response.rethrow()
        assert len(response.body) == (options.num_chunks * options.chunk_size)
        logging.warning("fetch completed in %s seconds", response.request_time)
        IOLoop.instance().stop()

    logging.warning("Starting fetch with curl client")
    curl_client = CurlAsyncHTTPClient()
    curl_client.fetch('http://localhost:%d/' % options.port,
                      callback=callback)
    IOLoop.instance().start()

    logging.warning("Starting fetch with simple client")
    simple_client = SimpleAsyncHTTPClient()
    simple_client.fetch('http://localhost:%d/' % options.port,
                        callback=callback)
    IOLoop.instance().start()

def main_curl():
    # parse_command_line()
    # app = Application([('/', ChunkHandler)])
    # app.listen(options.port, address='127.0.0.1')
    def callback(response):
        # response.rethrow()
        # assert len(response.body) == (options.num_chunks * options.chunk_size)
        logging.warning("fetch completed in %s seconds", response.request_time)
        logging.warning("response status: %s", response.code)
        IOLoop.instance().stop()

    url = 'http://localhost:%d/chunked' % options.port
    logging.warning("Starting fetch with curl client from %s", url)
    curl_client = CurlAsyncHTTPClient()
    curl_client.fetch(request=url,
                      method='POST',
                      callback=callback,
                      body='testdatatestdata',
                      headers={'Transfer-Encoding': 'chunked',
                               'Expect': '100-continue'})
    IOLoop.instance().start()