def encode_headers(headers):
    twisted_headers = Headers()
    for k, v in headers.items():
        key = k.encode('ascii', 'ignore')
        val = v.encode('ascii', 'ignore')
        twisted_headers.addRawHeader(key, val)
    return twisted_headers

def http_get(reactor, url, accept=None):
    agent = client.Agent(reactor)
    headers = Headers()
    if accept is not None:
        headers.addRawHeader('Accept', str(accept))
    d = agent.request('GET', url, headers=headers)
    return _handle_agent_response(d)

    def request(self, method, url, **kwargs):
        method = method.upper()

        # Join parameters provided in the URL
        # and the ones passed as argument.
        params = kwargs.get('params')
        if params:
            url = _combine_query_params(url, params)

        # Convert headers dictionary to
        # twisted raw headers format.
        headers = kwargs.get('headers')
        if headers:
            if isinstance(headers, dict):
                h = Headers({})
                for k, v in headers.iteritems():
                    if isinstance(v, str):
                        h.addRawHeader(k, v)
                    else:
                        h.setRawHeaders(k, v)

                headers = h
        else:
            headers = Headers({})

        # Here we choose a right producer
        # based on the parameters passed in.
        bodyProducer = None
        data = kwargs.get('data')
        files = kwargs.get('files')
        if files:
            # If the files keyword is present we will issue a
            # multipart/form-data request as it suits better for cases
            # with files and/or large objects.
            files = list(_convert_files(files))
            boundary = uuid.uuid4()
            headers.setRawHeaders(
                'content-type', [
                    'multipart/form-data; boundary=%s' % (boundary,)])
            if data:
                data = _convert_params(data)
            else:
                data = []

            bodyProducer = multipart.MultiPartProducer(
                data + files, boundary=boundary)
        elif data:
            # Otherwise stick to x-www-form-urlencoded format
            # as it's generally faster for smaller requests.
            if isinstance(data, (dict, list, tuple)):
                headers.setRawHeaders(
                    'content-type', ['application/x-www-form-urlencoded'])
                data = urlencode(data, doseq=True)
            bodyProducer = IBodyProducer(data)

        d = self._agent.request(
            method, url, headers=headers,
            bodyProducer=bodyProducer)

        return d

 def test_initializer(self):
     """
     The header values passed to L{Headers.__init__} can be retrieved via
     L{Headers.getRawHeaders}.
     """
     h = Headers({b"Foo": [b"bar"]})
     self.assertEqual(h.getRawHeaders(b"foo"), [b"bar"])

    def headers(self, rawHeaders):
        # type: (RawHeaders) -> IMutableHTTPHeaders
        headers = Headers()
        for rawName, rawValue in rawHeaders:
            headers.addRawHeader(rawName, rawValue)

        return HTTPHeadersWrappingHeaders(headers=headers)

    def request(self, method, uri, headers=None, bodyProducer=None):
        parsedURI = client._parse(uri)
        
        host_addr = address.IPv4Address('TCP', parsedURI.host, parsedURI.port)
        
        
        # ripped from _AgentBase._requestWithEndpoint
        if headers is None:
            headers = Headers()
        if not headers.hasHeader('host'):
            headers = headers.copy()
            headers.addRawHeader(
                'host', self._computeHostValue(parsedURI.scheme, parsedURI.host,
                                               parsedURI.port))
        request = client.Request(method, parsedURI.path, headers, bodyProducer,
                                 persistent=False)

        c = ClientProtocol(request)
        
        # ouch
        self.root.putChild('', self.root)
        
        server = Site(self.root).buildProtocol(self.addr)
        loopbackAsync(server, c, host_addr, self.addr)
        return c.response.addBoth(self._done, c)

def url_upload_data_async(url, files={}, data={}, progressCallback=None, doneCallback=None):

    def produce_finished(data):
        print 'produce finished',data
    def produce_error(error):
        print 'produce error', error

    producerDeferred = Deferred()
    producerDeferred.addCallback(produce_finished)
    producerDeferred.addErrback(produce_error)

    def receive_finished(data):
        print 'recieve finished', data
        doneCallback(data)

    def receive_error(error):
        Logger.error('Problem uploading file')
        print 'recieve error', error

    receiverDeferred = Deferred()
    receiverDeferred.addCallback(receive_finished)
    receiverDeferred.addErrback(receive_error)

    producer = MultiPartProducer(files, data, progressCallback, producerDeferred)
    receiver = StringReceiver(receiverDeferred)

    agent = Agent(reactor)
    headers = Headers({'User-Agent': ['Twisted Web Client Example']})
    headers.addRawHeader("Content-Type", "multipart/form-data; boundary=%s" % producer.boundary)

    requestDeffered = agent.request('POST', url, headers, producer)
    requestDeffered.addCallback(lambda response: response.deliverBody(receiver))

    def _make_request(self):
        metrics = []
        for x in xrange(HTTP_BATCH):
            if not self._mq:
                break
            metrics.append(self._mq.popleft())
        if not metrics:
            return defer.succeed(None)

        serialized_metrics = json.dumps({"metrics": metrics})
        body_writer = StringProducer(serialized_metrics)

        headers = Headers({
            'User-Agent': ['Zenoss Metric Publisher: %s' % self._agent_suffix],
            'Content-Type': ['application/json']})

        if self._needsAuth and not self._authenticated:
            log.info("Adding auth for metric http post %s", self._url)
            headers.addRawHeader('Authorization',
                                 basic_auth_string_content(self._username, self._password))

        d = self._agent.request(
            'POST', self._url, headers,
            body_writer)

        d.addCallbacks(self._metrics_published, errback=self._publish_failed,
        callbackArgs = [len(metrics), len(self._mq)], errbackArgs = [metrics])
        d.addCallbacks(self._response_finished, errback=self._publish_failed,
                       errbackArgs = [metrics])

        return d

 def test_initializer(self):
     """
     The header values passed to L{Headers.__init__} can be retrieved via
     L{Headers.getRawHeaders}.
     """
     h = Headers({'Foo': ['bar']})
     self.assertEqual(h.getRawHeaders('foo'), ['bar'])

def post(url, data=None, on_created=None, on_error=None):
    errback = on_error or make_errback(frames_back=2)
    try:
        def handle_response(response):
            if response.code == 200:
                callback = partial(
                    _handle_post,
                    on_created=on_created,
                    on_error=errback)
                response.deliverBody(JsonReceiver.create(callback, errback))
            else:
                errback('returned %s' % response.code)

        agent = Agent(reactor)
        headers = Headers(get_auth_headers())
        headers.addRawHeader('Content-Type', 'application/json')
        if not data:
            data = {'dummy': 1}
        d = agent.request(
            'POST',
            add_params(url, rid=uuid4().hex),
            headers=headers,
            bodyProducer=JsonProducer(data) if data else None)
        d.addCallbacks(handle_response, errback)
    except Exception as ex:
        errback('error %s' % ex)

 def obtain_auth_token(self):
     """Return a valid Microsoft Cognitive Services authentication
     token, obtained with the current subscription key if necessary.
     """
     start_time = time.time()
     if self.auth_token is None or self.token_expiry < start_time:
         headers = Headers()
         headers.addRawHeader('Ocp-Apim-Subscription-Key',
                              self.subscription_key)
         headers.addRawHeader('Content-Length', '0')
         response = yield self.agent.request(
             'POST', AUTH_URL, headers=headers)
         if response.code != 200:
             data = yield readBody(response)
             self.log.error(
                 'Could not authenticate to Microsoft Cognitive '
                 'Services: {data}', data=data)
             raise UserVisibleError(
                 'Could not authenticate to Microsoft Cognitive '
                 'Services. Try again later.')
         # Coerce the access token to a byte string to avoid problems
         # inside Twisted's header handling code down the line.
         self.auth_token = (
             (yield readBody(response)).strip().decode('ascii'))
         self.token_expiry = start_time + AUTH_TOKEN_TTL
     returnValue(self.auth_token)

 def request(self, method, uri, headers=None, data=None):
     if data is not None:
         if headers is None:
             headers = Headers()
         headers.removeHeader('Content-Type')
         headers.addRawHeader('Content-Type', 'application/json; charset=utf-8')
     return BasicAgent.request(self, method, uri, headers, data)

    def request(self, url, method='GET', data=None):
        d = defer.Deferred()
        _headers = {}
        headers = None

        if method == 'GET':
            data = None
        elif method == 'POST':
            _headers['Content-Type'] = 'application/x-www-form-urlencoded'
        else:
            d.errback(Exception('Not Support Method:%s.' % method))
            return

        if _headers:
            headers = Headers()
            for k, v in _headers.iteritems():
                headers.setRawHeaders(urllib.quote(k), [urllib.quote(v)])

        if data:
            data = StrBodyProducer(data)
        #print 'Req url:', url, ' Head:', headers
        self.agent.request(
                method,
                url,
                headers,
                data).addCallback(self.received, d).addErrback(
                    self.errback, d)

        return d

def endpoint(endpoint, username='about', apikey=None, data=None):
    global url, user_agent
    def encode_body(data):
        if (data):
            if (isinstance(data, six.string_types)):
                data = json_loads(data)
            return ('POST', FileBodyProducer(six.StringIO(json_dumps(data))))
        else:
            return ('GET', None)
    address = make_endpoint_address(endpoint, username)
    headers = Headers({
        'Accept': ['application/json,*/*;q=0.8'],
        'Accept-Encoding': ['gzip,deflate,sdch'],
        'Connection': ['keep-alive'],
        'User-Agent': [user_agent],
        'Content-Type': ['application/json'],
    })
    if (apikey):
        headers.addRawHeader('Authorization', 'Basic ' + encode_apikey(str(apikey)))
    method, body = encode_body(data)
    agent = Agent(reactor)
    response = yield agent.request(method, address.encode('utf-8'), headers, body)
    if (response.code == 200):
        finished = Deferred()
        collector = WebBodyCollector(finished)
        response.deliverBody(collector)
        x = yield finished
        returnValue(json_loads(collector.body))
    else:
        raise RequestError(response.phrase)

	def _doPush( self, filename ):
		print 'beginning push of file:', filename
		producerDeferred = defer.Deferred()
		producerDeferred.addCallback( self._finished )
		producerDeferred.addErrback( self._error )
		
		multiPartProducer = MultiPartProducer(
			files={
				'metrics': filename,
			},
			data={
				'secret': agent_settings.PUSH_DETAILS['secret'],
			},
			callback=self._progress,
			deferred=producerDeferred
		)
	
		headers = Headers()
		headers.addRawHeader( 'Content-Type', 'multipart/form-data; boundary=%s' % multiPartProducer.boundary )
	
		agent = client.Agent( reactor )
		request = agent.request(
			'POST',
			agent_settings.PUSH_DETAILS['url'],
			headers,
			multiPartProducer
		)
		request.addCallback( self._responseReady, filename )

    def request(self, method, uri, headers=None, bodyProducer=None,
                parameters=None):
        """Make a request, optionally signing it.

        Any query string passed in `uri` will get clobbered by the urlencoded
        version of `parameters`.
        """
        if headers is None:
            headers = Headers()
        if parameters is None:
            parameters = {}
        req = oauth2.Request.from_consumer_and_token(
            self.consumer, token=self.token,
            http_method=method, http_url=uri, parameters=parameters,
            is_form_encoded=True)
        req.sign_request(self.signatureMethod, self.consumer, self.token)
        for header, value in req.to_header().iteritems():
            # oauth2, for some bozotic reason, gives unicode header values
            headers.addRawHeader(header, value.encode())
        parsed = urlparse.urlparse(uri)
        parameters = {k.encode('utf-8'): v.encode('utf-8')
                      for k, v in req.get_nonoauth_parameters().iteritems()}
        uri = urlparse.urlunparse(
            parsed._replace(query=urllib.urlencode(parameters)))
        return self.agent.request(method, uri, headers, bodyProducer)

	def _baseHeaders(self):
		"""return a dictionary of the base headers needed in a peer request"""
		headers = Headers()
		headers.addRawHeader('Target',self.target)
		headers.addRawHeader('Signature',self._signature)

		return headers

 def headers(self):
     if not self.result:
         raise Exception("No result yet.")
     h = Headers()
     for i in self.result["headers"]:
         h.addRawHeader(*i)
     return h

    def request(self, method, uri, headers=None, bodyProducer=None):
        """
        Issue a new request to the wrapped L{Agent}.

        Send a I{Cookie} header if a cookie for C{uri} is stored in
        L{CookieAgent.cookieJar}. Cookies are automatically extracted and
        stored from requests.

        If a C{'cookie'} header appears in C{headers} it will override the
        automatic cookie header obtained from the cookie jar.

        @see: L{Agent.request}
        """
        if headers is None:
            headers = Headers()
        lastRequest = _FakeUrllib2Request(uri)
        # Setting a cookie header explicitly will disable automatic request
        # cookies.
        if not headers.hasHeader('cookie'):
            self.cookieJar.add_cookie_header(lastRequest)
            cookieHeader = lastRequest.get_header('Cookie', None)
            if cookieHeader is not None:
                headers = headers.copy()
                headers.addRawHeader('cookie', cookieHeader)

        d = self._agent.request(method, uri, headers, bodyProducer)
        d.addCallback(self._extractCookies, lastRequest)
        return d

def request(rq):
    headers = Headers(dict((k, [v]) for k, v in rq.headers.iteritems()))
    scheme, host, port, path = _parse(rq.url)
    headers.setRawHeaders('Host', [host])
    return (rq.method, rq.url,
            headers,
            StringProducer(rq.data) if rq.data else None)