def test_stream_chunked(self):

        connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        connection.connect(("127.0.0.1", self.proxy.port))
        fconn = connection.makefile()
        spec = '200:h"Transfer-Encoding"="chunked":r:b"4\\r\\nthis\\r\\n7\\r\\nisatest\\r\\n0\\r\\n\\r\\n"'
        connection.send(
            "GET %s/p/%s HTTP/1.1\r\n" %
            (self.server.urlbase, spec))
        connection.send("\r\n")

        httpversion, code, msg, headers, content = http.read_response(
            fconn, "GET", None, include_body=False)

        assert headers["Transfer-Encoding"][0] == 'chunked'
        assert code == 200

        chunks = list(
            content for _,
            content,
            _ in http.read_http_body_chunked(
                fconn,
                headers,
                None,
                "GET",
                200,
                False))
        assert chunks == ["this", "isatest", ""]

        connection.close()

 def print_requests(self, reqs, respdump, reqdump, fp=sys.stdout):
     """
         Performs a series of requests, and prints results to the specified
         file pointer.
     """
     for i in reqs:
         try:
             r = rparse.parse_request(self.settings, i)
             req = r.serve(self.wfile, None, self.host)
             if reqdump:
                 print >> fp, "\n>>", req["method"], repr(req["path"])
                 for a in req["actions"]:
                     print >> fp, "\t",
                     for x in a:
                         print >> fp, x,
                     print >> fp
             self.wfile.flush()
             resp = http.read_response(self.rfile, r.method, None)
         except rparse.ParseException, v:
             print >> fp, "Error parsing request spec: %s"%v.msg
             print >> fp, v.marked()
             return
         except rparse.FileAccessDenied, v:
             print >> fp, "File access error: %s"%v
             return

    def http(self, r):
        """
            Performs a single request.

            r: A language.Request object, or a string representing one request.

            Returns Response if we have a non-ignored response.

            May raise http.HTTPError, tcp.NetLibError
        """
        if isinstance(r, basestring):
            r = language.parse_requests(r)[0]
        resp, req = None, None
        if self.showreq:
            self.wfile.start_log()
        if self.showresp:
            self.rfile.start_log()
        try:
            req = language.serve(r, self.wfile, self.settings)
            self.wfile.flush()
            resp = list(
                http.read_response(
                    self.rfile,
                    req["method"],
                    None
                )
            )
            resp.append(self.sslinfo)
            resp = Response(*resp)
        except http.HttpError, v:
            if self.showsummary:
                print >> self.fp, "<< HTTP Error:", v.message
            raise

    def from_stream(cls, rfile, request_method, include_body=True, body_size_limit=None):
        """
        Parse an HTTP response from a file stream
        """

        timestamp_start = utils.timestamp()

        if hasattr(rfile, "reset_timestamps"):
            rfile.reset_timestamps()

        httpversion, code, msg, headers, content = http.read_response(
            rfile,
            request_method,
            body_size_limit,
            include_body=include_body)

        if hasattr(rfile, "first_byte_timestamp"):  # more accurate timestamp_start
            timestamp_start = rfile.first_byte_timestamp

        if include_body:
            timestamp_end = utils.timestamp()
        else:
            timestamp_end = None

        return HTTPResponse(
            httpversion,
            code,
            msg,
            headers,
            content,
            timestamp_start,
            timestamp_end
        )

    def handle_request(self, cc):
        try:
            request, err = None, None
            request = self.read_request(cc)
            if request is None:
                return
            cc.requestcount += 1

            app = self.server.apps.get(request)
            if app:
                err = app.serve(request, self.wfile)
                if err:
                    self.log(cc, "Error in wsgi app.", err.split("\n"))
                    return
            else:
                request_reply = self.channel.ask(request)
                if request_reply is None or request_reply == KILL:
                    return
                elif isinstance(request_reply, flow.Response):
                    request = False
                    response = request_reply
                    response_reply = self.channel.ask(response)
                else:
                    request = request_reply
                    if self.config.reverse_proxy:
                        scheme, host, port = self.config.reverse_proxy
                    elif self.config.forward_proxy:
                        scheme, host, port = self.config.forward_proxy
                    else:
                        scheme, host, port = request.scheme, request.host, request.port

                    # If we've already pumped a request over this connection,
                    # it's possible that the server has timed out. If this is
                    # the case, we want to reconnect without sending an error
                    # to the client.
                    while 1:
                        sc = self.get_server_connection(cc, scheme, host, port, self.sni)
                        sc.send(request)
                        if sc.requestcount == 1: # add timestamps only for first request (others are not directly affected)
                            request.tcp_setup_timestamp = sc.tcp_setup_timestamp
                            request.ssl_setup_timestamp = sc.ssl_setup_timestamp
                        sc.rfile.reset_timestamps()
                        try:
                            tsstart = utils.timestamp()
                            httpversion, code, msg, headers, content = http.read_response(
                                sc.rfile,
                                request.method,
                                self.config.body_size_limit
                            )
                        except http.HttpErrorConnClosed, v:
                            self.del_server_connection()
                            if sc.requestcount > 1:
                                continue
                            else:
                                raise
                        except http.HttpError, v:
                            raise ProxyError(502, "Invalid server response.")
                        else:

    def test_simple(self):
        sc = ServerConnection((self.d.IFACE, self.d.port), None)
        sc.connect()
        r = tutils.treq()
        r.flow.server_conn = sc
        r.path = "/p/200:da"
        sc.send(r._assemble())
        assert http.read_response(sc.rfile, r.method, 1000)
        assert self.d.last_log()

        sc.finish()

    def test_simple(self):
        sc = proxy.ServerConnection(proxy.ProxyConfig(), self.d.IFACE, self.d.port)
        sc.connect("http")
        r = tutils.treq()
        r.path = "/p/200:da"
        sc.send(r)
        assert http.read_response(sc.rfile, r.method, 1000)
        assert self.d.last_log()

        r.content = flow.CONTENT_MISSING
        tutils.raises("incomplete request", sc.send, r)

    def test_simple(self):
        sc = ServerConnection((self.d.IFACE, self.d.port))
        sc.connect()
        f = tutils.tflow()
        f.server_conn = sc
        f.request.path = "/p/200:da"
        sc.send(f.request.assemble())
        assert http.read_response(sc.rfile, f.request.method, 1000)
        assert self.d.last_log()

        sc.finish()

    def request(self, spec):
        """
            Return an (httpversion, code, msg, headers, content) tuple.

            May raise rparse.ParseException, netlib.http.HttpError or
            rparse.FileAccessDenied.
        """
        r = rparse.parse_request(self.settings, spec)
        ret = r.serve(self.wfile, None, self.host)
        self.wfile.flush()
        return http.read_response(self.rfile, r.method, None)

    def request(self, spec):
        """
            Return a PathocResult namedtuple.

            May raise language.ParseException, netlib.http.HttpError or
            language.FileAccessDenied.
        """
        r = language.parse_request(self.settings, spec)
        ret = language.serve(r, self.wfile, self.settings, self.host)
        self.wfile.flush()
        return PathocResult._make(http.read_response(self.rfile, r.method, None))

    def request(self, spec):
        """
            Return an (httpversion, code, msg, headers, content) tuple.

            May raise language.ParseException, netlib.http.HttpError or
            language.FileAccessDenied.
        """
        r = language.parse_request(self.settings, spec)
        language.serve(r, self.wfile, self.settings, self.address.host)
        self.wfile.flush()
        ret = list(http.read_response(self.rfile, r.method, None))
        ret.append(self.sslinfo)
        return Response(*ret)

    def handle_request(self, cc):
        try:
            request, err = None, None
            request = self.read_request(cc)
            if request is None:
                return
            cc.requestcount += 1

            app = self.server.apps.get(request)
            if app:
                app.serve(request, self.wfile)
            else:
                request = request._send(self.mqueue)
                if request is None:
                    return

                if isinstance(request, flow.Response):
                    response = request
                    request = False
                    response = response._send(self.mqueue)
                else:
                    if self.config.reverse_proxy:
                        scheme, host, port = self.config.reverse_proxy
                    else:
                        scheme, host, port = request.scheme, request.host, request.port
                    self.server_connect(scheme, host, port)
                    self.server_conn.send(request)
                    httpversion, code, msg, headers, content = http.read_response(
                        self.server_conn.rfile,
                        request.method,
                        self.config.body_size_limit
                    )
                    response = flow.Response(
                        request, httpversion, code, msg, headers, content, self.server_conn.cert
                    )
                    response = response._send(self.mqueue)
                    if response is None:
                        self.server_conn.terminate()
                if response is None:
                    return
                self.send_response(response)
                if http.request_connection_close(request.httpversion, request.headers):
                    return
                # We could keep the client connection when the server
                # connection needs to go away.  However, we want to mimic
                # behaviour as closely as possible to the client, so we
                # disconnect.
                if http.response_connection_close(response.httpversion, response.headers):
                    return
        except IOError, v:
            cc.connection_error = v

def create_http_response(flowheader, respbuf, request):
    sfp = StringIO(respbuf)
    httpversion, code, msg, headers, content = http.read_response(sfp, request.method, None)
    return flow.Response(
        request,
        httpversion,
        code,
        msg,
        headers,
        content,
        None,
        flowheader.ts_response_start,
        flowheader.ts_response_finish,
    )

    def connect(self):
        super(WebSocketsClient, self).connect()

        preamble = http.request_preamble("GET", "/")
        self.wfile.write(preamble + "\r\n")
        headers = websockets.client_handshake_headers()
        self.client_nonce = headers.get_first("sec-websocket-key")
        self.wfile.write(headers.format() + "\r\n")
        self.wfile.flush()

        resp = http.read_response(self.rfile, "get", None)
        server_nonce = websockets.check_server_handshake(resp.headers)

        if not server_nonce == websockets.create_server_nonce(self.client_nonce):
            self.close()

 def run(self):
     try:
         r = self.flow.request
         server = ServerConnection(self.config, r.host, r.port)
         server.connect(r.scheme)
         server.send(r)
         httpversion, code, msg, headers, content = http.read_response(
             server.rfile, r.method, self.config.body_size_limit
         )
         response = flow.Response(
             self.flow.request, httpversion, code, msg, headers, content, server.cert
         )
         response._send(self.masterq)
     except (ProxyError, http.HttpError, tcp.NetLibError), v:
         err = flow.Error(self.flow.request, str(v))
         err._send(self.masterq)

 def run(self):
     try:
         r = self.flow.request
         server = ServerConnection(self.config, r.scheme, r.host, r.port, r.host)
         server.connect()
         server.send(r)
         httpversion, code, msg, headers, content = http.read_response(
             server.rfile, r.method, self.config.body_size_limit
         )
         response = flow.Response(
             self.flow.request, httpversion, code, msg, headers, content, server.cert, 
             server.rfile.first_byte_timestamp
         )
         self.channel.ask("response", response)
     except (ProxyError, http.HttpError, tcp.NetLibError), v:
         err = flow.Error(self.flow.request, str(v))
         self.channel.ask("error", err)

    def pipelined_requests(self, specs):
        """
            Return a list of PathocResult namedtuple's.
            The requests defined by the list of spec are sent using http pipelining.

            May raise language.ParseException, netlib.http.HttpError or
            language.FileAccessDenied.
        """
        parsed_specs = {}  # spec -> r, ret
        for spec in specs:
            r = language.parse_request(self.settings, spec)
            ret = language.serve(r, self.wfile, self.settings, self.host)
            parsed_specs[spec] = r, ret
        self.wfile.flush()
        rval = []
        for spec in specs:
            r, ret = parsed_specs[spec]
            result = PathocResult._make(http.read_response(self.rfile, r.method, None))
            rval.append(result)
        return rval

    def from_stream(cls, rfile, request_method, include_content=True, body_size_limit=None):
        """
        Parse an HTTP response from a file stream
        """
        if not include_content:
            raise NotImplementedError  # pragma: nocover

        if hasattr(rfile, "reset_timestamps"):
            rfile.reset_timestamps()

        httpversion, code, msg, headers, content = http.read_response(
            rfile,
            request_method,
            body_size_limit)

        if hasattr(rfile, "first_byte_timestamp"):
            timestamp_start = rfile.first_byte_timestamp
        else:
            timestamp_start = utils.timestamp()

        timestamp_end = utils.timestamp()
        return HTTPResponse(httpversion, code, msg, headers, content, timestamp_start, timestamp_end)

    def print_request(
        self,
        r,
        showreq,
        showresp,
        explain,
        showssl,
        hexdump,
        ignorecodes,
        ignoretimeout,
        fp=sys.stdout
    ):
        """
            Performs a series of requests, and prints results to the specified
            file descriptor.

            spec: A request specification
            showreq: Print requests
            showresp: Print responses
            explain: Print request explanation
            showssl: Print info on SSL connection
            hexdump: When printing requests or responses, use hex dump output
            ignorecodes: Sequence of return codes to ignore

            Returns True if we have a non-ignored response.
        """
        resp, req = None, None
        if showreq:
            self.wfile.start_log()
        if showresp:
            self.rfile.start_log()
        try:
            req = language.serve(
                r,
                self.wfile,
                self.settings,
                self.address.host
            )
            self.wfile.flush()
            resp = http.read_response(self.rfile, r.method.string(), None)
        except http.HttpError as v:
            print("<< HTTP Error:", v.message, file=fp)
        except tcp.NetLibTimeout:
            if ignoretimeout:
                return
            print("<<", "Timeout", file=fp)
        except tcp.NetLibDisconnect: # pragma: nocover
            print("<<", "Disconnect", file=fp)

        if req:
            if ignorecodes and resp and resp[1] in ignorecodes:
                return

            if explain:
                print(">> Spec:", r.spec(), file=fp)

            if showreq:
                self._show(fp, ">> Request", self.wfile.get_log(), hexdump)

            if showresp:
                self._show(fp, "<< Response", self.rfile.get_log(), hexdump)
            else:
                if resp:
                    self._show_summary(fp, *resp)

            if showssl and self.sslinfo:
                print("Cipher: %s, %s bit, %s"%self.sslinfo.cipher, file=fp)
                print("SSL certificate chain:\n", file=fp)
                for i in self.sslinfo.certchain:
                    print("\tSubject: ", end=' ', file=fp)
                    for cn in i.get_subject().get_components():
                        print("%s=%s"%cn, end=' ', file=fp)
                    print(file=fp)
                    print("\tIssuer: ", end=' ', file=fp)
                    for cn in i.get_issuer().get_components():
                        print("%s=%s"%cn, end=' ', file=fp)
                    print(file=fp)
                    print("\tVersion: %s"%i.get_version(), file=fp)
                    print("\tValidity: %s - %s"%(
                        i.get_notBefore(), i.get_notAfter()
                    ), file=fp)
                    print("\tSerial: %s"%i.get_serial_number(), file=fp)
                    print("\tAlgorithm: %s"%i.get_signature_algorithm(), file=fp)
                    pk = i.get_pubkey()
                    types = {
                        OpenSSL.crypto.TYPE_RSA: "RSA",
                        OpenSSL.crypto.TYPE_DSA: "DSA"
                    }
                    t = types.get(pk.type(), "Uknown")
                    print("\tPubkey: %s bit %s"%(pk.bits(), t), file=fp)
                    s = certutils.SSLCert(i)
                    if s.altnames:
                        print("\tSANs:", " ".join(s.altnames), file=fp)
                    print(file=fp)
            return True

    def handle_request(self, cc):
        try:
            request, err = None, None
            request = self.read_request(cc)
            if request is None:
                return
            cc.requestcount += 1

            app = self.server.apps.get(request)
            if app:
                err = app.serve(request, self.wfile)
                self.log(cc, "Error in wsgi app.", err.split("\n"))
                if err:
                    return
            else:
                request = request._send(self.mqueue)
                if request is None:
                    return

                if isinstance(request, flow.Response):
                    response = request
                    request = False
                    response = response._send(self.mqueue)
                else:
                    if self.config.reverse_proxy:
                        scheme, host, port = self.config.reverse_proxy
                    else:
                        scheme, host, port = request.scheme, request.host, request.port
                    self.server_connect(scheme, host, port)
                    self.server_conn.send(request)
                    httpversion, code, msg, headers, content = http.read_response(
                        self.server_conn.rfile,
                        request.method,
                        self.config.body_size_limit
                    )
                    response = flow.Response(
                        request, httpversion, code, msg, headers, content, self.server_conn.cert
                    )
                    response = response._send(self.mqueue)
                    if response is None:
                        self.server_conn.terminate()
                if response is None:
                    return
                self.send_response(response)
                if http.request_connection_close(request.httpversion, request.headers):
                    return
                # We could keep the client connection when the server
                # connection needs to go away.  However, we want to mimic
                # behaviour as closely as possible to the client, so we
                # disconnect.
                if http.response_connection_close(response.httpversion, response.headers):
                    return
        except (IOError, ProxyError, http.HttpError, tcp.NetLibDisconnect), e:
            if hasattr(e, "code"):
                cc.error = "%s: %s"%(e.code, e.msg)
            else:
                cc.error = str(e)

            if request:
                err = flow.Error(request, cc.error)
                err._send(self.mqueue)
                self.log(
                    cc, cc.error,
                    ["url: %s"%request.get_url()]
                )
            else:
                self.log(cc, cc.error)

            if isinstance(e, ProxyError):
                self.send_error(e.code, e.msg)