def got_response_collecting(self, stream, request, response):
        logging.info("BitTorrent: collecting ... done")

        if self.success:
            #
            # Always measure at the receiver because there is more
            # information at the receiver and also to make my friend
            # Enrico happier :-P.
            # The following is not a bug: it's just that the server
            # returns a result using the point of view of the client,
            # i.e. upload_speed is _our_ upload speed.
            #
            m = json.loads(response.body.read())
            self.my_side["upload_speed"] = m["upload_speed"]

            upload = utils.speed_formatter(m["upload_speed"])
            STATE.update("test_progress", "100%", publish=False)
            STATE.update("test_upload", upload)
            logging.info('BitTorrent: upload speed: %s', upload)

            if privacy.collect_allowed(self.my_side):
                if DATABASE.readonly:
                    logging.warning('bittorrent_client: readonly database')
                else:
                    table_bittorrent.insert(DATABASE.connection(), self.my_side)

            # Update the upstream channel estimate
            target_bytes = int(m["target_bytes"])
            if target_bytes > 0:
                estimate.UPLOAD = target_bytes

            self.final_state = True

        stream.close()

 def _schedule_after(self, interval):
     ''' Schedule next rendezvous after interval seconds '''
     logging.info('background_rendezvous: next rendezvous in %d seconds',
                  interval)
     timestamp = POLLER.sched(interval, self.run)
     STATE.update('idle', publish=False)
     STATE.update('next_rendezvous', timestamp)

 def _start_collect(self, stream, result):
     ''' Start the COLLECT phase '''
     STATE.update('collect')
     logging.debug('raw_negotiate: collect in progress...')
     context = stream.opaque
     extra = context.extra
     extra['local_result'] = result
     body = six.b(json.dumps(result))
     host_header = utils_net.format_epnt((extra['address'], extra['port']))
     self.append_request(stream, 'POST', '/collect/raw', 'HTTP/1.1')
     self.append_header(stream, 'Host', host_header)
     self.append_header(stream, 'User-Agent', utils_version.HTTP_HEADER)
     self.append_header(stream, 'Content-Type', 'application/json')
     self.append_header(stream, 'Content-Length', str(len(body)))
     self.append_header(stream, 'Cache-Control', 'no-cache')
     self.append_header(stream, 'Pragma', 'no-cache')
     self.append_header(stream, 'Connection', 'close')
     if extra['authorization']:
         self.append_header(stream, 'Authorization', extra['authorization'])
     self.append_end_of_headers(stream)
     self.append_bytes(stream, body)
     http_utils.prettyprint_json(result, '>')
     self.send_message(stream)
     context.body = six.StringIO()  # Want to save body
     extra['requests'] += 1

    def _api_config(self, stream, request, query):
        response = Message()

        indent, mimetype, sort_keys = None, "application/json", False
        dictionary = cgi.parse_qs(query)
        if "debug" in dictionary and utils.intify(dictionary["debug"][0]):
            indent, mimetype, sort_keys = 4, "text/plain", True

        if request.method == "POST":
            s = request.body.read()
            updates = qs_to_dictionary(s)
            privacy.check(updates)

            # Very low barrier to prevent damage from kiddies
            if "agent.interval" in updates:
                interval = int(updates["agent.interval"])
                if interval < 1380 and interval != 0:
                    raise ConfigError("Bad agent.interval")

            CONFIG.merge_api(updates, DATABASE.connection())
            STATE.update("config", updates)
            # Empty JSON b/c '204 No Content' is treated as an error
            s = "{}"
        else:
            s = json.dumps(CONFIG.conf, sort_keys=sort_keys, indent=indent)

        stringio = StringIO.StringIO(s)
        response.compose(code="200", reason="Ok", body=stringio,
                         mimetype=mimetype)
        stream.send_response(request, response)

    def got_response_negotiating(self, stream, request, response):
        m = json.loads(response.body.read())

        PROPERTIES = ("authorization", "queue_pos", "real_address", "unchoked")
        for k in PROPERTIES:
            self.conf["_%s" % k] = m[k]

        if not self.conf["_unchoked"]:
            LOG.complete("done (queue_pos %d)" % m["queue_pos"])
            STATE.update("negotiate", {"queue_pos": m["queue_pos"]})
            self.connection_ready(stream)
        else:
            LOG.complete("done (unchoked)")

            sha1 = hashlib.sha1()
            sha1.update(m["authorization"])
            self.conf["bittorrent.my_id"] = sha1.digest()
            LOG.debug("* My ID: %s" % sha1.hexdigest())
            self.http_stream = stream
            self.negotiating = False
            peer = PeerNeubot(self.poller)
            peer.complete = self.peer_test_complete
            peer.connection_lost = self.peer_connection_lost
            peer.connection_failed = self.peer_connection_failed
            peer.configure(self.conf)
            peer.connect((self.http_stream.peername[0],
                          self.conf["bittorrent.port"]))

def runner_api_done(state):
    ''' Invoked when the test completes successfully '''
    #
    # State value should be 'idle'.  This is needed otherwise the GUI stays
    # on collect after a test is run on demand.
    #
    STATE.update(state)

 def __init__(self, poller):
     ClientHTTP.__init__(self, poller)
     STATE.update("test_name", "speedtest")
     self.child = None
     self.streams = collections.deque()
     self.finished = False
     self.state = None

    def _waiting_pingback(self, stream, data):
        ''' Invoke when waiting for PINGBACK '''
        #context = stream.opaque
        #context.bufferise(data)
        #tmp = context.pullup(len(PINGBACK))
        #if not tmp:
        stream.recv(len(PINGBACK), self._waiting_pingback)
        return

        if tmp[4:5] != PINGBACK_CODE:
            raise RuntimeError('skype_clnt: received invalid message')
        timediff = utils.ticks() - context.alrtt_ticks
        context.state.setdefault('alrtt_list', []).append(timediff)
        logging.debug('< PINGBACK')
        logging.debug('skype_clnt: alrtt_sample: %f', timediff)
        context.alrtt_cnt -= 1
        if context.alrtt_cnt > 0:
            self._send_ping(stream)
            return
        alrtt_list = context.state['alrtt_list']
        alrtt_avg = sum(alrtt_list) / len(alrtt_list)
        context.state['alrtt_avg'] = alrtt_avg
        latency = utils.time_formatter(alrtt_avg)
        logging.info('skype_clnt: alrtt_avg: %s', latency)
        STATE.update('test_latency', latency)
        logging.info('skype_clnt: estimating ALRTT... complete')
        logging.info('skype_clnt: skype goodput test... in progress')
        logging.debug('> RAWTEST')
        stream.send(RAWTEST, self._skypetest_sent)

def runner_api_done():
    ''' Invoked when the test is done '''
    #
    # Needed otherwise the GUI stays on collect after a
    # test is run on demand.
    #
    STATE.update('idle')

    def got_response_collecting(self, stream, request, response):
        LOG.complete()

        if self.success:
            #
            # Always measure at the receiver because there is more
            # information at the receiver and also to make my friend
            # Enrico happier :-P.
            # The following is not a bug: it's just that the server
            # returns a result using the point of view of the client,
            # i.e. upload_speed is _our_ upload speed.
            #
            m = json.loads(response.body.read())
            self.my_side["upload_speed"] = m["upload_speed"]

            upload = utils.speed_formatter(m["upload_speed"])
            STATE.update("test_upload", upload)

            if privacy.collect_allowed(self.my_side):
                table_bittorrent.insert(DATABASE.connection(), self.my_side)

            # Update the upstream channel estimate
            target_bytes = int(m["target_bytes"])
            if target_bytes > 0:
                estimate.UPLOAD = target_bytes

        stream.close()

 def _waiting_pingback(self, stream, data):
     """ Invoke when waiting for PINGBACK """
     context = stream.opaque
     context.bufferise(data)
     tmp = context.pullup(len(PINGBACK))
     if not tmp:
         stream.recv(len(PINGBACK), self._waiting_pingback)
         return
     if tmp[4:5] != PINGBACK_CODE:
         raise RuntimeError("raw_clnt: received invalid message")
     timediff = utils.ticks() - context.alrtt_ticks
     context.state.setdefault("alrtt_list", []).append(timediff)
     logging.debug("< PINGBACK")
     logging.debug("raw_clnt: alrtt_sample: %f", timediff)
     context.alrtt_cnt -= 1
     if context.alrtt_cnt > 0:
         self._send_ping(stream)
         return
     alrtt_list = context.state["alrtt_list"]
     alrtt_avg = sum(alrtt_list) / len(alrtt_list)
     context.state["alrtt_avg"] = alrtt_avg
     latency = utils.time_formatter(alrtt_avg)
     logging.info("raw_clnt: alrtt_avg: %s", latency)
     STATE.update("test_progress", "50%", publish=False)
     STATE.update("test_latency", latency)
     logging.info("raw_clnt: estimating ALRTT... complete")
     logging.info("raw_clnt: raw goodput test... in progress")
     logging.debug("> RAWTEST")
     stream.send(RAWTEST, self._rawtest_sent)

 def handle_connect(self, connector, sock, rtt, sslconfig, state):
     logging.info("raw_clnt: connection established with %s", connector)
     logging.info("raw_clnt: connect_time: %s", utils.time_formatter(rtt))
     state["connect_time"] = rtt
     Stream(sock, self._connection_ready, self._connection_lost, sslconfig, "", ClientContext(state))
     STATE.update("test", "raw")
     state["mss"] = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_MAXSEG)
     state["rcvr_data"] = []

 def handle_connect(self, connector, sock, rtt, sslconfig, state):
     logging.info('raw_clnt: connection established with %s', connector)
     logging.info('raw_clnt: connect_time: %s', utils.time_formatter(rtt))
     state['connect_time'] = rtt
     Stream(sock, self._connection_ready, self._connection_lost,
       sslconfig, '', ClientContext(state))
     STATE.update('test', 'raw')
     state['mss'] = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_MAXSEG)
     state['rcvr_data'] = []

    def connect_uri(self, uri=None, count=None):
        self._task = None

        if not uri:
            uri = "http://%s:9773/rendezvous" % CONFIG["agent.master"]

        LOG.start("* Rendezvous with %s" % uri)
        STATE.update("rendezvous")

        # We need to make just one connection
        ClientHTTP.connect_uri(self, uri, 1)

    def connection_ready(self, stream):
        LOG.complete()

        STATE.update("negotiate")
        LOG.start("BitTorrent: negotiating")

        request = Message()
        body = json.dumps({"target_bytes": self.conf["bittorrent.bytes.up"]})
        request.compose(method="GET", pathquery="/negotiate/bittorrent",
          host=self.host_header, body=body, mimetype="application/json")
        request["authorization"] = self.conf.get("_authorization", "")
        stream.send_request(request)

    def got_response(self, stream, request, response):
        if response.code != "200":
            LOG.complete("bad response")
            self._schedule()
        else:
            LOG.complete()
            s = response.body.read()
            try:
                m1 = marshal.unmarshal_object(s, "application/json", compat.RendezvousResponse)
            except ValueError:
                LOG.exception()
                self._schedule()
            else:
                if "version" in m1.update and "uri" in m1.update:
                    ver, uri = m1.update["version"], m1.update["uri"]
                    LOG.info("Version %s available at %s" % (ver, uri))
                    STATE.update("update", {"version": ver, "uri": uri})
                    _open_browser_on_windows("update.html")

                # Update tests known by the runner
                runner_lst.update(m1.available)

                #
                # Choose the test we would like to run even if
                # we're not going to run it because we're running
                # in debug mode or tests are disabled.
                # This allows us to print to the logger the test
                # we /would/ have choosen if we were allowed to run
                # it.
                #
                test = runner_lst.get_next_test()
                if not test:
                    LOG.warning("No test available")
                    self._schedule()
                    return

                LOG.info("* Chosen test: %s" % test)

                # Are we allowed to run a test?
                if not CONFIG["enabled"] or CONFIG["rendezvous.client.debug"]:
                    LOG.info("Tests are disabled... not running")
                    self._schedule()
                else:

                    # Do we have negotiate URI for test?
                    negotiate_uri = runner_lst.test_to_negotiate_uri(test)
                    if not negotiate_uri:
                        LOG.warning("No negotiate URI for test")
                        self._schedule()
                    else:

                        # Actually run the test
                        runner_core.run(test, negotiate_uri, self._schedule)

 def __init__(self, poller):
     ClientHTTP.__init__(self, poller)
     STATE.update("test_latency", "---", publish=False)
     STATE.update("test_download", "---", publish=False)
     STATE.update("test_upload", "---", publish=False)
     STATE.update("test_progress", "0%", publish=False)
     STATE.update("test_name", "bittorrent")
     self.negotiating = True
     self.http_stream = None
     self.success = False
     self.my_side = {}
     self.final_state = False

    def _after_rendezvous(self, unused):
        ''' After rendezvous actions '''

        #
        # This function is invoked both when the rendezvous fails
        # and succeeds.  If it succeeds, OK we have fresh information
        # on available tests and updates and we use it.  Otherwise,
        # if rendezvous fails, we may either have old information, or
        # no information, if this is the first rendezvous.  In any
        # case, we do our best to use the available information.
        #

        logging.info('background_rendezvous: automatic rendezvous... done')

        # Inform the user when we have updates
        new_version = RUNNER_UPDATES.get_update_version()
        new_uri = RUNNER_UPDATES.get_update_uri()
        if new_version and new_uri and not CONFIG['win32_updater']:
            logging.info('runner_rendezvous: version %s available at %s',
                         new_version, new_uri)
            STATE.update('update', {'version': new_version,
                                    'uri': new_uri})
            self._open_browser_on_windows('update.html')

        #
        # Choose the test we would like to run even if
        # we're not going to run it because tests are
        # disabled.  So we can print the test name also
        # when tests are disabled.
        #
        test = RUNNER_POLICY.get_next_test()
        logging.info('background_rendezvous: chosen test: %s', test)

        # Are we allowed to run a test?
        if not CONFIG['enabled']:
            raise RuntimeError('background_rendezvous: automatic '
                               'tests disabled')

        #
        # RAW test requires auto_discover to be True, since it uses mlab-ns
        # to discover servers.  Other tests don't need that, since, at the
        # moment, they discover servers during the rendezvous.  So, if their
        # auto_discover were True, they'd end up running two rendezvous in
        # a row for no good reason.
        #
        auto_discover = (test == 'raw')

        # Actually run the test
        deferred = Deferred()
        deferred.add_callback(self._schedule)
        RUNNER_CORE.run(test, deferred, auto_discover, None)

    def connection_ready(self, stream):
        uri = "http://%s/" % self.host_header
        logging.info("BitTorrent: connecting to %s ... done", uri)

        STATE.update("negotiate")
        logging.info("BitTorrent: negotiating in progress...")

        request = Message()
        body = json.dumps({"test_version": CONFIG['bittorrent_test_version'],
                           "target_bytes": self.conf['bittorrent.bytes.up']})
        request.compose(method="POST", pathquery="/negotiate/bittorrent",
          host=self.host_header, body=body, mimetype="application/json")
        request["authorization"] = self.conf.get("_authorization", "")
        stream.send_request(request)

    def _after_rendezvous(self, unused):
        ''' After rendezvous actions '''

        #
        # This function is invoked both when the rendezvous fails
        # and succeeds.  If it succeeds, OK we have fresh information
        # on available tests and updates and we use it.  Otherwise,
        # if rendezvous fails, we may either have old information, or
        # no information, if this is the first rendezvous.  In any
        # case, we do our best to use the available information.
        #

        logging.info('background_rendezvous: automatic rendezvous... done')

        # Inform the user when we have updates
        new_version = RUNNER_UPDATES.get_update_version()
        new_uri = RUNNER_UPDATES.get_update_uri()
        if new_version and new_uri and not CONFIG['win32_updater']:
            logging.info('runner_rendezvous: version %s available at %s',
                         new_version, new_uri)
            STATE.update('update', {'version': new_version,
                                    'uri': new_uri})

        #
        # Choose the test we would like to run even if
        # we're not going to run it because tests are
        # disabled.  So we can print the test name also
        # when tests are disabled.
        #
        # Note: we pick a test at random because now we
        # have a fixed probability of running a test.
        #
        test = RUNNER_POLICY.get_random_test()
        logging.info('background_rendezvous: chosen test: %s', test)

        # Are we allowed to run a test?
        if not CONFIG['enabled']:
            raise RuntimeError('background_rendezvous: automatic '
                               'tests disabled')

        #
        # The two legacy tests, speedtest and bittorent, use the rendezvous
        # to discover the servers. Other tests use mlab-ns.
        #
        use_mlabns = (test != 'speedtest' and test != 'bittorrent')

        # Actually run the test
        deferred = Deferred()
        deferred.add_callback(self._schedule)
        RUNNER_CORE.run(test, deferred, use_mlabns, None)