def get_user_by_req(self, request, allow_guest=False, rights="access"):
        """ Get a registered user's ID.

        Args:
            request - An HTTP request with an access_token query parameter.
        Returns:
            defer.Deferred: resolves to a ``synapse.types.Requester`` object
        Raises:
            AuthError if no user by that token exists or the token is invalid.
        """
        # Can optionally look elsewhere in the request (e.g. headers)
        try:
            user_id = yield self._get_appservice_user_id(request)
            if user_id:
                request.authenticated_entity = user_id
                defer.returnValue(synapse.types.create_requester(user_id))

            access_token = get_access_token_from_request(
                request, self.TOKEN_NOT_FOUND_HTTP_STATUS
            )

            user_info = yield self.get_user_by_access_token(access_token, rights)
            user = user_info["user"]
            token_id = user_info["token_id"]
            is_guest = user_info["is_guest"]

            # device_id may not be present if get_user_by_access_token has been
            # stubbed out.
            device_id = user_info.get("device_id")

            ip_addr = self.hs.get_ip_from_request(request)
            user_agent = request.requestHeaders.getRawHeaders(
                "User-Agent",
                default=[""]
            )[0]
            if user and access_token and ip_addr:
                preserve_context_over_fn(
                    self.store.insert_client_ip,
                    user=user,
                    access_token=access_token,
                    ip=ip_addr,
                    user_agent=user_agent,
                    device_id=device_id,
                )

            if is_guest and not allow_guest:
                raise AuthError(
                    403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN
                )

            request.authenticated_entity = user.to_string()

            defer.returnValue(synapse.types.create_requester(
                user, token_id, is_guest, device_id))
        except KeyError:
            raise AuthError(
                self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token.",
                errcode=Codes.MISSING_TOKEN
            )

    def enqueue_presence(self, destination, states):
        self.pending_presence_by_dest.setdefault(destination, {}).update({
            state.user_id: state for state in states
        })

        preserve_context_over_fn(
            self._attempt_new_transaction, destination
        )

    def enqueue_device_messages(self, destination):
        if destination == self.server_name or destination == "localhost":
            return

        if not self.can_send_to(destination):
            return

        preserve_context_over_fn(
            self._attempt_new_transaction, destination
        )

    def get_user_by_req(self, request, allow_guest=False):
        """ Get a registered user's ID.

        Args:
            request - An HTTP request with an access_token query parameter.
        Returns:
            tuple of:
                UserID (str)
                Access token ID (str)
        Raises:
            AuthError if no user by that token exists or the token is invalid.
        """
        # Can optionally look elsewhere in the request (e.g. headers)
        try:
            user_id = yield self._get_appservice_user_id(request.args)
            if user_id:
                request.authenticated_entity = user_id
                defer.returnValue(
                    Requester(UserID.from_string(user_id), "", False)
                )

            access_token = request.args["access_token"][0]
            user_info = yield self._get_user_by_access_token(access_token)
            user = user_info["user"]
            token_id = user_info["token_id"]
            is_guest = user_info["is_guest"]

            ip_addr = self.hs.get_ip_from_request(request)
            user_agent = request.requestHeaders.getRawHeaders(
                "User-Agent",
                default=[""]
            )[0]
            if user and access_token and ip_addr:
                preserve_context_over_fn(
                    self.store.insert_client_ip,
                    user=user,
                    access_token=access_token,
                    ip=ip_addr,
                    user_agent=user_agent
                )

            if is_guest and not allow_guest:
                raise AuthError(
                    403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN
                )

            request.authenticated_entity = user.to_string()

            defer.returnValue(Requester(user, token_id, is_guest))
        except KeyError:
            raise AuthError(
                self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token.",
                errcode=Codes.MISSING_TOKEN
            )

    def enqueue_failure(self, failure, destination):
        if destination == self.server_name or destination == "localhost":
            return

        if not self.can_send_to(destination):
            return

        self.pending_failures_by_dest.setdefault(
            destination, []
        ).append(failure)

        preserve_context_over_fn(
            self._attempt_new_transaction, destination
        )

def fetch_server_key(server_name, ssl_context_factory, path=KEY_API_V1):
    """Fetch the keys for a remote server."""

    factory = SynapseKeyClientFactory()
    factory.path = path
    endpoint = matrix_federation_endpoint(
        reactor, server_name, ssl_context_factory, timeout=30
    )

    for i in range(5):
        try:
            protocol = yield preserve_context_over_fn(
                endpoint.connect, factory
            )
            server_response, server_certificate = yield preserve_context_over_deferred(
                protocol.remote_key
            )
            defer.returnValue((server_response, server_certificate))
            return
        except SynapseKeyClientError as e:
            logger.exception("Error getting key for %r" % (server_name,))
            if e.status.startswith("4"):
                # Don't retry for 4xx responses.
                raise IOError("Cannot get key for %r" % server_name)
        except Exception as e:
            logger.exception(e)
    raise IOError("Cannot get key for %r" % server_name)

    def runInteraction(self, desc, func, *args, **kwargs):
        """Wraps the .runInteraction() method on the underlying db_pool."""
        current_context = LoggingContext.current_context()

        start_time = time.time() * 1000

        after_callbacks = []

        def inner_func(conn, *args, **kwargs):
            with LoggingContext("runInteraction") as context:
                sql_scheduling_timer.inc_by(time.time() * 1000 - start_time)

                if self.database_engine.is_connection_closed(conn):
                    logger.debug("Reconnecting closed database connection")
                    conn.reconnect()

                current_context.copy_to(context)
                return self._new_transaction(
                    conn, desc, after_callbacks, func, *args, **kwargs
                )

        result = yield preserve_context_over_fn(
            self._db_pool.runWithConnection,
            inner_func, *args, **kwargs
        )

        for after_callback, after_args in after_callbacks:
            after_callback(*after_args)
        defer.returnValue(result)

    def get_raw(self, uri, args={}):
        """ Gets raw text from the given URI.

        Args:
            uri (str): The URI to request, not including query parameters
            args (dict): A dictionary used to create query strings, defaults to
                None.
                **Note**: The value of each key is assumed to be an iterable
                and *not* a string.
        Returns:
            Deferred: Succeeds when we get *any* 2xx HTTP response, with the
            HTTP body at text.
        Raises:
            On a non-2xx HTTP response. The response body will be used as the
            error message.
        """
        if len(args):
            query_bytes = urllib.urlencode(args, True)
            uri = "%s?%s" % (uri, query_bytes)

        response = yield self.request(
            "GET",
            uri.encode("ascii"),
            headers=Headers({
                b"User-Agent": [self.user_agent],
            })
        )

        body = yield preserve_context_over_fn(readBody, response)

        if 200 <= response.code < 300:
            defer.returnValue(body)
        else:
            raise CodeMessageException(response.code, body)

    def enqueue_edu(self, edu, key=None):
        destination = edu.destination

        if not self.can_send_to(destination):
            return

        if key:
            self.pending_edus_keyed_by_dest.setdefault(
                destination, {}
            )[(edu.edu_type, key)] = edu
        else:
            self.pending_edus_by_dest.setdefault(destination, []).append(edu)

        preserve_context_over_fn(
            self._attempt_new_transaction, destination
        )

    def request(self, method, uri, *args, **kwargs):
        # A small wrapper around self.agent.request() so we can easily attach
        # counters to it
        outgoing_requests_counter.inc(method)
        d = preserve_context_over_fn(
            self.agent.request,
            method, uri, *args, **kwargs
        )

        logger.info("Sending request %s %s", method, uri)

        def _cb(response):
            incoming_responses_counter.inc(method, response.code)
            logger.info(
                "Received response to  %s %s: %s",
                method, uri, response.code
            )
            return response

        def _eb(failure):
            incoming_responses_counter.inc(method, "ERR")
            logger.info(
                "Error sending request to  %s %s: %s %s",
                method, uri, failure.type, failure.getErrorMessage()
            )
            return failure

        d.addCallbacks(_cb, _eb)

        return d

                    def send_request():
                        request_deferred = preserve_context_over_fn(
                            self.agent.request, method, url_bytes, Headers(headers_dict), producer
                        )

                        return self.clock.time_bound_deferred(
                            request_deferred, time_out=timeout / 1000.0 if timeout else 60
                        )

    def _generate_remote_thumbnails(self, server_name, media_id, media_info):
        media_type = media_info["media_type"]
        file_id = media_info["filesystem_id"]
        requirements = self._get_thumbnail_requirements(media_type)
        if not requirements:
            return

        remote_thumbnails = []

        input_path = self.filepaths.remote_media_filepath(server_name, file_id)
        thumbnailer = Thumbnailer(input_path)
        m_width = thumbnailer.width
        m_height = thumbnailer.height

        def generate_thumbnails():
            if m_width * m_height >= self.max_image_pixels:
                logger.info("Image too large to thumbnail %r x %r > %r", m_width, m_height, self.max_image_pixels)
                return

            scales = set()
            crops = set()
            for r_width, r_height, r_method, r_type in requirements:
                if r_method == "scale":
                    t_width, t_height = thumbnailer.aspect(r_width, r_height)
                    scales.add((min(m_width, t_width), min(m_height, t_height), r_type))
                elif r_method == "crop":
                    crops.add((r_width, r_height, r_type))

            for t_width, t_height, t_type in scales:
                t_method = "scale"
                t_path = self.filepaths.remote_media_thumbnail(
                    server_name, file_id, t_width, t_height, t_type, t_method
                )
                self._makedirs(t_path)
                t_len = thumbnailer.scale(t_path, t_width, t_height, t_type)
                remote_thumbnails.append([server_name, media_id, file_id, t_width, t_height, t_type, t_method, t_len])

            for t_width, t_height, t_type in crops:
                if (t_width, t_height, t_type) in scales:
                    # If the aspect ratio of the cropped thumbnail matches a purely
                    # scaled one then there is no point in calculating a separate
                    # thumbnail.
                    continue
                t_method = "crop"
                t_path = self.filepaths.remote_media_thumbnail(
                    server_name, file_id, t_width, t_height, t_type, t_method
                )
                self._makedirs(t_path)
                t_len = thumbnailer.crop(t_path, t_width, t_height, t_type)
                remote_thumbnails.append([server_name, media_id, file_id, t_width, t_height, t_type, t_method, t_len])

        yield preserve_context_over_fn(threads.deferToThread, generate_thumbnails)

        for r in remote_thumbnails:
            yield self.store.store_remote_media_thumbnail(*r)

        defer.returnValue({"width": m_width, "height": m_height})

    def get_json(self, destination, path, args={}, retry_on_dns_fail=True,
                 timeout=None):
        """ GETs some json from the given host homeserver and path

        Args:
            destination (str): The remote server to send the HTTP request
                to.
            path (str): The HTTP path.
            args (dict): A dictionary used to create query strings, defaults to
                None.
            timeout (int): How long to try (in ms) the destination for before
                giving up. None indicates no timeout and that the request will
                be retried.
        Returns:
            Deferred: Succeeds when we get *any* HTTP response.

            The result of the deferred is a tuple of `(code, response)`,
            where `response` is a dict representing the decoded JSON body.
        """
        logger.debug("get_json args: %s", args)

        encoded_args = {}
        for k, vs in args.items():
            if isinstance(vs, basestring):
                vs = [vs]
            encoded_args[k] = [v.encode("UTF-8") for v in vs]

        query_bytes = urllib.urlencode(encoded_args, True)
        logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail)

        def body_callback(method, url_bytes, headers_dict):
            self.sign_request(destination, method, url_bytes, headers_dict)
            return None

        response = yield self._create_request(
            destination.encode("ascii"),
            "GET",
            path.encode("ascii"),
            query_bytes=query_bytes,
            body_callback=body_callback,
            retry_on_dns_fail=retry_on_dns_fail,
            timeout=timeout,
        )

        if 200 <= response.code < 300:
            # We need to update the transactions table to say it was sent?
            c_type = response.headers.getRawHeaders("Content-Type")

            if "application/json" not in c_type:
                raise RuntimeError(
                    "Content-Type not application/json"
                )

        body = yield preserve_context_over_fn(readBody, response)

        defer.returnValue(json.loads(body))

    def put_json(self, destination, path, data={}, json_data_callback=None,
                 long_retries=False, timeout=None):
        """ Sends the specifed json data using PUT

        Args:
            destination (str): The remote server to send the HTTP request
                to.
            path (str): The HTTP path.
            data (dict): A dict containing the data that will be used as
                the request body. This will be encoded as JSON.
            json_data_callback (callable): A callable returning the dict to
                use as the request body.
            long_retries (bool): A boolean that indicates whether we should
                retry for a short or long time.
            timeout(int): How long to try (in ms) the destination for before
                giving up. None indicates no timeout.

        Returns:
            Deferred: Succeeds when we get a 2xx HTTP response. The result
            will be the decoded JSON body. On a 4xx or 5xx error response a
            CodeMessageException is raised.
        """

        if not json_data_callback:
            def json_data_callback():
                return data

        def body_callback(method, url_bytes, headers_dict):
            json_data = json_data_callback()
            self.sign_request(
                destination, method, url_bytes, headers_dict, json_data
            )
            producer = _JsonProducer(json_data)
            return producer

        response = yield self._create_request(
            destination.encode("ascii"),
            "PUT",
            path.encode("ascii"),
            body_callback=body_callback,
            headers_dict={"Content-Type": ["application/json"]},
            long_retries=long_retries,
            timeout=timeout,
        )

        if 200 <= response.code < 300:
            # We need to update the transactions table to say it was sent?
            c_type = response.headers.getRawHeaders("Content-Type")

            if "application/json" not in c_type:
                raise RuntimeError(
                    "Content-Type not application/json"
                )

        body = yield preserve_context_over_fn(readBody, response)
        defer.returnValue(json.loads(body))

    def get_file(self, url, output_stream, max_size=None):
        """GETs a file from a given URL
        Args:
            url (str): The URL to GET
            output_stream (file): File to write the response body to.
        Returns:
            A (int,dict,string,int) tuple of the file length, dict of the response
            headers, absolute URI of the response and HTTP response code.
        """

        response = yield self.request(
            "GET",
            url.encode("ascii"),
            headers=Headers({
                b"User-Agent": [self.user_agent],
            })
        )

        headers = dict(response.headers.getAllRawHeaders())

        if 'Content-Length' in headers and headers['Content-Length'] > max_size:
            logger.warn("Requested URL is too large > %r bytes" % (self.max_size,))
            raise SynapseError(
                502,
                "Requested file is too large > %r bytes" % (self.max_size,),
                Codes.TOO_LARGE,
            )

        if response.code > 299:
            logger.warn("Got %d when downloading %s" % (response.code, url))
            raise SynapseError(
                502,
                "Got error %d" % (response.code,),
                Codes.UNKNOWN,
            )

        # TODO: if our Content-Type is HTML or something, just read the first
        # N bytes into RAM rather than saving it all to disk only to read it
        # straight back in again

        try:
            length = yield preserve_context_over_fn(
                _readBodyToFile,
                response, output_stream, max_size
            )
        except Exception as e:
            logger.exception("Failed to download body")
            raise SynapseError(
                502,
                ("Failed to download remote body: %s" % e),
                Codes.UNKNOWN,
            )

        defer.returnValue((length, headers, response.request.absoluteURI, response.code))

    def generate_local_exact_thumbnail(self, media_id, t_width, t_height, t_method, t_type):
        input_path = self.filepaths.local_media_filepath(media_id)

        t_path = self.filepaths.local_media_thumbnail(media_id, t_width, t_height, t_type, t_method)
        self._makedirs(t_path)

        t_len = yield preserve_context_over_fn(
            threads.deferToThread, self._generate_thumbnail, input_path, t_path, t_width, t_height, t_method, t_type
        )

        if t_len:
            yield self.store.store_local_thumbnail(media_id, t_width, t_height, t_type, t_method, t_len)

            defer.returnValue(t_path)

    def enqueue_pdu(self, pdu, destinations, order):
        # We loop through all destinations to see whether we already have
        # a transaction in progress. If we do, stick it in the pending_pdus
        # table and we'll get back to it later.

        destinations = set(destinations)
        destinations = set(
            dest for dest in destinations if self.can_send_to(dest)
        )

        logger.debug("Sending to: %s", str(destinations))

        if not destinations:
            return

        for destination in destinations:
            self.pending_pdus_by_dest.setdefault(destination, []).append(
                (pdu, order)
            )

            preserve_context_over_fn(
                self._attempt_new_transaction, destination
            )

    def post_json_get_json(self, uri, post_json):
        json_str = encode_canonical_json(post_json)

        logger.debug("HTTP POST %s -> %s", json_str, uri)

        response = yield self.request(
            "POST",
            uri.encode("ascii"),
            headers=Headers({b"Content-Type": [b"application/json"], b"User-Agent": [self.user_agent]}),
            bodyProducer=FileBodyProducer(StringIO(json_str)),
        )

        body = yield preserve_context_over_fn(readBody, response)

        defer.returnValue(json.loads(body))

    def get_file(self, destination, path, output_stream, args={},
                 retry_on_dns_fail=True, max_size=None):
        """GETs a file from a given homeserver
        Args:
            destination (str): The remote server to send the HTTP request to.
            path (str): The HTTP path to GET.
            output_stream (file): File to write the response body to.
            args (dict): Optional dictionary used to create the query string.
        Returns:
            A (int,dict) tuple of the file length and a dict of the response
            headers.
        """

        encoded_args = {}
        for k, vs in args.items():
            if isinstance(vs, basestring):
                vs = [vs]
            encoded_args[k] = [v.encode("UTF-8") for v in vs]

        query_bytes = urllib.urlencode(encoded_args, True)
        logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail)

        def body_callback(method, url_bytes, headers_dict):
            self.sign_request(destination, method, url_bytes, headers_dict)
            return None

        response = yield self._create_request(
            destination.encode("ascii"),
            "GET",
            path.encode("ascii"),
            query_bytes=query_bytes,
            body_callback=body_callback,
            retry_on_dns_fail=retry_on_dns_fail
        )

        headers = dict(response.headers.getAllRawHeaders())

        try:
            length = yield preserve_context_over_fn(
                _readBodyToFile,
                response, output_stream, max_size
            )
        except:
            logger.exception("Failed to download body")
            raise

        defer.returnValue((length, headers))

    def post_json(self, destination, path, data={}, long_retries=True):
        """ Sends the specifed json data using POST

        Args:
            destination (str): The remote server to send the HTTP request
                to.
            path (str): The HTTP path.
            data (dict): A dict containing the data that will be used as
                the request body. This will be encoded as JSON.
            long_retries (bool): A boolean that indicates whether we should
                retry for a short or long time.

        Returns:
            Deferred: Succeeds when we get a 2xx HTTP response. The result
            will be the decoded JSON body. On a 4xx or 5xx error response a
            CodeMessageException is raised.
        """

        def body_callback(method, url_bytes, headers_dict):
            self.sign_request(
                destination, method, url_bytes, headers_dict, data
            )
            return _JsonProducer(data)

        response = yield self._create_request(
            destination.encode("ascii"),
            "POST",
            path.encode("ascii"),
            body_callback=body_callback,
            headers_dict={"Content-Type": ["application/json"]},
            long_retries=True,
        )

        if 200 <= response.code < 300:
            # We need to update the transactions table to say it was sent?
            c_type = response.headers.getRawHeaders("Content-Type")

            if "application/json" not in c_type:
                raise RuntimeError(
                    "Content-Type not application/json"
                )

        body = yield preserve_context_over_fn(readBody, response)

        defer.returnValue(json.loads(body))