def post_collection(
        project, th_collection, status=None, expect_errors=False,
        consumer_key=None, consumer_secret=None):

    # Set the credentials
    OAuthCredentials.set_credentials(SampleData.get_credentials())

    credentials = OAuthCredentials.get_credentials(project)

    # The only time the credentials should be overridden are when
    # a client needs to test authentication failure confirmation
    consumer_key = consumer_key or credentials['consumer_key']
    consumer_secret = consumer_secret or credentials['consumer_secret']

    auth = TreeherderAuth(consumer_key, consumer_secret, project)
    client = TreeherderClient(protocol='http', host='localhost', auth=auth)
    uri = client._get_project_uri(project, th_collection.endpoint_base)

    req = Request('POST', uri,
                  json=th_collection.get_collection_data(),
                  auth=auth)
    prepped_request = req.prepare()

    response = TestApp(application).post_json(
        prepped_request.url,
        params=th_collection.get_collection_data(),
        status=status
    )

    return response

    def test_send_with_oauth(self, mock_post, mock_time,
                             mock_generate_nonce):

        """Tests that oauth data is sent to server"""
        mock_time.return_value = 1342229050
        mock_generate_nonce.return_value = "46810593"
        mock_post.return_value = self._expected_response_return_object()

        client = TreeherderClient(
            protocol='http',
            host='host',
            )

        tjc = TreeherderJobCollection()

        for job in self.job_data:

            tjc.add(tjc.get_job(job))
            break

        client.post_collection('project', 'key', 'secret', tjc)

        self.assertEqual(mock_post.call_count, 1)

        path, resp = mock_post.call_args
        self.assertEqual(path[0], "http://host/api/project/project/objectstore/?oauth_body_hash=IKbDoi5GvTRaqjRTCDyKIN5wWiY%3D&oauth_nonce=46810593&oauth_timestamp=1342229050&oauth_consumer_key=key&oauth_signature_method=HMAC-SHA1&oauth_version=1.0&oauth_token=&user=project&oauth_signature=uq%2BrkJCRPyPUdXExSasm25ab8m4%3D")

def post_treeherder_collections(th_collections, chunk_size=1):

    errors = []
    cli = TreeherderClient(
        protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
        host=settings.TREEHERDER_REQUEST_HOST,
    )

    for project in th_collections:

        credentials = OAuthCredentials.get_credentials(project)

        auth = TreeherderAuth(credentials.get('consumer_key'),
                              credentials.get('consumer_secret'),
                              project)

        logger.info(
            "collection loading request for project {0}: {1}".format(
                project,
                th_collections[project].endpoint_base))

        collection_chunks = th_collections[project].get_chunks(chunk_size)

        for collection in collection_chunks:
            try:
                cli.post_collection(project, collection, auth=auth)
            except Exception:
                errors.append({
                    "project": project,
                    "url": th_collections[project].endpoint_base,
                    "message": traceback.format_exc()
                })

    if errors:
        raise CollectionNotLoadedException(errors)

    def test_send_result_collection(self, mock_post):
        """Can add a treeherder collections to a TreeherderRequest."""
        mock_post.return_value = self._expected_response_return_object()

        trc = TreeherderResultSetCollection()

        for resultset in self.resultset_data:

            trc.add(trc.get_resultset(resultset))

        client = TreeherderClient(
            protocol='http',
            host='host',
            )

        auth = TreeherderAuth('key', 'secret', 'project')
        client.post_collection('project', trc, auth=auth)

        path, resp = mock_post.call_args

        self.assertEqual(mock_post.call_count, 1)
        self.assertEqual(
            trc.get_collection_data(),
            resp['json']
            )

    def test_post_job_collection(self, mock_post):
        """Can add a treeherder collections to a TreeherderRequest."""
        mock_post.return_value = self._expected_response_return_object()

        tjc = TreeherderJobCollection()

        for job in self.job_data:
            tjc.add(tjc.get_job(job))

        client = TreeherderClient(
            protocol='http',
            host='host',
            client_id='client-abc',
            secret='secret123',
            )

        client.post_collection('project', tjc)

        path, resp = mock_post.call_args

        self.assertEqual(mock_post.call_count, 1)
        self.assertEqual(
            tjc.get_collection_data(),
            resp['json']
            )

    def test_send_artifact_collection(self, mock_post):
        """Can add a artifact collections to a TreeherderRequest."""
        mock_post.return_value = self._expected_response_return_object()

        tac = TreeherderArtifactCollection()

        for artifact in self.artifact_data:
            tac.add(tac.get_artifact(artifact))

        client = TreeherderClient(
            protocol='http',
            host='host',
            client_id='client-abc',
            secret='secret123',
            )

        client.post_collection('project', tac)

        path, resp = mock_post.call_args

        self.assertEqual(mock_post.call_count, 1)
        self.assertEqual(
            tac.get_collection_data(),
            resp['json']
            )

def post_treeherder_collections(th_collections, chunk_size=1):

    errors = []
    credentials = Credentials.objects.get(client_id=settings.ETL_CLIENT_ID)

    cli = TreeherderClient(
        protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
        host=settings.TREEHERDER_REQUEST_HOST,
        client_id=credentials.client_id,
        secret=str(credentials.secret),
    )

    for project in th_collections:
        logger.info(
            "collection loading request for project {0}: {1}".format(
                project,
                th_collections[project].endpoint_base))

        collection_chunks = th_collections[project].get_chunks(chunk_size)

        for collection in collection_chunks:
            try:
                cli.post_collection(project, collection)
            except Exception:
                errors.append({
                    "project": project,
                    "url": th_collections[project].endpoint_base,
                    "message": traceback.format_exc()
                })

    if errors:
        raise CollectionNotLoadedException(errors)

def post_collection(
        project, th_collection, status=None, expect_errors=False,
        consumer_key=None, consumer_secret=None):

    # Set the credentials
    OAuthCredentials.set_credentials(SampleData.get_credentials())

    credentials = OAuthCredentials.get_credentials(project)

    # The only time the credentials should be overridden are when
    # a client needs to test authentication failure confirmation
    if consumer_key:
        credentials['consumer_key'] = consumer_key

    if consumer_secret:
        credentials['consumer_secret'] = consumer_secret

    cli = TreeherderClient(
        protocol='http',
        host='localhost',
    )

    jsondata = th_collection.to_json()
    signed_uri = cli._get_uri(project, th_collection.endpoint_base,
                              data=jsondata,
                              oauth_key=credentials['consumer_key'],
                              oauth_secret=credentials['consumer_secret'],
                              method='POST')

    response = TestApp(application).post_json(
        str(signed_uri), params=th_collection.get_collection_data(),
        status=status
    )

    return response

def post_log_artifacts(project, job_guid, job_log_url, retry_task, extract_artifacts_cb):
    """Post a list of artifacts to a job."""

    def _retry(e):
        # Initially retry after 1 minute, then for each subsequent retry
        # lengthen the retry time by another minute.
        retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60)
        # .retry() raises a RetryTaskError exception,
        # so nothing after this function will be executed

    log_description = "%s %s (%s)" % (project, job_guid, job_log_url)
    logger.debug("Downloading/parsing log for %s", log_description)

    job_log = JobLog.objects.get(job__guid=job_guid, url=job_log_url)

    credentials = Credentials.objects.get(client_id=settings.ETL_CLIENT_ID)
    client = TreeherderClient(
        protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
        host=settings.TREEHERDER_REQUEST_HOST,
        client_id=credentials.client_id,
        secret=str(credentials.secret),
    )

    try:
        artifact_list = extract_artifacts_cb(project, job_log_url, job_guid)
    except Exception as e:
        job_log.update_status(JobLog.FAILED)

        # unrecoverable http error (doesn't exist or permission denied)
        # (apparently this can happen somewhat often with taskcluster if
        # the job fails, so just warn about it -- see
        # https://bugzilla.mozilla.org/show_bug.cgi?id=1154248)
        if isinstance(e, urllib2.HTTPError) and e.code in (403, 404):
            logger.warning("Unable to retrieve log for %s: %s", log_description, e)
            return
        # possibly recoverable http error (e.g. problems on our end)
        elif isinstance(e, urllib2.URLError):
            logger.error("Failed to download log for %s: %s", log_description, e)
            _retry(e)
        # parse error or other unrecoverable error
        else:
            logger.error("Failed to download/parse log for %s: %s", log_description, e)
        # re-raise exception if we're not retrying, so new relic sees the
        # error
        raise

    # store the artifacts generated
    tac = TreeherderArtifactCollection()
    for artifact in artifact_list:
        ta = tac.get_artifact(artifact)
        tac.add(ta)

    try:
        client.post_collection(project, tac)
        job_log.update_status(JobLog.PARSED)
        logger.debug("Finished posting artifact for %s %s", project, job_guid)
    except Exception as e:
        logger.error("Failed to upload parsed artifact for %s: %s", log_description, e)
        _retry(e)

def test_update_parse_status_nonexistent_id(test_project, mock_post_json):
    """
    Attempting to update the parse status for a non-existent log should return a 404.
    """
    client = TreeherderClient(protocol='http', host='localhost')
    non_existent_id = 9999999
    with pytest.raises(AppError) as e:
        client.update_parse_status(test_project, non_existent_id, 'parsed')
    assert "404 NOT FOUND" in str(e.value)

    def test_get_results(self):
        tdc = TreeherderClient()
        url = tdc._get_project_uri("mozilla-inbound", tdc.RESULTSET_ENDPOINT)
        content = {"meta": {"count": 3, "repository": "mozilla-inbound", "offset": 0}, "results": self.RESULTSETS}
        responses.add(responses.GET, url, json=content, match_querystring=True, status=200)

        resultsets = tdc.get_resultsets("mozilla-inbound")
        self.assertEqual(len(resultsets), 3)
        self.assertEqual(resultsets, self.RESULTSETS)

    def test_get_results(self, mock_get):

        mock_get.return_value = self._get_mock_response({
            "meta": {"count": 3, "repository": "mozilla-inbound",
                     "offset": 0},
            "results": self.RESULTSETS})

        tdc = TreeherderClient()
        resultsets = tdc.get_resultsets("mozilla-inbound")
        self.assertEqual(len(resultsets), 3)
        self.assertEqual(resultsets, self.RESULTSETS)

    def test_get_job(self):
        tdc = TreeherderClient()
        url = tdc._get_endpoint_url(tdc.JOBS_ENDPOINT, project='mozilla-inbound')
        content = {
            "meta": {"count": 3,
                     "repository": "mozilla-inbound",
                     "offset": 0},
            "results": self.JOB_RESULTS
        }
        responses.add(responses.GET, url, json=content, match_querystring=True, status=200)

        jobs = tdc.get_jobs("mozilla-inbound")
        self.assertEqual(len(jobs), 3)
        self.assertEqual(jobs, self.JOB_RESULTS)

    def test_send_artifact_collection(self, mock_post):
        """Can add a artifact collections to a TreeherderRequest."""
        mock_post.return_value = self._expected_response_return_object()

        tac = TreeherderArtifactCollection()

        for artifact in self.artifact_data:

            tac.add(tac.get_artifact(artifact))

        client = TreeherderClient(protocol="http", host="host")

        auth = TreeherderAuth("key", "secret", "project")
        client.post_collection("project", tac, auth=auth)

        path, resp = mock_post.call_args

        self.assertEqual(mock_post.call_count, 1)
        self.assertEqual(tac.get_collection_data(), resp["json"])

    def test_post_job_collection(self, mock_post):
        """Can add a treeherder collections to a TreeherderRequest."""
        mock_post.return_value = self._expected_response_return_object()

        tjc = TreeherderJobCollection()

        for job in self.job_data:

            tjc.add(tjc.get_job(job))

        client = TreeherderClient(protocol="http", host="host")

        auth = TreeherderAuth("key", "secret", "project")
        client.post_collection("project", tjc, auth=auth)

        path, resp = mock_post.call_args

        self.assertEqual(mock_post.call_count, 1)
        self.assertEqual(tjc.get_collection_data(), resp["json"])

    def test_post_job_collection(self):
        """Can add a treeherder collections to a TreeherderRequest."""
        tjc = TreeherderJobCollection()

        for job in self.job_data:
            tjc.add(tjc.get_job(job))

        client = TreeherderClient(protocol="http", host="host", client_id="client-abc", secret="secret123")

        def request_callback(request):
            # Check that the expected content was POSTed.
            posted_json = json.loads(request.body)
            self.assertEqual(posted_json, tjc.get_collection_data())
            return (200, {}, '{"message": "Job successfully updated"}')

        url = client._get_project_uri("project", tjc.endpoint_base)
        responses.add_callback(
            responses.POST, url, match_querystring=True, callback=request_callback, content_type="application/json"
        )

        client.post_collection("project", tjc)

    def test_send_result_collection(self):
        """Can add a treeherder collections to a TreeherderRequest."""
        trc = TreeherderResultSetCollection()

        for resultset in self.resultset_data:
            trc.add(trc.get_resultset(resultset))

        client = TreeherderClient(protocol="http", host="host", client_id="client-abc", secret="secret123")

        def request_callback(request):
            # Check that the expected content was POSTed.
            posted_json = json.loads(request.body)
            self.assertEqual(posted_json, trc.get_collection_data())
            return (200, {}, '{"message": "well-formed JSON stored", "resultsets": [123, 456]}')

        url = client._get_project_uri("project", trc.endpoint_base)
        responses.add_callback(
            responses.POST, url, match_querystring=True, callback=request_callback, content_type="application/json"
        )

        client.post_collection("project", trc)

    def test_send_artifact_collection(self):
        """Can add a artifact collections to a TreeherderRequest."""
        tac = TreeherderArtifactCollection()

        for artifact in self.artifact_data:
            tac.add(tac.get_artifact(artifact))

        client = TreeherderClient(protocol="http", host="host", client_id="client-abc", secret="secret123")

        def request_callback(request):
            # Check that the expected content was POSTed.
            posted_json = json.loads(request.body)
            self.assertEqual(posted_json, tac.get_collection_data())
            return (200, {}, '{"message": "Artifacts stored successfully"}')

        url = client._get_project_uri("project", tac.endpoint_base)
        responses.add_callback(
            responses.POST, url, match_querystring=True, callback=request_callback, content_type="application/json"
        )

        client.post_collection("project", tac)

    def handle(self, *args, **options):
        server_params = urlparse(options['server'])
        c = TreeherderClient(protocol=server_params.scheme,
                             host=server_params.netloc)

        # options / option collection hashes
        for (uuid, props) in c.get_option_collection_hash().iteritems():
            for prop in props:
                option, _ = Option.objects.get_or_create(name=prop['name'])
                OptionCollection.objects.get_or_create(
                    option_collection_hash=uuid,
                    option=option)

        # machine platforms
        for machine_platform in c.get_machine_platforms():
            MachinePlatform.objects.get_or_create(
                    os_name=machine_platform['os_name'],
                    platform=machine_platform['platform'],
                    architecture=machine_platform['architecture'],
                    defaults={
                        'active_status': machine_platform['active_status']
                    })

def post_log_artifacts(project,
                       job_guid,
                       job_log_url,
                       retry_task,
                       extract_artifacts_cb,
                       check_errors=False):
    """Post a list of artifacts to a job."""
    def _retry(e):
        # Initially retry after 1 minute, then for each subsequent retry
        # lengthen the retry time by another minute.
        retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60)
        # .retry() raises a RetryTaskError exception,
        # so nothing after this function will be executed

    log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
    logger.debug("Downloading/parsing log for %s", log_description)

    credentials = OAuthCredentials.get_credentials(project)
    client = TreeherderClient(
        protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
        host=settings.TREEHERDER_REQUEST_HOST
    )

    try:
        artifact_list = extract_artifacts_cb(job_log_url['url'],
                                             job_guid, check_errors)
    except Exception as e:
        client.update_parse_status(project, credentials.get('consumer_key'),
                                   credentials.get('consumer_secret'),
                                   job_log_url['id'], 'failed')
        if isinstance(e, urllib2.HTTPError) and e.code in (403, 404):
            logger.debug("Unable to retrieve log for %s: %s", log_description, e)
            return
        logger.error("Failed to download/parse log for %s: %s", log_description, e)
        _retry(e)

    # store the artifacts generated
    tac = TreeherderArtifactCollection()
    for artifact in artifact_list:
        ta = tac.get_artifact(artifact)
        tac.add(ta)

    try:
        client.post_collection(project, credentials.get('consumer_key'),
                               credentials.get('consumer_secret'),
                               tac)
        client.update_parse_status(project, credentials.get('consumer_key'),
                                   credentials.get('consumer_secret'),
                                   job_log_url['id'], 'parsed')
        logger.debug("Finished posting artifact for %s %s", project, job_guid)
    except Exception as e:
        logger.error("Failed to upload parsed artifact for %s: %s", log_description, e)
        _retry(e)