def _post_json_data(url, data):

        th_collection = data[jm.project]

        OAuthCredentials.set_credentials( SampleData.get_credentials() )
        credentials = OAuthCredentials.get_credentials(jm.project)

        tr = TreeherderRequest(
            protocol='http',
            host='localhost',
            project=jm.project,
            oauth_key=credentials['consumer_key'],
            oauth_secret=credentials['consumer_secret']
            )
        signed_uri = tr.oauth_client.get_signed_uri(
            th_collection.to_json(),
            tr.get_uri(th_collection.endpoint_base),
            "POST"
            )

        response = TestApp(application).post_json(
            str(signed_uri), params=th_collection.get_collection_data()
            )

        response.getcode = lambda: response.status_int
        return response

def post_collection(project, th_collection, status=None, expect_errors=False, consumer_key=None, consumer_secret=None):

    # Set the credentials
    OAuthCredentials.set_credentials(SampleData.get_credentials())

    credentials = OAuthCredentials.get_credentials(project)

    # The only time the credentials should be overridden are when
    # a client needs to test authentication failure confirmation
    if consumer_key:
        credentials["consumer_key"] = consumer_key

    if consumer_secret:
        credentials["consumer_secret"] = consumer_secret

    tr = TreeherderRequest(
        protocol="http",
        host="localhost",
        project=project,
        oauth_key=credentials["consumer_key"],
        oauth_secret=credentials["consumer_secret"],
    )

    signed_uri = tr.oauth_client.get_signed_uri(
        th_collection.to_json(), tr.get_uri(th_collection.endpoint_base), "POST"
    )

    response = TestApp(application).post_json(
        str(signed_uri), params=th_collection.get_collection_data(), status=status
    )

    return response

def post_job_data(
    project, uri, data, status=None, expect_errors=False):

    # Since the uri is passed in it's not generated by the
    # treeherder request or collection and is missing the protocol
    # and host. Add those missing elements here.
    uri = 'http://localhost{0}'.format(uri)

    # Set the credentials
    OAuthCredentials.set_credentials( SampleData.get_credentials() )

    credentials = OAuthCredentials.get_credentials(project)

    tr = TreeherderRequest(
        protocol='http',
        host='localhost',
        project=project,
        oauth_key=credentials['consumer_key'],
        oauth_secret=credentials['consumer_secret']
        )

    signed_uri = tr.get_signed_uri(
        json.dumps(data), uri
        )

    response = TestApp(application).post_json(
        str(signed_uri), params=data, status=status,
        expect_errors=expect_errors
        )

    return response

def post_collection(
        project, th_collection, status=None, expect_errors=False,
        consumer_key=None, consumer_secret=None):

    # Set the credentials
    OAuthCredentials.set_credentials(SampleData.get_credentials())

    credentials = OAuthCredentials.get_credentials(project)

    # The only time the credentials should be overridden are when
    # a client needs to test authentication failure confirmation
    if consumer_key:
        credentials['consumer_key'] = consumer_key

    if consumer_secret:
        credentials['consumer_secret'] = consumer_secret

    cli = TreeherderClient(
        protocol='http',
        host='localhost',
    )

    jsondata = th_collection.to_json()
    signed_uri = cli._get_uri(project, th_collection.endpoint_base,
                              data=jsondata,
                              oauth_key=credentials['consumer_key'],
                              oauth_secret=credentials['consumer_secret'],
                              method='POST')

    response = TestApp(application).post_json(
        str(signed_uri), params=th_collection.get_collection_data(),
        status=status
    )

    return response

def post_collection(
        project, th_collection, status=None, expect_errors=False,
        consumer_key=None, consumer_secret=None):

    # Set the credentials
    OAuthCredentials.set_credentials(SampleData.get_credentials())

    credentials = OAuthCredentials.get_credentials(project)

    # The only time the credentials should be overridden are when
    # a client needs to test authentication failure confirmation
    consumer_key = consumer_key or credentials['consumer_key']
    consumer_secret = consumer_secret or credentials['consumer_secret']

    auth = TreeherderAuth(consumer_key, consumer_secret, project)
    client = TreeherderClient(protocol='http', host='localhost', auth=auth)
    uri = client._get_project_uri(project, th_collection.endpoint_base)

    req = Request('POST', uri,
                  json=th_collection.get_collection_data(),
                  auth=auth)
    prepped_request = req.prepare()

    response = TestApp(application).post_json(
        prepped_request.url,
        params=th_collection.get_collection_data(),
        status=status
    )

    return response

    def authenticate(self, request):

        if not self.auth_detected(request):
            return None

        user = request.resolver_match.kwargs.get('project') or request.query_params['user']
        project_credentials = OAuthCredentials.get_credentials(user)

        if not project_credentials:
            raise exceptions.ValidationError(
                'project {0} has no OAuth credentials'.format(user)
            )
        parameters = OAuthCredentials.get_parameters(request.query_params)

        oauth_consumer_key = parameters['oauth_consumer_key']

        if oauth_consumer_key != project_credentials['consumer_key']:
            raise exceptions.AuthenticationFailed(
                'oauth_consumer_key does not match credentials for project {0}'.format(user)
            )

        uri = '{0}://{1}{2}'.format(
            settings.TREEHERDER_REQUEST_PROTOCOL,
            request.get_host(),
            request.path
        )
        # Construct the OAuth request based on the django request object
        json_renderer = JSONRenderer()
        req_obj = oauth.Request(
            method=request.method,
            url=uri,
            parameters=parameters,
            body=json_renderer.render(request.data),
        )
        server = oauth.Server()
        token = oauth.Token(key='', secret='')
        # Get the consumer object
        cons_obj = oauth.Consumer(
            oauth_consumer_key,
            project_credentials['consumer_secret']
        )
        # Set the signature method
        server.add_signature_method(oauth.SignatureMethod_HMAC_SHA1())

        try:
            # verify oauth django request and consumer object match
            server.verify_request(req_obj, cons_obj, token)
        except oauth.Error:
            raise exceptions.AuthenticationFailed(
                'Client authentication failed for project {0}'.format(user)
            )
        request.legacy_oauth_authenticated = True
        return (DummyUser(), None)

def test_artifact_create_text_log_summary_and_bug_suggestions(
        webapp, test_project, eleven_jobs_processed,
        mock_post_collection, mock_error_summary,
        sample_data):
    """
    test submitting text_log_summary and Bug suggestions artifacts

    This should NOT generate a Bug suggestions artifact, just post the one
    submitted.
    """

    credentials = OAuthCredentials.get_credentials(test_project)

    with JobsModel(test_project) as jobs_model:
        job = jobs_model.get_job_list(0, 1)[0]
    tls = sample_data.text_log_summary
    bs_blob = ["flim", "flam"]

    tac = client.TreeherderArtifactCollection()
    tac.add(client.TreeherderArtifact({
        'type': 'json',
        'name': 'text_log_summary',
        'blob': json.dumps(tls['blob']),
        'job_guid': job['job_guid']
    }))
    tac.add(client.TreeherderArtifact({
        'type': 'json',
        'name': 'Bug suggestions',
        'blob': json.dumps(bs_blob),
        'job_guid': job['job_guid']
    }))

    cli = client.TreeherderClient(protocol='http', host='localhost')
    credentials = OAuthCredentials.get_credentials(test_project)
    cli.post_collection(test_project, credentials['consumer_key'],
                        credentials['consumer_secret'], tac)

    with ArtifactsModel(test_project) as artifacts_model:
        artifacts = artifacts_model.get_job_artifact_list(0, 10, conditions={
            'job_id': {('=', job["id"])}
        })

    assert len(artifacts) == 2
    artifact_names = {x['name'] for x in artifacts}
    act_bs_obj = [x['blob'] for x in artifacts if x['name'] == 'Bug suggestions'][0]

    assert set(artifact_names) == {'Bug suggestions', 'text_log_summary'}
    assert bs_blob == act_bs_obj

def post_treeherder_collections(th_collections, chunk_size=1):

    errors = []
    cli = TreeherderClient(
        protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
        host=settings.TREEHERDER_REQUEST_HOST,
    )

    for project in th_collections:

        credentials = OAuthCredentials.get_credentials(project)

        auth = TreeherderAuth(credentials.get('consumer_key'),
                              credentials.get('consumer_secret'),
                              project)

        logger.info(
            "collection loading request for project {0}: {1}".format(
                project,
                th_collections[project].endpoint_base))

        collection_chunks = th_collections[project].get_chunks(chunk_size)

        for collection in collection_chunks:
            try:
                cli.post_collection(project, collection, auth=auth)
            except Exception:
                errors.append({
                    "project": project,
                    "url": th_collections[project].endpoint_base,
                    "message": traceback.format_exc()
                })

    if errors:
        raise CollectionNotLoadedException(errors)

def post_treeherder_collections(th_collections):
    errors = []
    for project in th_collections:

        credentials = OAuthCredentials.get_credentials(project)

        th_request = TreeherderRequest(
            protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
            host=settings.TREEHERDER_REQUEST_HOST,
            project=project,
            oauth_key=credentials.get('consumer_key', None),
            oauth_secret=credentials.get('consumer_secret', None)
        )

        logger.info(
            "collection loading request: {0}".format(
                th_request.get_uri(th_collections[project].endpoint_base)))
        response = th_request.post(th_collections[project])

        if not response or response.status_code != 200:
            errors.append({
                "project": project,
                "url": th_collections[project].endpoint_base,
                "message": response.text
            })

    if errors:
        raise CollectionNotLoadedException(errors)

def do_post_collection(project, collection):
    # assume if there were no exceptions we're ok
    cli = client.TreeherderClient(protocol='http', host='localhost')
    credentials = OAuthCredentials.get_credentials(project)
    auth = TreeherderAuth(credentials['consumer_key'],
                          credentials['consumer_secret'],
                          project)

    cli.post_collection(project, collection, auth=auth)

    def _send(th_request, th_collection):

        OAuthCredentials.set_credentials(SampleData.get_credentials())
        credentials = OAuthCredentials.get_credentials(jm.project)

        th_request.oauth_key = credentials['consumer_key']
        th_request.oauth_secret = credentials['consumer_secret']

        signed_uri = th_request.get_signed_uri(
            th_collection.to_json(), th_request.get_uri(th_collection)
        )

        response = TestApp(application).post_json(
            str(signed_uri), params=th_collection.get_collection_data()
        )

        response.getcode = lambda: response.status_int
        return response

def post_log_artifacts(project,
                       job_guid,
                       job_log_url,
                       retry_task,
                       extract_artifacts_cb,
                       check_errors=False):
    """Post a list of artifacts to a job."""
    def _retry(e):
        # Initially retry after 1 minute, then for each subsequent retry
        # lengthen the retry time by another minute.
        retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60)
        # .retry() raises a RetryTaskError exception,
        # so nothing after this function will be executed

    log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
    logger.debug("Downloading/parsing log for %s", log_description)

    credentials = OAuthCredentials.get_credentials(project)
    req = TreeherderRequest(
        protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
        host=settings.TREEHERDER_REQUEST_HOST,
        project=project,
        oauth_key=credentials.get('consumer_key', None),
        oauth_secret=credentials.get('consumer_secret', None),
    )

    try:
        artifact_list = extract_artifacts_cb(job_log_url['url'],
                                             job_guid, check_errors)
    except Exception as e:
        update_parse_status(req, job_log_url, 'failed')
        if isinstance(e, urllib2.HTTPError) and e.code == 404:
            logger.debug("Log not found for %s", log_description)
            return
        logger.error("Failed to download/parse log for %s: %s", log_description, e)
        _retry(e)

    # store the artifacts generated
    tac = TreeherderArtifactCollection()
    for artifact in artifact_list:
        ta = tac.get_artifact({
            "job_guid": artifact[0],
            "name": artifact[1],
            "type": artifact[2],
            "blob": artifact[3]
        })
        tac.add(ta)

    try:
        req.post(tac)
        update_parse_status(req, job_log_url, 'parsed')
        logger.debug("Finished posting artifact for %s %s", project, job_guid)
    except Exception as e:
        logger.error("Failed to upload parsed artifact for %s: %s", log_description, e)
        _retry(e)

def post_log_artifacts(project,
                       job_guid,
                       job_log_url,
                       retry_task,
                       extract_artifacts_cb,
                       check_errors=False):
    """Post a list of artifacts to a job."""
    def _retry(e):
        # Initially retry after 1 minute, then for each subsequent retry
        # lengthen the retry time by another minute.
        retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60)
        # .retry() raises a RetryTaskError exception,
        # so nothing after this function will be executed

    log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
    logger.debug("Downloading/parsing log for %s", log_description)

    credentials = OAuthCredentials.get_credentials(project)
    client = TreeherderClient(
        protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
        host=settings.TREEHERDER_REQUEST_HOST
    )

    try:
        artifact_list = extract_artifacts_cb(job_log_url['url'],
                                             job_guid, check_errors)
    except Exception as e:
        client.update_parse_status(project, credentials.get('consumer_key'),
                                   credentials.get('consumer_secret'),
                                   job_log_url['id'], 'failed')
        if isinstance(e, urllib2.HTTPError) and e.code in (403, 404):
            logger.debug("Unable to retrieve log for %s: %s", log_description, e)
            return
        logger.error("Failed to download/parse log for %s: %s", log_description, e)
        _retry(e)

    # store the artifacts generated
    tac = TreeherderArtifactCollection()
    for artifact in artifact_list:
        ta = tac.get_artifact(artifact)
        tac.add(ta)

    try:
        client.post_collection(project, credentials.get('consumer_key'),
                               credentials.get('consumer_secret'),
                               tac)
        client.update_parse_status(project, credentials.get('consumer_key'),
                                   credentials.get('consumer_secret'),
                                   job_log_url['id'], 'parsed')
        logger.debug("Finished posting artifact for %s %s", project, job_guid)
    except Exception as e:
        logger.error("Failed to upload parsed artifact for %s: %s", log_description, e)
        _retry(e)

    def _send(th_request, endpoint,  method=None, data=None):

        OAuthCredentials.set_credentials(SampleData.get_credentials())
        credentials = OAuthCredentials.get_credentials(jm.project)

        th_request.oauth_key = credentials['consumer_key']
        th_request.oauth_secret = credentials['consumer_secret']

        if data and not isinstance(data, str):
            data = json.dumps(data)

        signed_uri = th_request.oauth_client.get_signed_uri(
            data, th_request.get_uri(endpoint), method
        )

        response = getattr(TestApp(application), method.lower())(
            str(signed_uri),
            params=data,
            content_type='application/json'
        )

        response.getcode = lambda: response.status_int
        return response

def test_artifact_create_text_log_summary(webapp, test_project, eleven_jobs_processed,
                                          mock_post_collection, mock_error_summary,
                                          sample_data):
    """
    test submitting a text_log_summary artifact which auto-generates bug suggestions
    """

    credentials = OAuthCredentials.get_credentials(test_project)

    with JobsModel(test_project) as jobs_model:
        job = jobs_model.get_job_list(0, 1)[0]
    tls = sample_data.text_log_summary

    tac = client.TreeherderArtifactCollection()
    ta = client.TreeherderArtifact({
        'type': 'json',
        'name': 'text_log_summary',
        'blob': json.dumps(tls['blob']),
        'job_guid': job['job_guid']
    })
    tac.add(ta)

    cli = client.TreeherderClient(protocol='http', host='localhost')
    credentials = OAuthCredentials.get_credentials(test_project)
    cli.post_collection(test_project, credentials['consumer_key'],
                        credentials['consumer_secret'], tac)

    with ArtifactsModel(test_project) as artifacts_model:
        artifacts = artifacts_model.get_job_artifact_list(0, 10, conditions={
            'job_id': {('=', job["id"])}
        })

    artifact_names = {x['name'] for x in artifacts}
    act_bs_obj = [x['blob'] for x in artifacts if x['name'] == 'Bug suggestions'][0]

    assert set(artifact_names) == {'Bug suggestions', 'text_log_summary'}
    assert mock_error_summary == act_bs_obj

def test_post_job_with_parsed_log(test_project, result_set_stored,
                                  mock_send_request):
    """
    test submitting a job with a pre-parsed log gets the right job_log_url
    parse_status value.
    """

    credentials = OAuthCredentials.get_credentials(test_project)

    tjc = client.TreeherderJobCollection()
    tj = client.TreeherderJob({
        'project': test_project,
        'revision_hash': result_set_stored[0]['revision_hash'],
        'job': {
            'job_guid': 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33',
            'state': 'completed',
            'log_references': [{
                'url': 'http://ftp.mozilla.org/pub/mozilla.org/spidermonkey/...',
                'name': 'builbot_text',
                'parse_status': 'parsed'
            }]
        }
    })

    tjc.add(tj)

    req = client.TreeherderRequest(
        protocol='http',
        host='localhost',
        project=test_project,
        oauth_key=credentials['consumer_key'],
        oauth_secret=credentials['consumer_secret']
        )

    # Post the request to treeherder
    resp = req.post(tjc)
    assert resp.status_int == 200
    assert resp.body == '{"message": "well-formed JSON stored"}'

    with JobsModel(test_project) as jm:
        jm.process_objects(10)

        job_ids = [x['id'] for x in jm.get_job_list(0, 20)]
        job_log_list = jm.get_job_log_url_list(job_ids)

        assert len(job_log_list) == 1
        assert job_log_list[0]['parse_status'] == 'parsed'

    def load(self, th_collections):

        for project in th_collections:

            credentials = OAuthCredentials.get_credentials(project)

            th_request = TreeherderRequest(
                protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
                host=settings.TREEHERDER_REQUEST_HOST,
                project=project,
                oauth_key=credentials.get('consumer_key', None),
                oauth_secret=credentials.get('consumer_secret', None)
            )

            response = th_request.post(th_collections[project])

            if not response or response.status != 200:
                message = response.read()
                logger.error("collection loading failed: {0}".format(message))

def test_two_legged_oauth_project_via_user(monkeypatch, jm, set_oauth_credentials):
    view = AuthenticatedView.as_view()
    credentials = OAuthCredentials.get_credentials(jm.project)

    test_url = '/api/'
    request = factory.get(test_url, {
        'oauth_body_hash': '',
        'oauth_consumer_key': credentials['consumer_key'],
        'oauth_nonce': '',
        'oauth_signature_method': '',
        'oauth_timestamp': '',
        'oauth_token': '',
        'oauth_version': '',
        'oauth_signature': '',
        'user': jm.project
    })
    monkeypatch.setattr(oauth.Server, 'verify_request', lambda *x, **y: True)
    request.resolver_match = resolve(test_url)
    response = view(request)

    assert response.data == {'authenticated': True}

    def load(self, th_collections):
        for project in th_collections:

            credentials = OAuthCredentials.get_credentials(project)

            th_request = TreeherderRequest(
                protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
                host=settings.TREEHERDER_REQUEST_HOST,
                project=project,
                oauth_key=credentials.get('consumer_key', None),
                oauth_secret=credentials.get('consumer_secret', None)
            )

            logger.info(
                "collection loading request: {0}".format(
                    th_request.get_uri(th_collections[project].endpoint_base)))
            response = th_request.post(th_collections[project])

            if not response or response.status != 200:
                message = response.read()
                logger.error('[{0}]Error posting data to {1} : {2}'.format(
                    project, th_collections[project].endpoint_base, message))

def set_oauth_credentials():
    OAuthCredentials.set_credentials(SampleData.get_credentials())