def get_ssh_user(self, args):
     url = args.remote_url
     parse_result = urlparse(url)
     if not parse_result.scheme:
         url = 'ssh://' + url
         parse_result = urlparse(url)
     return parse_result.username if parse_result.username else ""

    def assertRedirectsNoFollow(self, response, expected_url, use_params=True,
                                status_code=302):
        """Checks response redirect without loading the destination page.

        Django's assertRedirects method loads the destination page, which
        requires that the page be renderable in the current test context
        (possibly requiring additional, unrelated setup).
        """
        # Assert that the response has the correct redirect code.
        self.assertEqual(
            response.status_code, status_code,
            "Response didn't redirect as expected: Response code was {0} "
            "(expected {1})".format(response.status_code, status_code))

        # Assert that the response redirects to the correct base URL.
        # Use force_text to force evaluation of anything created by
        # reverse_lazy.
        response_url = force_text(response['location'])
        expected_url = force_text(expected_url)
        parsed1 = urlparse(response_url)
        parsed2 = urlparse(expected_url)
        self.assertEquals(
            parsed1.path, parsed2.path,
            "Response did not redirect to the expected URL: Redirect "
            "location was {0} (expected {1})".format(parsed1.path, parsed2.path))

        # Optionally assert that the response redirect URL has the correct
        # GET parameters.
        if use_params:
            self.assertDictEqual(
                parse_qs(parsed1.query), parse_qs(parsed2.query),
                "Response did not have the GET parameters expected: GET "
                "parameters were {0} (expected "
                "{1})".format(parsed1.query or {}, parsed2.query or {}))

    def _list_buckets_non_empty_helper(self, project, use_default=False):
        from six.moves.urllib.parse import parse_qs
        from six.moves.urllib.parse import urlencode
        from six.moves.urllib.parse import urlparse
        from gcloud._testing import _monkey_defaults as _base_monkey_defaults
        from gcloud.storage._testing import _monkey_defaults
        from gcloud.storage.connection import Connection
        BUCKET_NAME = 'bucket-name'
        conn = Connection()
        query_params = urlencode({'project': project, 'projection': 'noAcl'})
        BASE_URI = '/'.join([
            conn.API_BASE_URL,
            'storage',
            conn.API_VERSION,
        ])
        URI = '/'.join([BASE_URI, 'b?%s' % (query_params,)])
        http = conn._http = Http(
            {'status': '200', 'content-type': 'application/json'},
            '{{"items": [{{"name": "{0}"}}]}}'.format(BUCKET_NAME)
            .encode('utf-8'),
        )

        if use_default:
            with _base_monkey_defaults(project=project):
                with _monkey_defaults(connection=conn):
                    buckets = list(self._callFUT())
        else:
            buckets = list(self._callFUT(project=project, connection=conn))

        self.assertEqual(len(buckets), 1)
        self.assertEqual(buckets[0].name, BUCKET_NAME)
        self.assertEqual(http._called_with['method'], 'GET')
        self.assertTrue(http._called_with['uri'].startswith(BASE_URI))
        self.assertEqual(parse_qs(urlparse(http._called_with['uri']).query),
                         parse_qs(urlparse(URI).query))

def relative_uri(source, target):
    """
    Make a relative URI from source to target.
    """
    su = urlparse.urlparse(source)
    tu = urlparse.urlparse(target)
    extra = list(tu[3:])
    relative = None
    if tu[0] == '' and tu[1] == '':
        if tu[2] == su[2]:
            relative = ''
        elif not tu[2].startswith('/'):
            relative = tu[2]
    elif su[0:2] != tu[0:2]:
        return target

    if relative is None:
        if tu[2] == su[2]:
            relative = ''
        else:
            relative = os.path.relpath(tu[2], os.path.dirname(su[2]))
    if relative == '.':
        relative = ''
    relative = urlparse.urlunparse(["", "", relative] + extra)
    return relative

    def umount_check(self, hosts):
        """
        Check for and print unmounted drives

        :param hosts: set of hosts to check. in the format of:
            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
        """
        unmounted = {}
        errors = {}
        recon = Scout("unmounted", self.verbose, self.suppress_errors,
                      self.timeout)
        print("[%s] Getting unmounted drives from %s hosts..." %
              (self._ptime(), len(hosts)))
        for url, response, status, ts_start, ts_end in self.pool.imap(
                recon.scout, hosts):
            if status == 200:
                unmounted[url] = []
                errors[url] = []
                for i in response:
                    if not isinstance(i['mounted'], bool):
                        errors[url].append(i['device'])
                    else:
                        unmounted[url].append(i['device'])
        for host in unmounted:
            node = urlparse(host).netloc
            for entry in unmounted[host]:
                print("Not mounted: %s on %s" % (entry, node))
        for host in errors:
            node = urlparse(host).netloc
            for entry in errors[host]:
                print("Device errors: %s on %s" % (entry, node))
        print("=" * 79)

def test_add_handle_by_names(hurl, hpath, cpath, lcpath):

    class mocked_dirs:
        user_data_dir = lcpath

    with patch('datalad.cmdline.helpers.dirs', mocked_dirs), \
            swallow_logs() as cml:

        # get testrepos and make them known to datalad:
        handle = install_handle(hurl, hpath)
        collection = register_collection(cpath)
        assert_not_in(handle.name, collection)

        return_value = add_handle(handle.name, collection.name)

        # now handle is listed by collection:
        collection._reload()
        assert_in(handle.name, collection)

        # test collection repo:
        ok_clean_git(cpath, annex=False)
        ok_(isdir(opj(cpath, handle.name)))
        ok_(exists(opj(cpath, handle.name, REPO_CONFIG_FILE)))
        ok_(exists(opj(cpath, handle.name, REPO_STD_META_FILE)))

        # evaluate return value:
        assert_is_instance(return_value, Handle,
                           "install_handle() returns object of "
                           "incorrect class: %s" % type(return_value))
        eq_(return_value.name, handle.name)
        eq_(urlparse(return_value.url).path, urlparse(handle.url).path)

    def create_plugin(self, session, version, url, raw_status=None):
        """Handle default Keystone endpoint configuration

        Build the actual API endpoint from the scheme, host and port of the
        original auth URL and the rest from the returned version URL.
        """

        ver_u = urlparse.urlparse(url)

        # Only hack this if it is the default setting
        if ver_u.netloc.startswith('localhost'):
            auth_u = urlparse.urlparse(self.auth_url)
            # from original auth_url: scheme, netloc
            # from api_url: path, query (basically, the rest)
            url = urlparse.urlunparse((
                auth_u.scheme,
                auth_u.netloc,
                ver_u.path,
                ver_u.params,
                ver_u.query,
                ver_u.fragment,
            ))
            LOG.debug('Version URL updated: %s' % url)

        return super(OSCGenericPassword, self).create_plugin(
            session=session,
            version=version,
            url=url,
            raw_status=raw_status,
        )

    def jenkins_job_building(self, jobName):
        server = self.jenkins_server()

        nodes = server.get_nodes()
        for node in nodes:
            print('nodes')
            # the name returned is not the name to lookup when
            # dealing with master :/
            if node['name'] == 'master':
                node_name = '(master)'
            else:
                node_name = node['name']
            try:
                info = server.get_node_info(node_name, depth=2)
            except:
                print('except')
                # # Jenkins may 500 on depth >0. If the node info comes back
                # # at depth 0 treat it as a node not running any jobs.
                # if ('[500]' in str(e) and
                #         self.get_node_info(node_name, depth=0)):
                #     continue
                # else:
                #     raise
            for executor in info['executors']:
                executable = executor['currentExecutable']
                if executable:
                    executor_number = executor['number']
                    build_number = executable['number']
                    url = executable['url']
                    print(urlparse(url).path)
                    m = re.match(r'.*?/job/([^/]+)/.*', urlparse(url).path)
                    job_name = m.group(1)
                    if job_name == jobName:
                        return True
        return False

    def build_integration(self, state):
        data = state['identity']['data']
        oauth_data = self.get_oauth_data(data)
        user = get_user_info(data['access_token'], state['installation_data'])
        group = self.get_group_info(data['access_token'], state['installation_data'])
        scopes = sorted(GitlabIdentityProvider.oauth_scopes)
        base_url = state['installation_data']['url']

        integration = {
            'name': group['name'],
            'external_id': u'{}:{}'.format(urlparse(base_url).netloc, group['id']),
            'metadata': {
                'icon': group['avatar_url'],
                'domain_name': group['web_url'].replace('https://', ''),
                'scopes': scopes,
                'verify_ssl': state['installation_data']['verify_ssl'],
                'base_url': base_url,
            },
            'user_identity': {
                'type': 'gitlab',
                'external_id': u'{}:{}'.format(urlparse(base_url).netloc, user['id']),
                'scopes': scopes,
                'data': oauth_data,
            },
        }

        return integration

def is_embedded_in_request(url, value):
    """Whether or not to encode the value as raw data content.

    Returns True if
      - value is a file:/// URI or a local path
      - value is a File-like instance
      - url is not localhost
      - value is a File object
      - value is already the string content
    """
    if hasattr(value, 'read'):  # File-like
        return True

    u = urlparse(url)

    if isinstance(value, Path):  # pathlib.Path
        p = value
        scheme = 'file'
    else:  # String-like
        v = urlparse(value)
        p = Path(v.path)
        scheme = v.scheme

    if scheme == 'file':  # Explicit link to file
        if is_file(p):
            return 'localhost' not in u.netloc
        else:
            raise IOError("{} should be a local file but was not found on disk.".format(value))
    elif scheme == '':  # Could be a local path or just a string
        if is_file(p):
            return 'localhost' not in u.netloc
        else:
            return True
    else:  # Other URL (http, https, ftp, ...)
        return False

 def test_redirect_url_takes_custom_url_parameters(self):
     redirect_url = urlparse(self.client.get_redirect_url(
         state="applicationstate",
         redirect_uri="https://my.site/oauth",
         scope="profile profile:email",
         action="signup",
         email="[email protected]",
         code_challenge="challenge",
         code_challenge_method="S1234",
         access_type="offline",
         keys_jwk="MockJWK",
     ))
     server_url = urlparse(self.server_url)
     self.assertEqual(redirect_url.hostname, server_url.hostname)
     params = parse_qs(redirect_url.query, keep_blank_values=True)
     all_params = ["action", "email", "client_id", "redirect_uri",
                   "scope", "state", "access_type", "code_challenge",
                   "code_challenge_method", "keys_jwk"]
     self.assertEqual(sorted(params.keys()), sorted(all_params))
     self.assertEqual(params["client_id"][0], self.client.client_id)
     self.assertEqual(params["state"][0], "applicationstate")
     self.assertEqual(params["redirect_uri"][0], "https://my.site/oauth")
     self.assertEqual(params["scope"][0], "profile profile:email")
     self.assertEqual(params["action"][0], "signup")
     self.assertEqual(params["email"][0], "[email protected]")
     self.assertEqual(params["code_challenge"][0], "challenge")
     self.assertEqual(params["code_challenge_method"][0], "S1234")
     self.assertEqual(params["access_type"][0], "offline")
     self.assertEqual(params["keys_jwk"][0], "MockJWK")

    def test_create_and_read_and_delete(self):
        expected = self.test_create_file().data
        url = urlparse(expected["url"])
        response = self.client.get(url.path)
        contents = response.content.decode("utf-8")
        self.assertEquals(self.file_contents, contents)

        response = self.client.get("/rp/jobs/files/", format="json")
        self.assertEquals(1, len(response.data))
        data = response.data[0]
        self.assertEquals("File contents not shown.", data["contents"])
        self.assertEquals(expected["file"], data["file"])
        self.assertEquals(expected["filename"], data["filename"])
        self.assertEquals(self.get_hashed_contents(), data["hexhash"])
        self.assertEquals(url.path, urlparse(data["url"]).path)

        response = self.client.delete(url.path)
        self.assertEqual(204, response.status_code)

        response = self.client.get(url.path)
        self.assertEqual(404, response.status_code)

        response = self.client.get("/rp/jobs/files/", format="json")
        data = response.data
        self.assertEquals(0, len(data))

 def _wrapped(self):
     # If we are ignoring the request on the form, we should also ignore it
     # on the widget.  This means that when on the first widget we conclude
     # that the form should be ignored, we quickly ignore it on all widgets,
     # without needing to check the referer and method again and again.
     # When we do not ignore the request, we do still run these checks for
     # all widgets.  But it seems an international sport to override the
     # update or updateWidgets method of the base z3c form, which makes it
     # hard to fix all occurrences by one check on the form.
     if not self.ignoreRequest and getattr(self.form, 'ignoreRequest', False):
         self.ignoreRequest = True
     # If we are not already ignoring the request, check the request method.
     if (not self.ignoreRequest
             and hasattr(self.form, 'method')
             and hasattr(self.request, 'REQUEST_METHOD')):
         if self.request.REQUEST_METHOD.lower() != self.form.method.lower():
             # This is an unexpected request method.
             # For special cases we allow a form to bail out.
             if not getattr(self.form, ALLOW_PREFILL, False):
                 self.ignoreRequest = True
                 self.form.ignoreRequest = True
     # If we are not already ignoring the request, check the referer.
     if not self.ignoreRequest and hasattr(self.request, 'environ'):
         env = self.request.environ
         referrer = env.get('HTTP_REFERER', env.get('HTTP_REFERRER'))
         if referrer:
             req_url_parsed = urlparse(self.request.URL)
             referrer_parsed = urlparse(referrer)
             if req_url_parsed.netloc != referrer_parsed.netloc:
                 # We do not trust data from outside referrers.
                 self.ignoreRequest = True
                 self.form.ignoreRequest = True
     return update(self)

    def test_cross_repo_mount(self):
        target_url = urlparse('docker://localhost:8787/t/nova-api:latest')
        other_url = urlparse('docker://localhost:8787/t/nova-compute:latest')
        image_layers = {
            'sha256:1234': other_url
        }
        source_layers = [
            'sha256:1234', 'sha256:6789'
        ]
        source_blob_dir = os.path.join(image_export.IMAGE_EXPORT_DIR,
                                       'v2/t/nova-compute/blobs')
        source_blob_path = os.path.join(source_blob_dir, 'sha256:1234.gz')
        target_blob_dir = os.path.join(image_export.IMAGE_EXPORT_DIR,
                                       'v2/t/nova-api/blobs')
        target_blob_path = os.path.join(target_blob_dir, 'sha256:1234.gz')

        # call with missing source, no change
        image_export.cross_repo_mount(target_url, image_layers, source_layers)
        self.assertFalse(os.path.exists(source_blob_path))
        self.assertFalse(os.path.exists(target_blob_path))

        image_export.make_dir(source_blob_dir)
        with open(source_blob_path, 'w') as f:
            f.write('blob')
        self.assertTrue(os.path.exists(source_blob_path))

        # call with existing source
        image_export.cross_repo_mount(target_url, image_layers, source_layers)
        self.assertTrue(os.path.exists(target_blob_path))
        with open(target_blob_path, 'r') as f:
            self.assertEqual('blob', f.read())

    def test_previous_solution_incorrect(self):
        client = RecaptchaClient(_FAKE_PRIVATE_KEY, _FAKE_PUBLIC_KEY)
        urls = client._get_challenge_urls(
            was_previous_solution_incorrect=True,
            use_ssl=False,
        )

        javascript_challenge_url = urls['javascript_challenge_url']
        javascript_challenge_url_components = \
            urlparse(javascript_challenge_url)
        javascript_challenge_url_query = parse_qs(
            javascript_challenge_url_components.query,
        )

        self.assertIn('error', javascript_challenge_url_query)
        self.assertEqual(
            'incorrect-captcha-sol',
            javascript_challenge_url_query['error'][0]
        )

        noscript_challenge_url = urls['noscript_challenge_url']
        noscript_challenge_url_components = urlparse(noscript_challenge_url)
        self.assertEqual(
            javascript_challenge_url_components.query,
            noscript_challenge_url_components.query
        )

def submit_git_build(binstar, args):

    log.info("Submitting the following repo for package creation: %s" % args.git_url)
    builds = get_gitrepo(urlparse(args.git_url))

    if not args.package:
        user = binstar.user()
        user_name = user['login']
        package_name = builds['repo'].split('/')[1]
        log.info("Using repo name '%s' as the pkg name." % package_name)
        args.package = PackageSpec(user_name, package_name)

    try:
        _ = binstar.package(args.package.user, args.package.name)
    except errors.NotFound:
        raise errors.UserError("Package %s does not exist" % (args.package,))


    if not args.dry_run:
        log.info("Submitting the following repo for package creation: %s" % args.git_url)
        builds = get_gitrepo(urlparse(args.path))
        build = binstar.submit_for_url_build(args.package.user, args.package.name, builds,
                                             channels=args.channels, queue=args.queue, sub_dir=args.sub_dir,
                                             test_only=args.test_only, callback=upload_print_callback(args),
                                             filter_platform=args.platform,
                                                )

        print_build_results(args, build)

    else:
        log.info('Build not submitted (dry-run)')

    def test_list_buckets_non_empty(self):
        from six.moves.urllib.parse import parse_qs
        from six.moves.urllib.parse import urlencode
        from six.moves.urllib.parse import urlparse
        PROJECT = 'PROJECT'
        CREDENTIALS = _Credentials()
        client = self._makeOne(project=PROJECT, credentials=CREDENTIALS)

        BUCKET_NAME = 'bucket-name'
        query_params = urlencode({'project': PROJECT, 'projection': 'noAcl'})
        BASE_URI = '/'.join([
            client.connection.API_BASE_URL,
            'storage',
            client.connection.API_VERSION,
        ])
        URI = '/'.join([BASE_URI, 'b?%s' % (query_params,)])
        http = client.connection._http = _Http(
            {'status': '200', 'content-type': 'application/json'},
            '{{"items": [{{"name": "{0}"}}]}}'.format(BUCKET_NAME)
            .encode('utf-8'),
        )
        buckets = list(client.list_buckets())
        self.assertEqual(len(buckets), 1)
        self.assertEqual(buckets[0].name, BUCKET_NAME)
        self.assertEqual(http._called_with['method'], 'GET')
        self.assertTrue(http._called_with['uri'].startswith(BASE_URI))
        self.assertEqual(parse_qs(urlparse(http._called_with['uri']).query),
                         parse_qs(urlparse(URI).query))

def get_repo_information(repo_file_raw_content):
    base_url = re.search('baseurl=((.*)/os)', repo_file_raw_content).group(1)
    base_url = base_url.replace("$basearch", "x86_64")
    version = urlparse(base_url).path.split('/')[5]
    repo_name = urlparse(base_url).path.split('/')[4]

    return base_url, version, repo_name

def pull_arbitrary(collector, image, **kwargs):
    """Pull an arbitrary image"""
    image_index_of = lambda image: urlparse("https://{0}".format(image)).netloc

    if image.startswith("file://"):
        parsed = urlparse(image)
        filename = parsed.netloc + parsed.path
        if not os.path.exists(filename):
            raise HarpoonError("Provided file doesn't exist!", wanted=image)
        with open(filename) as fle:
            image_indexes = [(line.strip(), image_index_of(line.strip())) for line in fle]
    else:
        image_indexes = [(image, image_index_of(image))]

    authentication = collector.configuration.get("authentication", NotSpecified)
    for index, (image, image_index) in enumerate(image_indexes):
        tag = sb.NotSpecified
        if ":" in image:
            image, tag = image.split(":", 1)

        image = {
              "image_name": image
            , "tag": tag
            , "harpoon": collector.configuration["harpoon"]
            , "commands": ["FROM scratch"]
            , "image_index": image_index
            , "assume_role": NotSpecified
            , "authentication": authentication
            }
        meta = Meta(collector.configuration, []).at("images").at("__arbitrary_{0}__".format(index))
        image = HarpoonSpec().image_spec.normalise(meta, image)
        Syncer().pull(image)

    def test_collection_ok_by_state(
            self, f_users, f_coprs,
            f_mock_chroots_many,
            f_build_many_chroots,
            f_db,
            f_users_api):

        self.db.session.commit()
        for status in StatusEnum.vals.values():
            expected_chroots = set([
                name
                for name, chroot_status in
                self.status_by_chroot.items()
                if chroot_status == status
            ])

            href = "/api_2/build_tasks?state={}&limit=50".format(StatusEnum(status))

            r0 = self.tc.get(href)
            assert r0.status_code == 200
            obj = json.loads(r0.data.decode("utf-8"))
            assert len(obj["build_tasks"]) == len(expected_chroots)
            assert set(bt["build_task"]["chroot_name"]
                       for bt in obj["build_tasks"]) == expected_chroots

            assert parse_qs(urlparse(obj["_links"]["self"]["href"]).query) \
                == parse_qs(urlparse(href).query)