def is_internal_url(url, reference):
    '''checks whether the given url is an internal url
    to the reference site'''
    refobj = utils.parse_url(reference)
    urlobj = utils.parse_url(url)
    if not url.startswith('http'):
        raise InvalidUrlError
    return urlobj.origin == refobj.origin

def absolute_to_relative(url, reference):
    '''converts absolute urls to relative urls based on the
    reference url'''
    if not url.startswith('http'):
        raise InvalidUrlError
    refobj = utils.parse_url(reference)
    urlobj = utils.parse_url(url)
    if urlobj.origin == refobj.origin:
        return urlobj.path

 def _patch_uris(self, uris, handler="/XMLRPC"):
     #Handles patching the uris when they're in a list.
     if type(uris) == type([]):
         ret = []
         for i in range(len(uris)):
             t = list(utils.parse_url(uris[i]))
             t[2] = handler
             ret.append(utils.unparse_url(t))
     #Handles patching the uri when it's a string.
     else:
         t = list(utils.parse_url(uris))
         t[2] = handler
         ret = utils.unparse_url(t)
     return ret

 def _post(self, url, data=None, headers={}):
     host, port, uri, is_ssl = parse_url(url)
     conn = HTTPSConnection(host, port, timeout=self.timeout)
     headers.update({'Content-type': 'application/x-www-form-urlencoded'})
     conn.request('POST', '/' + uri, urlencode(data) if data else None, headers)
     response = conn.getresponse()
     return conn, response

    def _extract_url_parameters(self, url):
        '''
        strip out the osdd url parameters

        note: not always emitted correctly as param={thing?}. could also be param=thing
              except the param=thing is probably a hardcoded term SO HOW DO WE MANAGE THAT?
              TODO: manage that (ex: ?product=MOD021QA&collection={mp:collection?})

        tuple: (parameter name, namespace(s), param namespace prefix, param type, format)
        '''
        assert url, 'No URL'

        query_params = parse_url(url)
        # deal with the namespaced parameters as [query param key, prefix, type]
        query_params = [[k] + list(self._extract_parameter_type(v)) for k, v
                        in query_params.iteritems()]

        return [
            tidy_dict({
                "name": qp[0],
                "namespaces": self.parser._namespaces,
                "prefix": qp[1],
                "type": qp[2],
                "format": self._parameter_formats.get(':'.join(qp[1:]))
            })
            for qp in query_params
        ]

    def save_page(self, url, html):
        '''saves the dom for the path'''
        try:
            urlobj = utils.parse_url(url)
            hsh = hashlib.sha1(html.encode('utf-8')).hexdigest()

            current = self.db.get_one('''
            select page_id, page_sha1 from page
            where
            site_hostname = %s and
            page_path = %s''', (urlobj.origin, urlobj.path))

            expires = datetime.datetime.now() + datetime.timedelta(0, default_expiry_time) #secs

            logging.info('Saving page: %s', url)
            if not current:
                # TODO paths should always be present beforehand
                self.db.put('''
                insert into page
                (site_hostname, page_path, page_content, page_sha1,
                page_expires, page_expiresevery, page_attempts) values
                (%s, %s, %s, %s, %s, %s, %s)
                ''', (urlobj.origin, urlobj.path, html, hsh, expires, default_expiry_time, 0))
            else:
                pageid = current[0]
                self.db.put('''
                update page
                set
                site_hostname = %s, page_content = %s, page_sha1 = %s,
                page_expires = %s, page_expiresevery = %s, page_attempts = %s
                where
                page_id = %s
                ''', (urlobj.origin, html, hsh, expires, default_expiry_time, pageid, 0))
        except Exception, e:
            logging.error('Could not save page', exc_info=True)

 def cdn_connect(self):
     """
     Setup the http connection instance for the CDN service.
     """
     (host, port, cdn_uri, is_ssl) = parse_url(self.cdn_url)
     self.cdn_connection = self.conn_class(host, port, timeout=self.timeout)
     self.cdn_enabled = True

 def insert(self, bl_type, url):
     dn, path, qp = utils.parse_url(url)
     dn, path, qp = self.generate_query_parameters(dn, path, qp)
     try:
         self.blacklists[bl_type][dn][path].add(qp)
     except KeyError:
         return

 def _get(self, url, params=None, headers={}):
     host, port, uri, is_ssl = parse_url(url)
     conn = HTTPSConnection(host, port, timeout=self.timeout)
     conn.request('GET', '/' + uri + ('?'+urlencode(params) if params else ''),
                  headers=headers)
     response = conn.getresponse()
     return conn, response

 def cdn_connect(self):
     """
     Setup the http connection instance for the CDN service.
     """
     (host, port, cdn_uri, is_ssl) = parse_url(self.cdn_url)
     conn_class = is_ssl and HTTPSConnection or HTTPConnection
     self.cdn_connection = conn_class(host, port)
     self.cdn_enabled = True

 def __init__(self, username, api_key, authurl=default_authurl):
     self.authurl = authurl
     self.headers = dict()
     self.headers['x-auth-user'] = username
     self.headers['x-auth-key'] = api_key
     self.headers['User-Agent'] = user_agent
     (self.host, self.port, self.uri, self.is_ssl) = parse_url(self.authurl)
     self.conn_class = self.is_ssl and HTTPSConnection or HTTPConnection

def capture(request):
    # Merge both QueryDict into dict
    parameters = dict([(k, v) for k, v in request.GET.items()])
    parameters.update(dict([(k, v) for k, v in request.POST.items()]))

    url = parameters.get('url')
    if not url:
        return HttpResponseBadRequest(_('Missing url parameter'))
    try:
        url = parse_url(request, url)
    except NoReverseMatch:
        error_msg = _("URL '%s' invalid (could not reverse)") % url
        return HttpResponseBadRequest(error_msg)

    method = parameters.get('method', request.method)
    selector = parameters.get('selector')
    data = parameters.get('data')
    waitfor = parameters.get('waitfor')
    wait = parameters.get('wait')
    render = parameters.get('render', 'png')
    size = parameters.get('size')
    crop = parameters.get('crop')

    try:
        width = int(parameters.get('width', ''))
    except ValueError:
        width = None
    try:
        height = int(parameters.get('height', ''))
    except ValueError:
        height = None

    stream = StringIO()
    try:
        casperjs_capture(stream, url, method=method.lower(), width=width,
                         height=height, selector=selector, data=data,
                         size=size, waitfor=waitfor, crop=crop, render=render,
                         wait=wait)
    except CaptureError as e:
        return HttpResponseBadRequest(e)
    except ImportError:
        error_msg = _('Resize not supported (PIL not available)')
        return HttpResponseBadRequest(error_msg)
    except UnsupportedImageFormat:
        error_msg = _('Unsupported image format: %s' % render)
        return HttpResponseBadRequest(error_msg)

    if render == "html":
        response = HttpResponse(mimetype='text/html')
        body = """<html><body onload="window.print();">
                <img src="data:image/png;base64,%s"/></body></html>
                """ % base64.encodestring(stream.getvalue())
        response.write(body)
    else:
        response = HttpResponse(mimetype=image_mimetype(render))
        response.write(stream.getvalue())

    return response

 def __init__(self, repo, ref=None, token=None):
     """Initialize repo."""
     if repo.count('/') > 1:
         _host, _user, _repo = parse_url(repo)
         self.repo = '{}/{}'.format(_user, _repo)
     else:
         self.repo = repo
     self.token = token
     self.ref = ref

 def insert(self, payload, url):
     """
     TODO, In order to allow search of query parameters in any order,
     we may be able to use sets. This needs to be investigated.
     For now, we'll sort the query arguments. Yuck!
     """
     dn, path, qp = utils.parse_url(url)
     domain, path_parameters = self.generate_query_parameters(dn, path, qp)
     return self._session.execute(self._insert_query.bind((domain, path_parameters, payload)))

 def __init__(self, repo, ref=None, token=None):
     """Initialize repo."""
     if repo.count('/') > 1:
         _host, _user, _repo = parse_url(repo)
         self.repo = '{}/{}'.format(_user, _repo)
     else:
         self.repo = repo
     self.token = token
     self.host = "https://gitlab.cern.ch"
     self.ref = ref  # branch/tag/commit

 def delete_page(self, url):
     try:
         urlobj = utils.parse_url(url)
         self.db.put('''
         delete from page
         where page_path = %s
         and
         site_hostname = %s''', (urlobj.path, urlobj.origin))
     except Exception, e:
         logging.error('Could not delete page', exc_info=True)

 def __init__(self, username, api_key, authurl=us_authurl, timeout=15,
              useragent=user_agent):
     self.authurl = authurl
     self.headers = dict()
     self.headers['x-auth-user'] = username
     self.headers['x-auth-key'] = api_key
     self.headers['User-Agent'] = useragent
     self.timeout = timeout
     (self.host, self.port, self.uri, self.is_ssl) = parse_url(self.authurl)
     self.conn_class = get_conn_class(self.is_ssl)

    def cdn_connect(self):
        """
        Setup the http connection instance for the CDN service.
        """
        (host, port, cdn_uri, is_ssl) = parse_url(self.cdn_url)
	if version_info[0] <= 2 and version_info[1] < 6:
            self.conn_class = is_ssl and THTTPSConnection or THTTPConnection
        else:
            self.conn_class = is_ssl and HTTPSConnection or HTTPConnection
        self.cdn_connection = self.conn_class(host, port, timeout=self.timeout)
        self.cdn_enabled = True

 def _authenticate(self):
     """
     Authenticate and setup this instance with the values returned.
     """
     (url, self.cdn_url, self.token) = self.auth.authenticate()
     self.connection_args = parse_url(url)
     self.conn_class = self.connection_args[3] and HTTPSConnection or \
                                                   HTTPConnection
     self.http_connect()
     if self.cdn_url:
         self.cdn_connect()

    def _authenticate(self):
        """
        Authenticate and setup this instance with the values returned.
        """
        (url, self.token) = self.auth.authenticate()
        self.connection_args = parse_url(url)

        if version_info[0] <= 2 and version_info[1] < 6:
            self.conn_class = self.connection_args[3] and THTTPSConnection or THTTPConnection
        else:
            self.conn_class = self.connection_args[3] and HTTPSConnection or HTTPConnection
        self.http_connect()