def simple(request, graph, startNode, endNode):
    start = url_unquote(startNode)
    end = url_unquote(endNode)
    if not (graph.has_node(start) and graph.has_node(end)):
        return request.respondJson({'message': 'node not in graph'},
                                   NOT_FOUND)

    ipaths = nx.shortest_simple_paths(graph, start, end)

    data = {'paths': tuple(ipaths)}
    request.respondJson(data)

def deleteEdge(request, graph, startNode, endNode):
    start = url_unquote(startNode)
    end = url_unquote(endNode)
    try:
        _edge = graph.edge[start][end]
    except KeyError:
        return request.respondJson({'message': 'edge not in graph'},
                                   NOT_FOUND)

    graph.remove_edge(start, end)
    request.respondJson({'message': 'edge deleted'})

def showEdge(request, graph, startNode, endNode):
    start = url_unquote(startNode)
    end = url_unquote(endNode)
    try:
        edge = graph.edge[start][end]
    except KeyError:
        return request.respondJson(
            {'message': 'nodes not in graph or not linked'},
            NOT_FOUND)

    edge = dict(edge)
    edge.update({'start': start, 'end': end,
                 'graph': graph.name})
    request.respondJson(edge)

def unquote_url_values(endpoint, values):
    """Preprocessor that URL-decodes the values given in the URL.

    """
    for key, val in values.items():
        if isinstance(val, basestring):
            values[key] = url_unquote(val)

def index():
    if request.args.has_key("magnet"):
        magnet = url_unquote(request.args["magnet"]).encode(request.charset)
        magnet_xt = url_decode(magnet[magnet.index("?") + 1 :])["xt"]
        torrent = cache.get(magnet_xt)
        if not torrent:
            try:
                handle = lt.add_magnet_uri(
                    ses,
                    magnet,
                    {"save_path": "./invalid", "paused": False, "auto_managed": False, "duplicate_is_error": False},
                )
                while not handle.has_metadata():
                    time.sleep(0.01)
                handle.pause()
                info = handle.get_torrent_info()
                torrent = create_torrent(info)
                cache.set(magnet_xt, torrent)
                ses.remove_torrent(handle, lt.options_t.delete_files)
            except:
                torrent = cache.get(magnet_xt)
        response = Response(response=torrent[1], mimetype="application/x-bittorrent")
        response.headers.add("Content-Disposition", "attachment", filename=torrent[0])
        return response
    return render_template("index.html")

def createEdge(request, graph, startNode, endNode):
    start = url_unquote(startNode)
    end = url_unquote(endNode)
    if not (graph.has_node(start) and graph.has_node(end)):
        return request.respondJson({'message': 'node not in graph'},
                                   NOT_FOUND)

    attrib = request.json()
    attrib['created'] = datetime.utcnow().strftime(DATETIME_FORMAT)

    graph.add_edge(start, end, **attrib)
    data = graph.edge[start][end]
    data.update({'start': start, 'end': end,
                 'graph': graph.name})

    request.respondJson(data, CREATED)

def showNode(request, graph, nodeName):
    nodeName = url_unquote(nodeName)
    try:
        data = _reprNode(graph, nodeName)
    except KeyError, err:
        return request.respondJson({'message': err.message},
                                   NOT_FOUND)

 def test_quoting(self):
     assert urls.url_quote(u'\xf6\xe4\xfc') == '%C3%B6%C3%A4%C3%BC'
     assert urls.url_unquote(urls.url_quote(u'#%="\xf6')) == u'#%="\xf6'
     assert urls.url_quote_plus('foo bar') == 'foo+bar'
     assert urls.url_unquote_plus('foo+bar') == 'foo bar'
     assert urls.url_encode({'a': None, 'b': 'foo bar'}) == 'b=foo+bar'
     assert urls.url_fix(u'http://de.wikipedia.org/wiki/Elf (Begriffsklärung)') == \
            'http://de.wikipedia.org/wiki/Elf%20%28Begriffskl%C3%A4rung%29'

 def test_quoting(self):
     self.assert_strict_equal(urls.url_quote(u'\xf6\xe4\xfc'), '%C3%B6%C3%A4%C3%BC')
     self.assert_strict_equal(urls.url_unquote(urls.url_quote(u'#%="\xf6')), u'#%="\xf6')
     self.assert_strict_equal(urls.url_quote_plus('foo bar'), 'foo+bar')
     self.assert_strict_equal(urls.url_unquote_plus('foo+bar'), u'foo bar')
     self.assert_strict_equal(urls.url_encode({b'a': None, b'b': b'foo bar'}), 'b=foo+bar')
     self.assert_strict_equal(urls.url_encode({u'a': None, u'b': u'foo bar'}), 'b=foo+bar')
     self.assert_strict_equal(urls.url_fix(u'http://de.wikipedia.org/wiki/Elf (Begriffsklärung)'),
            'http://de.wikipedia.org/wiki/Elf%20(Begriffskl%C3%A4rung)')

    def make_environ(self):
        request_url = url_parse(self.path)

        def shutdown_server():
            self.server.shutdown_signal = True

        url_scheme = 'http' if self.server.ssl_context is None else 'https'
        if not self.client_address:
            self.client_address = '<local>'
        if isinstance(self.client_address, str):
            self.client_address = (self.client_address, 0)
        else:
            pass
        path_info = url_unquote(request_url.path)

        environ = {
            'wsgi.version':         (1, 0),
            'wsgi.url_scheme':      url_scheme,
            'wsgi.input':           self.rfile,
            'wsgi.errors':          sys.stderr,
            'wsgi.multithread':     self.server.multithread,
            'wsgi.multiprocess':    self.server.multiprocess,
            'wsgi.run_once':        False,
            'werkzeug.server.shutdown': shutdown_server,
            'SERVER_SOFTWARE':      self.server_version,
            'REQUEST_METHOD':       self.command,
            'SCRIPT_NAME':          '',
            'PATH_INFO':            wsgi_encoding_dance(path_info),
            'QUERY_STRING':         wsgi_encoding_dance(request_url.query),
            # Non-standard, added by mod_wsgi, uWSGI
            "REQUEST_URI": wsgi_encoding_dance(self.path),
            # Non-standard, added by gunicorn
            "RAW_URI": wsgi_encoding_dance(self.path),
            'REMOTE_ADDR':          self.address_string(),
            'REMOTE_PORT':          self.port_integer(),
            'SERVER_NAME':          self.server.server_address[0],
            'SERVER_PORT':          str(self.server.server_address[1]),
            'SERVER_PROTOCOL':      self.request_version
        }

        for key, value in self.get_header_items():
            key = key.upper().replace('-', '_')
            if key not in ('CONTENT_TYPE', 'CONTENT_LENGTH'):
                key = 'HTTP_' + key
                if key in environ:
                    value = "{},{}".format(environ[key], value)
            environ[key] = value

        if environ.get('HTTP_TRANSFER_ENCODING', '').strip().lower() == 'chunked':
            environ['wsgi.input_terminated'] = True
            environ['wsgi.input'] = DechunkedInput(environ['wsgi.input'])

        if request_url.scheme and request_url.netloc:
            environ['HTTP_HOST'] = request_url.netloc

        return environ

 def test_quoting(self):
     assert urls.url_quote(u"\xf6\xe4\xfc") == "%C3%B6%C3%A4%C3%BC"
     assert urls.url_unquote(urls.url_quote(u'#%="\xf6')) == u'#%="\xf6'
     assert urls.url_quote_plus("foo bar") == "foo+bar"
     assert urls.url_unquote_plus("foo+bar") == "foo bar"
     assert urls.url_encode({"a": None, "b": "foo bar"}) == "b=foo+bar"
     assert (
         urls.url_fix(u"http://de.wikipedia.org/wiki/Elf (Begriffsklärung)")
         == "http://de.wikipedia.org/wiki/Elf%20%28Begriffskl%C3%A4rung%29"
     )

    def delete(self):
        from txplaya.playlistregistry import playlistRegistry
        playlistName = url_unquote(self.playlistNameArg)
        playlistRegistry.deletePlaylist(playlistName)

        event = {'event': 'PlaylistRegistryUpdated',
                 'data': {'list': playlistRegistry.list_()}}
        self.mainController.announce(event)

        return {'msg': 'Playlist deleted'}

def worker(domain, save_rules):
    """Worker process, fetches url from the front,
       crawl, push new urls to front
       and pushes content to sink if matched on a rule"""
    ctx = zmq.Context()
    worker_ = ctx.socket(zmq.REQ)
    worker_.connect('tcp://localhost:5050')
    saver = ctx.socket(zmq.PUSH)
    saver.connect('tcp://localhost:5051')
    urlsink = ctx.socket(zmq.PUSH)
    urlsink.connect('tcp://localhost:5052')

    matcher = Map(map(Rule, save_rules)).bind('', '/').match

    while True:
        worker_.send('')
        url = worker_.recv().decode('utf-8')

        try:
            q = rq.get(u'http://%s%s' % (domain, url_unquote(url)),
                       allow_redirects = False)
        except ConnectionError:
            continue

        if q.status_code == 301 or q.status_code == 302:
            redirect = q.headers['location']
            if domain in redirect:
                # only sent to front
                urlsink.send(redirect.split(domain)[1].encode('utf-8'))
            continue

        html = q.content
        try: _, data = matcher(url)
        except NotFound: pass
        else:
            # needs to be saved, sends html, url, data to saver
            data = zlib.compress(json.dumps([html, url, data]))
            saver.send(data)
            del data

        fetched = set()

        for link in fromstring(html).cssselect("a[href]"):
            link = link.attrib['href'].split('#')[0]
            if link.startswith('file://') or link.startswith('javascript:'): continue

            if not link.startswith('http'):
                fetched.add(link)
            elif domain in link:
                fetched.add(link.split(domain)[1])

        for l in fetched:
            urlsink.send(l.encode('utf-8'))

        def wrapped(request, **kwargs):
            graphName = kwargs.pop(argName)
            graphName = url_unquote(graphName)
            try:
                graph = request.app.graphs[graphName]
            except KeyError:
                return request.respondJson({'message': 'graph {0} not found'.format(graphName)},
                                           NOT_FOUND)

            graph.name = graphName
            kwargs[argName] = graph

            return fc(request, **kwargs)

def test_quoting():
    strict_eq(urls.url_quote(u'\xf6\xe4\xfc'), '%C3%B6%C3%A4%C3%BC')
    strict_eq(urls.url_unquote(urls.url_quote(u'#%="\xf6')), u'#%="\xf6')
    strict_eq(urls.url_quote_plus('foo bar'), 'foo+bar')
    strict_eq(urls.url_unquote_plus('foo+bar'), u'foo bar')
    strict_eq(urls.url_quote_plus('foo+bar'), 'foo%2Bbar')
    strict_eq(urls.url_unquote_plus('foo%2Bbar'), u'foo+bar')
    strict_eq(urls.url_encode({b'a': None, b'b': b'foo bar'}), 'b=foo+bar')
    strict_eq(urls.url_encode({u'a': None, u'b': u'foo bar'}), 'b=foo+bar')
    strict_eq(urls.url_fix(u'http://de.wikipedia.org/wiki/Elf (Begriffsklärung)'),
           'http://de.wikipedia.org/wiki/Elf%20(Begriffskl%C3%A4rung)')
    strict_eq(urls.url_quote_plus(42), '42')
    strict_eq(urls.url_quote(b'\xff'), '%FF')

def component_search(search_str):
    search_str = url_unquote(search_str)
    cpts = db.session.query(Component).filter(Component.description.like("%"+search_str+"%")).all()
    if not cpts:
        notfound_msg = Markup("Could not find software matching the search terms: <b>%s</b>") % (search_str)
        return render_template('notfound.html', message = notfound_msg)

    items = list()
    for cpt in cpts:
        items.append(component_to_item(cpt))

    return render_template('results.html',
        title = "Search results",
        items = items)

def listEdges(request, graph, startNode):
    if not graph.has_node(startNode):
        return request.respondJson({'message': 'node not in graph'},
                                   NOT_FOUND)

    start = url_unquote(startNode)
    iNeighborsEdges = _iterNeigborsEdges(graph, start)

    attrib = request.json()
    iNeighborsEdges = _iterFilterEdges(iNeighborsEdges, attrib)

    offset, limit = _getPaging(request)
    iNeighborsEdges = islice(iNeighborsEdges, offset, offset + limit)

    request.respondJson({'neighbors': dict(iNeighborsEdges)})

def test_quoting():
    strict_eq(urls.url_quote(u"\xf6\xe4\xfc"), "%C3%B6%C3%A4%C3%BC")
    strict_eq(urls.url_unquote(urls.url_quote(u'#%="\xf6')), u'#%="\xf6')
    strict_eq(urls.url_quote_plus("foo bar"), "foo+bar")
    strict_eq(urls.url_unquote_plus("foo+bar"), u"foo bar")
    strict_eq(urls.url_quote_plus("foo+bar"), "foo%2Bbar")
    strict_eq(urls.url_unquote_plus("foo%2Bbar"), u"foo+bar")
    strict_eq(urls.url_encode({b"a": None, b"b": b"foo bar"}), "b=foo+bar")
    strict_eq(urls.url_encode({u"a": None, u"b": u"foo bar"}), "b=foo+bar")
    strict_eq(
        urls.url_fix(u"http://de.wikipedia.org/wiki/Elf (Begriffsklärung)"),
        "http://de.wikipedia.org/wiki/Elf%20(Begriffskl%C3%A4rung)",
    )
    strict_eq(urls.url_quote_plus(42), "42")
    strict_eq(urls.url_quote(b"\xff"), "%FF")

    def post():
        """
        get vulnerable file content
        :return:
        """
        data = request.json
        if not data or data == "":
            return {'code': 1003, 'msg': 'Only support json, please post json data.'}

        sid = data.get('sid')
        file_path = url_unquote(data.get('file_path'))

        if not sid or sid == '':
            return {"code": 1002, "msg": "sid is required."}

        if not file_path or file_path == '':
            return {'code': 1002, 'msg': 'file_path is required.'}

        s_sid_file = os.path.join(running_path, '{sid}_data'.format(sid=sid))
        if not os.path.exists(s_sid_file):
            return {'code': 1002, 'msg': 'No such target.'}

        with open(s_sid_file, 'r') as f:
            target_directory = json.load(f).get('result').get('target_directory')

        if not target_directory or target_directory == '':
            return {'code': 1002, 'msg': 'No such directory'}

        if PY2:
            file_path = map(secure_filename, [path.decode('utf-8') for path in file_path.split('/')])
        else:
            file_path = map(secure_filename, [path for path in file_path.split('/')])

        filename = target_directory
        for _dir in file_path:
            filename = os.path.join(filename, _dir)
        if os.path.exists(filename):
            extension = guess_type(filename)
            if is_text(filename):
                with open(filename, 'r') as f:
                    file_content = f.read()
            else:
                file_content = 'This is a binary file.'
        else:
            return {'code': 1002, 'msg': 'No such file.'}

        return {'code': 1001, 'result': {'file_content': file_content,
                                         'extension': extension}}

def deleteNode(request, graph, nodeName):
    nodeName = url_unquote(nodeName)
    if not graph.has_node(nodeName):
        return request.respondJson({'message': 'node not in graph'},
                                   NOT_FOUND)
    try:
        next(graph.neighbors_iter(nodeName))
    except StopIteration:
        # no neighbours, ok to delete
        pass
    else:
        return request.respondJson({'message': 'node has edges'},
                                   CONFLICT)

    graph.remove_node(nodeName)
    request.respondJson({'message': 'node deleted'})