def clonescan(self, id):
        sf = SpiderFoot(self.config)
        dbh = SpiderFootDb(self.config)
        types = dbh.eventTypes()
        info = dbh.scanInstanceGet(id)
        scanconfig = dbh.scanConfigGet(id)
        scanname = info[0]
        scantarget = info[1]
        targetType = None

        if scanname == "" or scantarget == "" or len(scanconfig) == 0:
            return self.error("Something went wrong internally.")

        targetType = sf.targetType(scantarget)
        if targetType == None:
            # It must be a name, so wrap quotes around it
            scantarget = """ + scantarget + """

        modlist = scanconfig['_modulesenabled'].split(',')

        templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
        return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot,
                            modules=self.config['__modules__'], selectedmods=modlist,
                            scanname=unicode(scanname, 'utf-8', errors='replace'), 
                            scantarget=unicode(scantarget, 'utf-8', errors='replace'))

    def savesettings(self, allopts, token):
        if str(token) != str(self.token):
            return self.error("Invalid token (" + str(self.token) + ").")

        try:
            dbh = SpiderFootDb(self.config)
            # Reset config to default
            if allopts == "RESET":
                dbh.configClear()  # Clear it in the DB
                self.config = deepcopy(self.defaultConfig)  # Clear in memory
            else:
                useropts = json.loads(allopts)
                cleanopts = dict()
                for opt in useropts.keys():
                    cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]

                currentopts = deepcopy(self.config)

                # Make a new config where the user options override
                # the current system config.
                sf = SpiderFoot(self.config)
                self.config = sf.configUnserialize(cleanopts, currentopts)
                dbh.configSet(sf.configSerialize(currentopts))
        except Exception as e:
            return self.error("Processing one or more of your inputs failed: " + str(e))

        templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)
        self.token = random.randint(0, 99999999)
        return templ.render(opts=self.config, pageid='SETTINGS', updated=True,
                            docroot=self.docroot, token=self.token)

 def newscan(self):
     dbh = SpiderFootDb(self.config)
     types = dbh.eventTypes()
     templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
     return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot,
                         modules=self.config['__modules__'], scanname="",
                         selectedmods="", scantarget="")

    def scanopts(self, id):
        ret = dict()
        dbh = SpiderFootDb(self.config)
        ret['config'] = dbh.scanConfigGet(id)
        ret['configdesc'] = dict()
        for key in ret['config'].keys():
            if ':' not in key:
                ret['configdesc'][key] = self.config['__globaloptdescs__'][key]
            else:
                [modName, modOpt] = key.split(':')
                if modName not in self.config['__modules__'].keys():
                    continue

                if modOpt not in self.config['__modules__'][modName]['optdescs'].keys():
                    continue

                ret['configdesc'][key] = self.config['__modules__'][modName]['optdescs'][modOpt]

        sf = SpiderFoot(self.config)
        meta = dbh.scanInstanceGet(id)
        if meta[3] != 0:
            started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[3]))
        else:
            started = "Not yet"

        if meta[4] != 0:
            finished = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[4]))
        else:
            finished = "Not yet"
        ret['meta'] = [meta[0], meta[1], meta[2], started, finished, meta[5]]

        return json.dumps(ret)

    def scanexportjsonmulti(self, ids):
        dbh = SpiderFootDb(self.config)
        scaninfo = dict()

        for id in ids.split(','):
          scan_name = dbh.scanInstanceGet(id)[0]

          if scan_name not in scaninfo:
              scaninfo[scan_name] = []

          for row in dbh.scanResultEvent(id):
              lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
              event_data = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "")
              source_data = str(row[2])
              source_module = str(row[3])
              event_type = row[4]
              false_positive = row[13]

              if event_type == "ROOT":
                  continue

              scaninfo[scan_name].append({
                  "data": event_data,
                  "type": event_type,
                  "source_module": source_module,
                  "source_data": source_data,
                  "false_positive": false_positive,
                  "lastseen": lastseen
              })

        cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.json"
        cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
        cherrypy.response.headers['Pragma'] = "no-cache"

        return json.dumps(scaninfo)

    def searchBase(self, id=None, eventType=None, value=None):
        regex = ""
        if [id, eventType, value].count('') == 2 or \
                        [id, eventType, value].count(None) == 2:
            return None

        if value.startswith("/") and value.endswith("/"):
            regex = value[1:len(value) - 1]
            value = ""

        value = value.replace('*', '%')
        if value in [None, ""] and regex in [None, ""]:
            value = "%"
            regex = ""

        dbh = SpiderFootDb(self.config)
        criteria = {
            'scan_id': None if id == '' else id,
            'type': None if eventType == '' else eventType,
            'value': None if value == '' else value,
            'regex': None if regex == '' else regex
        }

        data = dbh.search(criteria)
        retdata = []
        for row in data:
            lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
            escapeddata = cgi.escape(row[1])
            escapedsrc = cgi.escape(row[2])
            retdata.append([lastseen, escapeddata, escapedsrc,
                            row[3], row[5], row[6], row[7], row[8], row[10], 
                            row[11], row[4], row[13], row[14]])

        return retdata

    def savesettingsraw(self, allopts, token):
        if str(token) != str(self.token):
            return json.dumps(["ERROR", "Invalid token (" + str(self.token) + ")."])

        try:
            dbh = SpiderFootDb(self.config)
            # Reset config to default
            if allopts == "RESET":
                dbh.configClear()  # Clear it in the DB
                self.config = deepcopy(self.defaultConfig)  # Clear in memory
            else:
                useropts = json.loads(allopts)
                cleanopts = dict()
                for opt in useropts.keys():
                    cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]

                currentopts = deepcopy(self.config)

                # Make a new config where the user options override
                # the current system config.
                sf = SpiderFoot(self.config)
                self.config = sf.configUnserialize(cleanopts, currentopts)
                dbh.configSet(sf.configSerialize(currentopts))
        except Exception as e:
            return json.dumps(["ERROR", "Processing one or more of your inputs failed: " + str(e)])

        return json.dumps(["SUCCESS", ""])

    def resultsetfp(self, id, resultids, fp):
        dbh = SpiderFootDb(self.config)
        if fp not in ["0", "1"]:
            return json.dumps(["ERROR", "No FP flag set or not set correctly."])

        ids = json.loads(resultids)
        if not ids:
            return json.dumps(["ERROR", "No IDs supplied."])

        # Cannot set FPs if a scan is not completed
        status = dbh.scanInstanceGet(id)
        if status[5] not in [ "ABORTED", "FINISHED", "ERROR-FAILED" ]:
            return json.dumps(["WARNING", "Scan must be in a finished state when " + \
                               "setting False Positives."])

        # Make sure the user doesn't set something as non-FP when the
        # parent is set as an FP.
        if fp == "0":
            data = dbh.scanElementSourcesDirect(id, ids)
            for row in data:
                if str(row[14]) == "1":
                    return json.dumps(["WARNING", 
                        "You cannot unset an element as False Positive " + \
                        "if a parent element is still False Positive."]);

        # Set all the children as FPs too.. it's only logical afterall, right?
        childs = dbh.scanElementChildrenAll(id, ids)
        allIds = ids + childs

        ret = dbh.scanResultsUpdateFP(id, allIds, fp)
        if not ret:
            return json.dumps(["ERROR", "Exception encountered."])
        else: 
            return json.dumps(["SUCCESS", ""])

    def stopscanmulti(self, ids):
        global globalScanStatus # running scans
        dbh = SpiderFootDb(self.config)
        error = list()

        for id in ids.split(","):
            errState = False
            scaninfo = dbh.scanInstanceGet(id)

            if globalScanStatus.getStatus(id) == "FINISHED" or scaninfo[5] == "FINISHED":
                error.append("Scan '" + scaninfo[0] + "' is in a finished state. <a href='/scandelete?id=" + \
                             id + "&confirm=1'>Maybe you want to delete it instead?</a>")
                errState = True

            if not errState and (globalScanStatus.getStatus(id) == "ABORTED" or scaninfo[5] == "ABORTED"):
                error.append("Scan '" + scaninfo[0] + "' is already aborted.")
                errState = True

            if not errState and globalScanStatus.getStatus(id) is None:
                error.append("Scan '" + scaninfo[0] + "' is not actually running. A data consistency " + \
                             "error for this scan probably exists. <a href='/scandelete?id=" + \
                             id + "&confirm=1'>Click here to delete it.</a>")
                errState = True
            
            if not errState:
                globalScanStatus.setStatus(id, "ABORT-REQUESTED")

        templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
        return templ.render(pageid='SCANLIST', stoppedscan=True, 
                            errors=error, docroot=self.docroot)

 def scanlog(self, id):
     dbh = SpiderFootDb(self.config)
     data = dbh.scanLogs(id)
     retdata = []
     for row in data:
         generated = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]/1000))
         retdata.append([generated, row[1], row[2], cgi.escape(row[3])])
     return json.dumps(retdata)

 def __init__(self, config):
     self.defaultConfig = deepcopy(config)
     dbh = SpiderFootDb(config)
     # 'config' supplied will be the defaults, let's supplement them
     # now with any configuration which may have previously been
     # saved.
     sf = SpiderFoot(config)
     self.config = sf.configUnserialize(dbh.configGet(), config)

 def scansummary(self, id):
     dbh = SpiderFootDb(self.config)
     data = dbh.scanResultSummary(id)
     retdata = []
     for row in data:
         lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[2]))
         retdata.append([row[0], row[1], lastseen, row[3], row[4]])
     return json.dumps(retdata)

    def scaninfo(self, id):
        dbh = SpiderFootDb(self.config)
        res = dbh.scanInstanceGet(id)
        if res == None:
            return self.error("Scan ID not found.")

        templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
        return templ.render(id=id, name=res[0], status=res[5], pageid='SCANLIST')

 def scaneventresultsunique(self, id, eventType, filterfp=False):
     dbh = SpiderFootDb(self.config)
     data = dbh.scanResultEventUnique(id, eventType, filterfp)
     retdata = []
     for row in data:
         escaped = cgi.escape(row[0])
         retdata.append([escaped, row[1], row[2]])
     return json.dumps(retdata, ensure_ascii=False)

    def scanelementtypediscovery(self, id, eventType):
        keepGoing = True
        sf = SpiderFoot(self.config)
        dbh = SpiderFootDb(self.config)
        pc = dict()
        datamap = dict()

        # Get the events we will be tracing back from
        leafSet = dbh.scanResultEvent(id, eventType)

        # Get the first round of source IDs for the leafs
        nextIds = list()
        for row in leafSet:
            # these must be unique values!
            parentId = row[9]
            childId = row[8]
            datamap[childId] = row

            if pc.has_key(parentId):
                if childId not in pc[parentId]:
                    pc[parentId].append(childId)
            else:
                pc[parentId] = [ childId ]

            # parents of the leaf set
            if parentId not in nextIds:
                nextIds.append(parentId)

        while keepGoing:
            parentSet = dbh.scanElementSources(id, nextIds)
            nextIds = list()
            keepGoing = False

            for row in parentSet:
                parentId = row[9]
                childId = row[8]
                datamap[childId] = row
                #print childId + " = " + str(row)

                if pc.has_key(parentId):
                    if childId not in pc[parentId]:
                        pc[parentId].append(childId)
                else:
                    pc[parentId] = [ childId ]
                if parentId not in nextIds:
                    nextIds.append(parentId)

                # Prevent us from looping at root
                if parentId != "ROOT":
                    keepGoing = True

        datamap[parentId] = row

        #print pc
        retdata = dict()
        retdata['tree'] = sf.dataParentChildToTree(pc)
        retdata['data'] = datamap
        return json.dumps(retdata, ensure_ascii=False)

    def scanstatus(self, id):
        dbh = SpiderFootDb(self.config)
        data = dbh.scanInstanceGet(id)
        created = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[2]))
        started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[3]))
        ended = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[4]))

        retdata = [data[0], data[1], created, started, ended, data[5]]
        return json.dumps(retdata)

    def scaninfo(self, id):
        dbh = SpiderFootDb(self.config)
        res = dbh.scanInstanceGet(id)
        if res is None:
            return self.error("Scan ID not found.")

        templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup, disable_unicode=True, input_encoding='utf-8')
        return templ.render(id=id, name=cgi.escape(res[0]), status=res[5], docroot=self.docroot,
                            pageid="SCANLIST")

 def scanerrors(self, id, limit=None):
     dbh = SpiderFootDb(self.config)
     data = dbh.scanErrors(id, limit)
     retdata = []
     for row in data:
         generated = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0] / 1000))
         retdata.append([generated, row[1],
                         cgi.escape(unicode(row[2], errors='replace'))])
     return json.dumps(retdata)

 def scaneventresults(self, id, eventType):
     dbh = SpiderFootDb(self.config)
     data = dbh.scanResultEvent(id, eventType)
     retdata = []
     for row in data:
         lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
         escaped = cgi.escape(row[1])
         retdata.append([lastseen, escaped, row[2], row[3], row[5], row[6], row[7]])
     return json.dumps(retdata, ensure_ascii=False)

    def startscan(self, scanname, scantarget, modulelist, typelist):
        modopts = dict() # Not used yet as module options are set globally
        modlist = list()
        sf = SpiderFoot(self.config)
        dbh = SpiderFootDb(self.config)
        types = dbh.eventTypes()

        [scanname, scantarget] = self.cleanUserInput([scanname, scantarget])

        if scanname == "" or scantarget == "":
            return self.error("Form incomplete.")

        if typelist == "" and modulelist == "":
            return self.error("Form incomplete.")

        if modulelist != "":
            modlist = modulelist.replace('module_', '').split(',')
        else:
            typesx = typelist.replace('type_', '').split(',')
            # 1. Find all modules that produce the requested types
            modlist = sf.modulesProducing(typesx)
            newmods = deepcopy(modlist)
            newmodcpy = deepcopy(newmods)
            # 2. For each type those modules consume, get modules producing
            while len(newmodcpy) > 0:
                for etype in sf.eventsToModules(newmodcpy):
                    xmods = sf.modulesProducing([etype])
                    for mod in xmods:
                        if mod not in modlist:
                            modlist.append(mod)
                            newmods.append(mod)
                newmodcpy = deepcopy(newmods)
                newmods = list()

        # Add our mandatory storage module..
        if "sfp__stor_db" not in modlist:
            modlist.append("sfp__stor_db")
        modlist.sort()

        # For now we don't permit multiple simultaneous scans
        for thread in threading.enumerate():
            if thread.name.startswith("SF_"):
                templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
                return templ.render(modules=self.config['__modules__'], 
                    alreadyRunning=True, runningScan=thread.name[3:], 
                    types=types, pageid="NEWSCAN")

        # Start running a new scan
        self.scanner = SpiderFootScanner(scanname, scantarget.lower(), modlist, 
            self.config, modopts)
        t = threading.Thread(name="SF_" + scanname, target=self.scanner.startScan)
        t.start()

        templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
        return templ.render(id=self.scanner.myId, name=scanname, 
            status=self.scanner.status, pageid="SCANLIST")