def scan_template():
    from lxml import etree
    from StringIO import StringIO

    response.title = "%s :: Nexpose Scan Templates" % (settings.title)
    formupload = SQLFORM.factory(
        Field('f_filename', 'upload', uploadfolder=os.path.join(request.folder, 'data', 'misc'), label=T('Import Nexpose Scan Template')),
             _formname='uploader')

    if formupload.accepts(request.vars, formname='uploader'):
        najax = NXAJAX(session.najaxsession)
        template_class = ScanTemplates()
        filename = os.path.join(request.folder,'data','misc',formupload.vars.f_filename)
        template_xml = etree.parse(filename, etree.XMLParser())
        imported = template_class.importtemplate(etree.tostring(template_xml), najax)
        response.flash = imported
        templates = ScanTemplates.listscantemps(True, najax)
        parse_templates = DIV(TAG(templates).elements('templateid'))
        return dict(form="", form2=formupload, html=parse_templates)

    najax = NXAJAX(session.najaxsession)
    najax.host = auth.user.f_nexpose_host
    najax.port = auth.user.f_nexpose_port
    if najax.login(user_id=auth.user.f_nexpose_user, password=auth.user.f_nexpose_pw):
        log("Logged in to Nexpose API. Session cached.")
        session.najaxsession = najax.getsession()
        template_class = ScanTemplates()
        templates = template_class.listscantemps(True, najax)
        response.flash = "Loaded %s scan templates" % (templates.count('<templateid>'))
        parse_templates = DIV(TAG(templates).elements('templateid'))
        return dict(form="", form2=formupload, html=parse_templates)
    else:
        response.flash = "Unable to login to Nexpose"

    return dict(form=formlogin, form2="", html="")

def process_medusa(line):
    """
    Process a medusa line and return a dictionary

    :param line: A line from a medusa output file
    :returns dict: A dictionary based upon the content
    { 'ip'  : ip address
      'port': port info - can be port # or module name
      'user': username,
      'pass': password,
      'hash': ntlm hash if smbnt hash used
      'msg' : status message
    }

    >>> process_medusa("# Medusa v.2.0a_rc1 (2012-12-06 10:44:10)")
    {}
    >>> process_medusa("ACCOUNT FOUND: [smbnt] Host: 10.89.172.23 User: spauser Password: password [SUCCESS]")
    {'ip': '10.89.172.23', 'error': False, 'user': 'spauser', 'pass': 'password', 'msg': '[SUCCESS]', 'type': 'cleartext', 'port': '[smbnt]'}
    >>> process_medusa("ACCOUNT FOUND: [smbnt] Host: 10.89.172.24 User: spauser Password: AAD3B435B51404EEAAD3B435B51404EE:31D6CFE0D16AE931B73C59D7E0C089C0::: [SUCCESS]")
    {'ip': '10.89.172.23', 'port': 'smbnt', 'user': 'spauser', 'password': '', 'hash': 'AAD3B435B51404EEAAD3B435B51404EE:31D6CFE0D16AE931B73C59D7E0C089C0', 'msg': '[SUCCESS]'}
    """
    retval = {}

    try:
        data = line.split()
    except Exception, e:
        log("Error processing medusa line: %s -- %s" % (e, line), logging.ERROR)
        return retval

def get_or_create_record(argument, **defaults):
    """
    Returns a t_host record based on the argument. If argument is an ipv4/ipv6 address it looks it up. If it's an
    integer it returns it. If none exist and argument is an ipv4/ipv6 address it creates a new record using the
    defaults provided.

    :param argument: ip address or db.t_hosts.id
    :param defaults: dictionary of db.t_hosts fields, validated before inserting
    :returns: Row with id

    >>> get_or_create_record('2.2.2.2')
    <Row {'f_confirmed': False, 'f_followup': None, 'f_macaddr': None, 'f_longitude': None, 'f_vuln_count': 0L, 'f_asset_group': 'undefined', 'f_accessed': False, 'id': 1L, 'f_vuln_graph': '0,0,0,0,0,0,0,0,0,0', 'f_engineer': 1L, 'f_exploit_count': 0L, 'f_hostname': None, 'f_ipaddr': '2.2.2.2', 'f_city': None, 'f_country': None, 'f_latitude': None, 'f_netbios_name': None, 'f_service_count': 0L}>

    >>> get_or_create_record('9.9.9.9', f_engineer=9999)
    None

    >>> get_or_create_record(1)
    <Row {'f_confirmed': False, 'f_followup': None, 'f_macaddr': None, 'f_longitude': None, 'f_vuln_count': 0L, 'f_asset_group': 'undefined', 'f_accessed': False, 'id': 1L, 'f_vuln_graph': '0,0,0,0,0,0,0,0,0,0', 'f_engineer': 1L, 'f_exploit_count': 0L, 'f_hostname': None, 'f_ipaddr': '2.2.2.2', 'f_city': None, 'f_country': None, 'f_latitude': None, 'f_netbios_name': None, 'f_service_count': 0L}>

    >>> get_or_create_record(9999)
    None
    """
    if argument is None:
        return None

    from gluon.validators import IS_IPADDRESS

    db = current.globalenv["db"]
    auth = current.globalenv["auth"]

    record = get_host_record(argument)
    if not record:
        fields = {}
        for k in defaults.keys():
            if k in db.t_hosts.fields:
                fields[k] = defaults[k]

        # set defaults for assetgroup/engineer if not set
        if "f_asset_group" not in fields:
            fields["f_asset_group"] = "undefined"
        if "f_engineer" not in fields:
            fields["f_engineer"] = auth.user_id or 1

        if IS_IPADDRESS()(argument)[1] == None:
            fields["f_ipaddr"] = argument
        else:
            # invalid ip address, clear the fields
            fields = None

        if fields:
            host_rec = db.t_hosts.validate_and_insert(**fields)
            if host_rec.errors:
                log("Error creating host record: %s" % host_rec.errors, logging.ERROR)
            else:
                db.commit()
                record = db.t_hosts(host_rec.get("id"))

    return record

def run_scan(blacklist=None, target_list=None, scan_options=None):
    """
    Executes nmap scan
    """
    from zenmapCore_Kvasir.NmapCommand import NmapCommand
    from zenmapCore_Kvasir.NmapOptions import NmapOptions
    from time import sleep

    if scan_options[0] is not "nmap":
        if "nmap" in settings:
            scan_options.insert(0, settings.nmap)
        else:
            scan_options.insert(0, "nmap")

    if target_list:
        data = []
        for ip in target_list:
            data.append(ip.strip(" \t\n\r"))
        target_list = data

    if blacklist:
        data = []
        for ip in blacklist:
            data.append(ip.strip(" \t\n\r"))
        blacklist = [",".join(map(str, data))]
        blacklist.insert(0, "--exclude")

    ops = NmapOptions()
    try:
        ops.parse(scan_options + target_list + blacklist)
    except Exception as e:
        log("[!] %s" % e)

    cmd = NmapCommand(ops.render_string())

    log(" [*] Starting Nmap Scan: %s" % (cmd.command))
    cmd.run_scan()

    try:
        cmd.scan_state()
    except Exception as e:
        log("[!] %s" % e)

    full_output = ""
    while cmd.scan_state():
        sleep(5)
        result = cmd.get_output()
        start = len(full_output) - len(result)
        output = result[start:]
        full_output = "%s%s" % (full_output, output)
        log(output)

    log(" [*] Nmap Scan Complete")

    filename = cmd.get_xml_output_filename()
    return filename

def process_report(
    filename=None,
    host_list=[],
    query=None,
    ip_ignore_list=None,
    ip_include_list=None,
    engineer=1,
    asset_group="ShodanHQ Import",
):
    """
    Processes a ShodanHQ XML Report adding records to the db
    """

    settings = current.globalenv['settings']

    #try:
    #    from shodan import WebAPI
    #    from shodan.api import WebAPIError
    #    webapi = WebAPI(settings.shodanhq_apikey)
    #except ImportError:
    #    webapi = None

    sd = ShodanData()
    sd.engineer = engineer
    sd.asset_group = asset_group

    # build the hosts only/exclude list
    if ip_ignore_list:
        sd.ip_exclude = ip_ignore_list.split('\r\n')
        # TODO: check for ip subnet/range and break it out to individuals
    if ip_include_list:
        sd.ip_only = ip_include_list.split('\r\n')
        # TODO: check for ip subnet/range and break it out to individuals

    hosts = []
    if filename:
        log(" [*] Processing ShodanHQ report file: %s" % (filename))
        try:
            from lxml import etree
        except ImportError:
            try:
                import xml.etree.cElementTree as etree
            except ImportError:
                try:
                    import xml.etree.ElementTree as etree
                except:
                    raise Exception("Unable to find valid ElementTree module.")

        try:
            xml = etree.parse(filename)
        except etree.ParseError, e:
            raise Exception(" [!] Invalid XML file (%s): %s " % (filename, e))
            return

        root = xml.getroot()
        hosts = root.findall("host")

def process_exploits(filename=None):
    """
    Process Nexpose exploits.xml file into the database
    """

    log("Processing Nexpose exploits file: %s ..." % filename)

    try:
        exploits = etree.parse(filename)
    except etree.ParseError, e:
        raise Exception("Error processing file: %s" % e)

def process_report_xml(
    filename=None,
    ip_ignore_list=None,
    ip_include_list=None,
    engineer=1,
    asset_group="Metasploit Import",
    update_hosts=True,
    ):
    """
    Processes a Metasploit XML Export for the following data and adds to the db:

    - Hosts and services
    - Credentials

    Generate the XML report by using db_export -t xml filename.xml or through WebUI

    TODO: Auto-exploits successful exploit attempts if matching CVE/VulnDB entry found
    """
    from gluon.validators import IS_IPADDRESS
    from skaldship.passwords.utils import lookup_hash
    from skaldship.hosts import get_host_record, get_or_create_record
    from skaldship.services import Services
    services = Services()

    db = current.globalenv['db']
    #cache = current.globalenv['cache']

    try:
        from lxml import etree
    except ImportError:
        try:
            import xml.etree.cElementTree as etree
        except ImportError:
            try:
                import xml.etree.ElementTree as etree
            except:
                raise Exception("Unable to find valid ElementTree module.")

    # build the hosts only/exclude list
    ip_exclude = []
    if ip_ignore_list:
        ip_exclude = ip_ignore_list.split('\r\n')
        # TODO: check for ip subnet/range and break it out to individuals
    ip_only = []
    if ip_include_list:
        ip_only = ip_include_list.split('\r\n')
        # TODO: check for ip subnet/range and break it out to individuals

    log(" [*] Processing Metasploit Pro report file: %s" % (filename))

    try:
        xml = etree.parse(filename)
    except etree.ParseError, e:
        raise Exception(" [!] Invalid XML file (%s): %s " % (filename, e))

def grab_screenshot(url=None, outfile=None, phantomjs="/usr/bin/phantomjs"):
    """
    Capture a PNG image of a URL using phantomjs

    @args:
        url: Website URL to retrieve
        outfile: Output filename, will overwrite but not remove failures
        phantomjs: Full path to phantomjs binary

    @output:
        [True/False, png image data]
    """
    import os
    db = current.globalenv['db']

    if not outfile:
        raise Exception("No output filename provided")

    try:
        os.stat(phantomjs)
    except OSError:
        phantomjs = "/usr/local/bin/phantomjs"
        try:
            os.stat(phantomjs)
        except OSError:
            logging.error("Unable to locate phantomjs binary")
            return [False, None]

    # encode the url to make sure it passes cleanly to phantomjs
    url = urllib.quote(url, safe='/:')
    folder = current.globalenv['request'].folder
    from sys import platform
    if platform in ["linux", "linux2"]:
        timeout = ["/usr/bin/timeout", "-k", "2", "5"]
    elif platform in ["darwin", "freebsd"]:
        timeout = [os.path.join(folder, 'private/timeout3'), "-t" "5"]
    else:
        timeout = []
    phantom = timeout + [phantomjs, "--ignore-ssl-errors=true", "%s/modules/skaldship/valkyries/webimaging.js" % (folder), url, outfile]
    log("calling: %s" % str(phantom), logging.DEBUG)
    call(phantom)
    try:
        f = file(outfile)
        imgdata = f.read()
        f.close()
        result = True
    except:
        result = False
        imgdata = None

    return [result, imgdata]

def vuln_time_convert(vtime=''):
    """Converts Nexpose timetsamp (YYYYMMDDTHHMMSSUUU) into python datetime"""
    if not vtime:
        tval = datetime(1970, 1, 1)
    else:
        if isinstance(vtime, str):
            if vtime[8] == "T":
                tstr = "%%Y%%m%%dT%%H%%M%%S%s" % vtime[15:]
                tval = time.strptime(vtime, tstr)
            else:
                log("Unknown datetime value: %s" % vtime, logging.ERROR)
        else:
            log("Invalid datetime value provided: %s" % vtime, logging.ERROR)
            tval = datetime(1970, 1, 1)
    return datetime.fromtimestamp(time.mktime(tval))

def process_xml(
    filename=None,
    asset_group=None,
    engineer=None,
    msf_settings={},
    ip_ignore_list=None,
    ip_include_list=None,
    update_hosts=False,
    ):
    """
    Process a Nessus XML Report file

    Args:
        filename: A local filename to process
        asset_group: Asset group to assign hosts to
        engineer: Engineer record number to assign hosts to
        msf_workspace: If set a Metasploit workspace to send the scanfile to via the API
        ip_ignore_list: List of IP addresses to ignore
        ip_include_list: List of IP addresses to ONLY import (skip all others)
        update_hosts: Boolean to update/append to hosts, otherwise hosts are skipped

    Returns:
        msg: A string status message
    """
    from skaldship.cpe import lookup_cpe

    db = current.globalenv['db']
    cache = current.globalenv['cache']
    settings = current.globalenv['settings']

    # build the hosts only/exclude list
    ip_exclude = []
    if ip_ignore_list:
        ip_exclude = ip_ignore_list.split('\r\n')
        # TODO: check for ip subnet/range and break it out to individuals
    ip_only = []
    if ip_include_list:
        ip_only = ip_include_list.split('\r\n')
        # TODO: check for ip subnet/range and break it out to individuals

    log(" [*] Processing Nessus scan file %s" % filename)

    try:
        nessus_xml = etree.parse(filename)
    except etree.ParseError, e:
        msg = " [!] Invalid Nessus scan file (%s): %s " % (filename, e)
        log(msg, logging.ERROR)
        return msg

def process_cracked_file(pw_file=None, file_type=None, message=""):
    """
    Process a file of cracked passwords and update the cleartext with
    the new results.
    """
    import fileinput

    db = current.globalenv['db']
    cache = current.globalenv['cache']

    if pw_file is not None:
        try:
            fIN = fileinput.input(files=pw_file)
        except IOError, e:
            log("Error opening %s: %s" % (pw_file, e), logging.ERROR)
            return "Error opening %s: %s" % (pw_file, e)

def process_file(filename=None, asset_group=None, engineer=None):

    # Upload and process hping Scan file
    from skaldship.hosts import get_host_record, do_host_status, add_or_update
    from gluon.validators import IS_IPADDRESS

    log(" [*] Processing hping scan file %s" % filename)

    hoststats = 0
    nodefields = {'f_engineer': engineer, 'f_asset_group': asset_group, 'f_confirmed': False}

    svc_db = db.t_services

    host_ip = None
    ICMP_type = ''
    answer_ip = ''

    with open(filename) as f:
        for line in f:
            if "IP: " in line:
                host_ip = line.split()[1]
                if IS_IPADDRESS()(host_ip)[1] == None:
                    nodefields['f_ipaddr'] = host_ip
                    db.t_hosts.update_or_insert(**nodefields)
		            db.commit()
                    hoststats += 1
                else:
                    log(" [!] ERROR: Not a valid IP Address (%s)" % host_ip, logging.ERROR)
            if "[*] " in line:
                ICMP_type = line.split()[1]
            if "ip=" in line:
                ip = line.split('=')[2]
                answer_ip = ip.split()[0]
            if "transmitted" in line:
                packets = line.split()
                if packets[0] == packets[3]:
                    if answer_ip != host_ip:
                        response = "No"
                    else:
                        response = "Yes"
                else:
                    response = "No"
                get_id = get_host_record(host_ip)
                svc_db.update_or_insert(
                    f_hosts_id=get_id.id, f_proto='ICMP', f_number='0', f_status=response, f_name=ICMP_type
                )
                db.commit()

def clean_html(htmldata):
    """Cleans up the HTML using lxml.html clean_html for now."""

    try:
        from lxml.html.clean import clean_html
    except ImportError:
        log("You don't have lxml installed", logging.ERROR)
        return htmldata

    if htmldata is None:
        return htmldata
    newdata = clean_html(htmldata)
    newdata = newdata.replace('\n', ' ')
    newdata = newdata.replace('<div>', '')
    newdata = newdata.replace('</div>', '')
    #newdata = re.compile('\s*\n\s*').sub('\n', newdata)

    return newdata

def clean_html(htmldata):
    """Cleans up the HTML using lxml.html clean_html for now."""
    import re
    try:
        from lxml.html.clean import clean_html
    except ImportError:
        log("You don't have lxml installed", logging.ERROR)
        return htmldata

    if htmldata is None:
        return htmldata
    newdata = clean_html(htmldata)
    newdata = newdata.replace('\n', ' ')
    newdata = newdata.replace('<div>', '')
    newdata = newdata.replace('</div>', '')
    newdata = newdata.replace('\t', '')         # tabs? not needed, no never
    newdata = re.sub(' +', ' ', newdata)

    return newdata

def process_hydra(line):
    """
    Process a hydra line and return a dictionary:

    { 'ip'  : ip address
      'port': port info - can be port # or module name
      'user': username,
      'pass': password,
      'hash': ntlm hash if smbnt hash used
      'msg' : status message
    }
    """
    # line: [22][ssh] host: 1.1.1.1   login: username   password: pw1234
    retval = {}
    try:
        data = line.split()
    except Exception, e:
        log("Error processing hydra line: %s -- %s" % (e, line), logging.ERROR)
        return retval

def process_password_file(pw_file=None, pw_data=None, file_type=None, source=None):
    """
    Process a password file and return a dictionary fit for t_accounts

    file_type values:

      ('PWDUMP', 'MSCa$h Dump', 'UNIX Passwd', 'UNIX Shadow', 'Medusa', 'Hydra', 'Username:Password', 'AccountDB')
    """
    import fileinput

    accounts = {}
    if pw_file is not None:
        try:
            pw_data = []
            for line in fileinput.input(files=pw_file):
                pw_data.append(line)
        except IOError, e:
            log("Error opening %s: %s" % (pw_file, e), logging.ERROR)
            return accounts

def process_mass_password(pw_file=None, pw_type=None, message=None, proto=None, portnum=None, add_hosts=False, user_id=1):
    """
    Process a medusa/hydra mass password run
    """
    import fileinput

    db = current.globalenv['db']
    cache = current.globalenv['cache']

    added = 0
    updated = 0
    new_hosts = 0
    ip_dict = {}
    if pw_file is not None:
        try:
            fIN = fileinput.input(files=pw_file)
        except IOError, e:
            log("Error opening %s: %s" % (pw_file, e), logging.ERROR)
            return "Error opening %s: %s" % (pw_file, e)

def process_cracked_file(pw_file=None, file_type=None, message=""):
    """
    Process a file of cracked passwords and update the cleartext with
    the new results.

    :param pw_file: Filename to process
    :param file_type: String of a file type
    :param message: Message string to add to f_message field
    """
    import fileinput

    db = current.globalenv['db']

    if pw_file is not None:
        try:
            fIN = fileinput.input(files=pw_file)
        except IOError, e:
            log("Error opening %s: %s" % (pw_file, e), logging.ERROR)
            return "Error opening %s: %s" % (pw_file, e)

def process_medusa(line):
    """
    Process a medusa line and return a dictionary:

    { 'ip'  : ip address
      'port': port info - can be port # or module name
      'user': username,
      'pass': password,
      'hash': ntlm hash if smbnt hash used
      'msg' : status message
    }
    """
    retval = {}

    try:
        data = line.split()
    except Exception, e:
        log("Error processing medusa line: %s -- %s" % (e, line), logging.ERROR)
        return retval

    def _update_or_insert(self, **fields):
        """
        Our own update_or_insert routine

        :param fields: Matching db.t_services fields (f_proto, f_number, etc)
        :returns: t_services record id
        """
        if not fields['f_proto'] or not fields['f_number'] or not fields['f_hosts_id']:
            log("No protocol, number or hosts_id sent", logging.ERROR)
            return None

        svc_id = self.svc_db.update_or_insert(**fields)
        if not svc_id:
            # update_or_insert will not return an id if a record is updated.
            record = self._get_record(**fields)
            if record:
                svc_id = record.id

        return svc_id