def longls(self,path='/', dirs = None):
        dbh=DB.DBO(self.case)
        if self.isdir(path):
            ## If we are listing a directory, we list the files inside the directory            
            if not path.endswith('/'):
                path=path+'/'

            where = DB.expand(" path=%r " ,path)
        else:
            ## We are listing the exact file specified:
            where = DB.expand(" path=%r and name=%r", (
                FlagFramework.normpath(posixpath.dirname(path)+'/'),
                posixpath.basename(path)))
                   
        mode =''
        if(dirs == 1):
            mode=" and file.mode like 'd%'"
        elif(dirs == 0):
            mode=" and file.mode like 'r%'"

        dbh.execute("select * from file where %s %s", (where, mode))
        result = [dent for dent in dbh]

        for dent in result:
            if dent['inode']:
                dbh.execute("select * from inode where inode = %r", dent['inode'])
                data = dbh.fetch()
                if data:
                    dent.update(data)

        return result

            def pane_cb(path,tmp):
                query['order']='Filename'

                ## If we are asked to show a file, we will show the
                ## contents of the directory the file is in:
                fsfd = FileSystem.DBFS( query["case"])
                if not fsfd.isdir(path):
                    path=os.path.dirname(path)

                tmp.table(
                    elements = [ InodeIDType(case=query['case']),
                                 FilenameType(basename=True, case=query['case']),
                                 DeletedType(),
                                 IntegerType('File Size','size'),
                                 TimestampType('Last Modified','mtime'),
                                 StringType('Mode','mode', table='file') ],
                    table='inode',
                    where=DB.expand("file.path=%r and file.mode!='d/d'", (path+'/')),
                    case=query['case'],
                    pagesize=10,
                    filter="filter2",
                    )

                target = tmp.defaults.get('open_tree','/')
                tmp.toolbar(text=DB.expand("Scan %s",target),
                            icon="examine.png",
                            link=query_type(family="Load Data", report="ScanFS",
                                            path=target,
                                            case=query['case']), pane='popup'
                            )

    def startup(self):
        print "Checking schema for compliance"
        ## Make sure that the schema conforms
        dbh = DB.DBO()
        dbh.execute("select value from meta where property='flag_db'")
        DB.check_column_in_table(None, 'sql_cache', 'status',
                                 'enum("progress","dirty","cached")')
        for row in dbh:
            try:
                DB.check_column_in_table(row['value'], 'sql_cache', 'status',
                                         'enum("progress","dirty","cached")')
            except: continue

        ## Check the schema:
        dbh.check_index("jobs", "state")
        DB.check_column_in_table(None, 'jobs', 'priority', 'int default 10')
        DB.check_column_in_table(None, 'jobs', 'pid', 'int default 0')
        DB.check_column_in_table(None, 'jobs', 'when_valid',
                                 'TIMESTAMP ON UPDATE CURRENT_TIMESTAMP NOT NULL')

        ## Check for the high_priority_jobs table (its basically
        ## another jobs table for high priority jobs - so workers
        ## first check this table before the main jobs table).
        try:
            dbh.execute("select * from high_priority_jobs limit 1")
        except:
            dbh.execute("create table if not exists high_priority_jobs like jobs")
        
        ## Schedule the first periodic task:
        task = Periodic()
        task.run()

 def startup(self):
     ## These check that the schema is up to date
     DB.convert_to_unicode(None, 'dictionary')
     dbh=DB.DBO()
     dbh.execute("desc dictionary")
     for row in dbh:
         if row['Field'] == 'word':
             if not 'varbinary' in row['Type']:
                 dbh.execute("alter table dictionary modify word VARBINARY(250)")
                 break

    def __str__(self):
        postfix = ''
        ## Some tags are never allowed to be outputted
        if self.name not in self.allowable_tags:
            if self.name in self.forbidden_tag:
                return ''
            #print "Rejected tag %s" % self.name
            return self.innerHTML()

        if self.name == 'head':
            self.children = [self.header,] + self.children
        elif self.name =='body':
            self.children = [self.body_extra, ] + self.children

        ## Frames without src are filtered because IE Whinges:
        if self.name == 'iframe' and 'src' not in self.attributes:
		return ''

        attributes = "".join([" %s='%s'" % (k,v) for k,v \
                              in self.attributes.items() if k in \
                              self.allowable_attributes])

	if 'style' in self.attributes:
            attributes += ' style=%r' % self.css_filter(self.attributes['style'] or '')

        if 'http-equiv' in self.attributes:
            if self.attributes['http-equiv'].lower() == "content-type":
                ## PF _always_ outputs in utf8
                attributes += ' http-equiv = "Content-Type" content="text/html; charset=UTF-8"'
                
        if 'src' in self.attributes:
            attributes += ' src=%s' % self.resolve_reference(self.attributes['src'])

        try:
            if 'href' in self.attributes:
                if self.name == 'link':
                    attributes += " href=%s" % self.resolve_reference(self.attributes['href'], 'text/css')
                else:
                    attributes += DB.expand(' href="javascript: alert(%r)"',
                                            iri_to_uri(DB.expand("%s",self.attributes['href'])[:100]))
                    postfix = self.mark_link(self.attributes['href'])
        except: pass
        
        ## CSS needs to be filtered extra well
        if self.name == 'style':
            return expand("<style %s>%s</style>" , (attributes,
                                             self.css_filter(self.innerHTML())))
        
        if self.type == 'selfclose':
            return expand("<%s%s/>%s" , (self.name, attributes, postfix))
        else:
            return expand("<%s%s>%s</%s>%s", (self.name, attributes,
                                            self.innerHTML(),
                                            self.name,postfix))

    def execute(self):
        for iosource in self.args:
            dbh = DB.DBO(self.environment._CASE)
            dbh2 = dbh.clone()
            dbh.delete('inode', where=DB.expand("inode like 'I%s|%%'", iosource))
            dbh.execute("select * from filesystems where iosource = %r", iosource)
            for row in dbh:
                dbh2.delete('file', where=DB.expand("path like '%s%%'", iosource))

            dbh.delete("iosources", where=DB.expand("name=%r", iosource))
            yield "Removed IOSource %s" % iosource

    def run(self, case, inode_id, *args):
        global INDEX
        if not INDEX: reindex()

        try:
            desired_version = int(args[0])
        except:
            desired_version = INDEX_VERSION

        ## Did they want a detailed index or a unique index?
        unique = desired_version < 2**30
        
        ## In unique mode we want to generate one hit per scan job per
        ## word
        if unique:
            INDEX.clear_set()

        pyflaglog.log(pyflaglog.VERBOSE_DEBUG, "Indexing inode_id %s (version %s)" % (inode_id, desired_version))
        fsfd = FileSystem.DBFS(case)
        fd = fsfd.open(inode_id=inode_id)
        buff_offset = 0
        dbh = DB.DBO(case)

        ## Clear old hits:
        dbh.check_index("LogicalIndexOffsets", "inode_id")
        dbh.delete("LogicalIndexOffsets", where=DB.expand("inode_id = %r",
                                                          inode_id))

        ## Get ready for scan
        dbh.mass_insert_start("LogicalIndexOffsets")

        while 1:
            data = fd.read(1024*1024)
            if len(data)==0: break

            for offset, matches in INDEX.index_buffer(data, unique = unique):
                for id, length in matches:
                    dbh.mass_insert(
                        inode_id = inode_id,
                        word_id = id,
                        offset = offset + buff_offset,
                        length = length)

            buff_offset += len(data)

        dbh.mass_insert_commit()
        
        ## Update the version
        dbh.update("inode",
                   where = DB.expand('inode_id = %r', inode_id),
                   version = desired_version)

        def pane_cb(path, result):
            if not path.endswith('/'): path=path+'/'
                
            result.heading("Path is %s" % path)
            case = query['case']
            dbh = DB.DBO(case)
            fsfd = Registry.FILESYSTEMS.dispatch(query['fstype'])(case)
            ## Try to see if the directory is already loaded:
            dbh.execute("select * from file where path=%r and name=''", path)
            if not dbh.fetch():
                fsfd.load(mount_point = query['mount_point'], iosource_name= query['iosource'],
                          directory = path)

            ## Now display the table
            result.table(
                elements = [ InodeIDType(case=query['case']),
                             FilenameType(case=query['case']),
                             DeletedType(),
                             IntegerType(name='File Size',column='size'),
                             TimestampType(name = 'Last Modified',column = 'mtime'),
                             ],
                table='inode',
                where=DB.expand("file.path=%r and file.mode!='d/d'",(path)),
                case = query['case'],
                pagesize=10,
                )

def drop_table(case, name):
    """ Drops the log table tablename """
    if not name: return
    
    dbh = DB.DBO(case)
    pyflaglog.log(pyflaglog.DEBUG, "Dropping log table %s in case %s" % (name, case))

    dbh.execute("select * from log_tables where table_name = %r limit 1" , name)
    row = dbh.fetch()

    ## Table not found
    if not row:
        return
    
    preset = row['preset']

    ## Get the driver for this table:
    log = load_preset(case, preset)
    log.drop(name)
    
    ## Ask the driver to remove its table:
    dbh.delete("log_tables",
               where= DB.expand("table_name = %r ", name));

    ## Make sure that the reports get all reset
    FlagFramework.reset_all(family='Load Data', report="Load Preset Log File",
                                       table = name, case=case)

def get_factories(case, scanners):
    """ Scanner factories are obtained from the Store or created as
    required. Scanners is a list in the form case:scanner
    """
    ## Ensure dependencies are satisfied
    scanners = ScannerUtils.fill_in_dependancies(scanners)

    ## First prepare the required factories:
    result = []
    for scanner in scanners:
        key = DB.expand("%s:%s", (case, scanner))
        try:
            f = factories.get(key)
        except KeyError:
            try:
                cls = Registry.SCANNERS.dispatch(scanner)
            except:
                # pyflaglog.log(pyflaglog.WARNING, "Unable to find scanner for %s", scanner)
                continue

            # Instatiate it:
            import pyflag.FileSystem as FileSystem

            f = cls(FileSystem.DBFS(case))

            ## Initialise it:
            f.prepare()

            ## Store it:
            factories.put(f, key=key)

        result.append(f)

    return result

    def explain(self, query, result):
        name = self.fd.name
        ## Trim the upload directory if present
        if name.startswith(config.UPLOADDIR):
            name = name[len(config.UPLOADDIR) :]

        result.row("Filename", DB.expand("%s", name), **{"class": "explainrow"})

    def _make_sql(self, query, ordering=True):
        """ Calculates the SQL for the table widget based on the query """
        ## Calculate the SQL
        query_str = "select "
        try:
            self.order = int(query.get('order',self.order))
        except: self.order=0

        try:
            self.direction = int(query.get('direction',self.direction))
        except: self.direction = 0

        total_elements = self.elements + self.filter_elements

        ## Fixup the elements - if no table specified use the global
        ## table - this is just a shortcut which allows us to be lazy:
        for e in total_elements:
            if not e.table: e.table = self.table
            if not e.case: e.case = self.case

        ## The columns and their aliases:
        query_str += ",".join([ e.select() + " as `" + e.name + "`" for e in self.elements ])
        
        query_str += _make_join_clause(total_elements)

        if self.where:
            w = ["(%s)" % self.where,]
        else:
            w = []
            
        for e in total_elements:
            tmp = e.where()
            if tmp: w.append(tmp)

        ## Is there a filter condition?
        if self.filter_str:
            filter_str = self.filter_str.replace('\r\n', ' ').replace('\n', ' ')
            filter_str = parser.parse_to_sql(filter_str, total_elements, ui=None)
            if not filter_str: filter_str=1
            
        else: filter_str = 1

        query_str += "where (%s and (%s)) " % (" and ".join(w), filter_str)

        if self.groupby:
            query_str += "group by %s " % DB.escape_column_name(self.groupby)
        elif self._groupby:
            query_str += "group by %s " % self.groupby
            
        ## Now calculate the order by:
        if ordering:
            try:
                query_str += "order by %s " % self.elements[self.order].order_by()
                if self.direction == 1:
                    query_str += "asc"
                else: query_str += "desc"
            except IndexError:
                pass

        return query_str

 def form(self, query, result):
     result.textfield("Inode ID", 'inode_id')
     dbh = DB.DBO(query['case'])
     try:
         result.selector("Table Name", 'table_name', DB.expand('select name as `key`,name as value from sqlite where inode_id=%r', query['inode_id']), case=query['case'])
     except KeyError, e:
         pass

def glob_sql(pattern):
    path,name = posixpath.split(pattern)

    if globbing_re.search(path):
        path_sql = "path rlike '^%s/?$'" % translate(path)
    else:
        ## Ensure that path has a / at the end:
        if not path.endswith("/"): path=path+'/'
        
        path_sql = "path='%s'" % path

    if globbing_re.search(name):
        name_sql = "name rlike '^%s$'" % translate(name)
    else:
        name_sql = DB.expand("name=%r", name)
    
    if name and path:
        sql = "select concat(path,name) as path from file where %s and %s group by file.path,file.name" % (path_sql,name_sql)
    elif name:
        sql = "select concat(path,name) as path from file where %s group by file.path,file.name" % name_sql
    elif path:
        #sql = "%s and name=''" % path_sql
        sql = "select path from file where %s group by file.path" % path_sql
    else:
        ## Dont return anything for an empty glob
        sql = "select * from file where 1=0"

    return sql

    def display(self,query,result):
        path=query['path']
        key=query['key']
        result.heading("Registry Key Contents")
        result.text(DB.expand("Key %s/%s:", (path,key)),style='red',font='typewriter')
        dbh=DB.DBO(query['case'])

        def hexdump(query,out):
            """ Show the hexdump for the key """
            dbh.execute("select value from reg where path=%r and reg_key=%r limit 1",(path,key))
            row=dbh.fetch()
            if row:
                HexDump(row['value'],out).dump()
            return out

        def strings(query,out):
            """ Draw the strings in the key """
            out.para("not implimented yet")
            return out

        def stats(query,out):
            """ display stats on a key """
            out.para("not implemented yet")
            return out

        result.notebook(
            names=["HexDump","Strings","Statistics"],
            callbacks=[hexdump,strings,stats],
            context="display_mode"
            )

    def execute(self):
        if len(self.args) < 2:
            yield self.help()
            return

        ## Try to glob the inode list:
        dbh = DB.DBO(self.environment._CASE)
        dbh.execute("select inode from inode where inode rlike %r", DB.glob2re(self.args[0]))
        pdbh = DB.DBO()
        pdbh.mass_insert_start("jobs")
        ## This is a cookie used to identify our requests so that we
        ## can check they have been done later.
        cookie = int(time.time())
        scanners = []
        for i in range(1, len(self.args)):
            scanners.extend(fnmatch.filter(Registry.SCANNERS.scanners, self.args[i]))

        scanners = ScannerUtils.fill_in_dependancies(scanners)

        for row in dbh:
            inode = row["inode"]
            pdbh.mass_insert(
                command="Scan", arg1=self.environment._CASE, arg2=row["inode"], arg3=",".join(scanners), cookie=cookie
            )

        pdbh.mass_insert_commit()

        ## Wait for the scanners to finish:
        if self.environment.interactive:
            self.wait_for_scan(cookie)

        yield "Scanning complete"

    def __init__(self, case, fd, inode):
        File.__init__(self, case, fd, inode)
        # strategy: must determine basepath from parent, get our path
        # from db and then return the file:

        ## Note this _must_ work because we can only ever be called on
        ## a mounted iosource - it is an error otherwise:
        basepath = fd.io.directory

        self.case = case
        dbh = DB.DBO(case)
        dbh.check_index("file", "inode")
        dbh.execute("select path,name from file where inode=%r limit 1", (inode))
        row = dbh.fetch()

        path = row["path"]
        mount_point = fd.io.mount_point
        ## Prune the path down to the mount point:
        if path[: len(mount_point)] != mount_point:
            raise RuntimeError(DB.expand("Something went wrong - %s should be mounted on %s", (path, mount_point)))

        path = path[len(mount_point) :]
        path = basepath + "/" + path + "/" + row["name"]
        if not path.startswith(posixpath.normpath(config.UPLOADDIR)):
            path = FlagFramework.sane_join(config.UPLOADDIR, path)

        if os.path.isdir(path):
            self.fd = StringIO.StringIO("")
        else:
            self.fd = open(path, "r")

        s = os.stat(path)
        self.size = s.st_size

 def pane_cb(path, result):
     tlds = path.split("/")
     try:
         result.defaults.set('filter', DB.expand('TLD = %r and "Content Type"  contains html',tlds[1]))
         Reports.CaseTableReports.display(self, query, result)
     except IndexError:
         result.para("Click on a TLD to view all URLs from that TLD")

 def finish(self):
     self.dbh.mass_insert_commit()
     ## Update the version
     self.dbh.update("inode",
                     where = DB.expand('inode_id = %r', self.inode_id),
                     version = INDEX_VERSION)
     
     del self.dbh

def drop_preset(preset):
    """ Drops the specified preset name """
    pyflaglog.log(pyflaglog.DEBUG, "Droppping preset %s" % preset)
    for case, table in find_tables(preset):
        drop_table(case, table)

    dbh = DB.DBO()
    if preset:
        dbh.delete("log_presets", where=DB.expand("name = %r",preset))