def prosite_to_grouped_re(pattern):
    """convert a valid Prosite pattern into an re with groups for each term"""
    flg = (pattern[:2] == "[<")
    s = string.replace(pattern, "{", "[^")
    # Don't delete the "-" characters: use them to place the ()s
    s = string.translate(s, _prosite_trans, ".")

    # Get the [< and >] terms correct
    if flg:
        i = string.index(s, "]")
        s = "(?:^|[" + s[2:i] + "])" + s[i+1:]
    if s[-2:] == "$]":
        i = string.rindex(s, "[")
        s = s[:i] + "(?:" + s[i:-2] + "]|$)"
    if s[-3:] == "$]$":
        i = string.rindex(s, "[")
        s = s[:i] + "(?:" + s[i:-3] + "]|$)$"

    # Watch out for unescaped < and > terms
    if s[:1] == "^":
        s = "^(" + s[1:]
    else:
        s = "(" + s
    if s[-1:] == "$":
        s = s[:-1] + ")$"
    else:
        s = s + ")"

    return string.replace(s, "-", ")(")

def establish_sqlite_connection(address, profile):
    """Establish an SQLite connection."""
    
    import sys
    i = ""
    if sys.platform != "win32":
        try:
            i = string.rindex(address, "/")
        except ValueError:
            print ValueError
    else:
        try:
            i = string.rindex(address, "\\")
        except ValueError:
            return ValueError
    name = address[i:]
    address = address[:i+1]
    #check if already connected to database
    dbID = profile.conn_not_exist('sqlite', name, address)
    if dbID != False:
        connID = datahandler.ConnectionManager.create_new_data_connection(u"sqlite", address, name, dbID = dbID)
    else:
        connID = datahandler.ConnectionManager.create_new_data_connection(u"sqlite", address, name)
    if connID != False:
        if datahandler.DataHandler.add(connID):
            pub.sendMessage('dataconnection.save', connID = connID, type = "sqlite", address = address, dbName = name)
            pub.sendMessage('database.added', connID)
            return True                
        else:
            return False
    else:
        return False

def uncompressdir(dir, androiddir, uncatdir, undecompdir):
    tarpath = dir + '/*.tar*'
    tarfiles = glob.glob(tarpath)
    for onetar in tarfiles:
        periodindex = string.index(onetar, ".tar")
        lastslashindex = string.rindex(onetar, "/")
        tarname = onetar[lastslashindex+1 : periodindex]
        nowtarpath = dir + '/' + tarname
        if not os.path.exists(nowtarpath):
            os.makedirs(nowtarpath)
        tar = tarfile.open(onetar, 'r')
        for item in tar:
            tar.extract(item, nowtarpath)
        categorizedir(nowtarpath, androiddir, uncatdir)

    zippath = dir + '/*.zip'
    zipfiles = glob.glob(zippath)
    for onezip in zipfiles:
        periodindex = string.index(onezip, ".zip")
        lastslashindex = string.rindex(onezip, "/")
        zipname = onezip[lastslashindex+1 : periodindex]
        nowzippath = dir + '/' + zipname
        if not os.path.exists(nowzippath):
            os.makedirs(nowzippath)
        fZip = open(onezip, 'rb')
        zip = zipfile.ZipFile(fZip)
        is_encpted = 0
        for zinfo in zip.infolist():
            is_encpted = zinfo.flag_bits & 0x1
            if is_encpted:
                break
        if is_encpted:
            passwd = 'infected666' + zipname[len(zipname) - 1]  # This is default password used, need change for other uses
            for item in zip.namelist():
                try:
                    zip.extract(item, nowzippath, passwd)    # Sometimes password is needed
                except RuntimeError as e:
                    if 'password' in e[0]:
                        passwd = 'infected'
                        try:
                            zip.extract(item, nowzippath, passwd)
                        except RuntimeError as e:
                            print 'nowzip',
                            print onezip
                            print 'RuntimeError in second trail e: ',
                            print e[0]
                            os.system("mv " + onezip + " " + undecompdir)
                            os.system("rm -rf " + nowzippath)
                            break
        else:
            for item in zip.namelist():
                zip.extract(item, nowzippath)

        categorizedir(nowzippath, androiddir, uncatdir)

def umlComment(witness):
    quotedelim = split(witness, '"');
    test = split(witness)[1]+"-"+split(witness)[3][0]
    term = ' '.join(quotedelim[1].split())
    if "uML type error" in witness:
        shouldbe = "type error"
    else:
        shouldbe = quotedelim[3]
    if "uncaught exception" in witness:
        got = "exception "+witness[rindex(witness, ' ')+1:-3]
    elif "CPU time" in witness:
        got = "CPU timeout"
    elif "signalled a bug in type inference" in witness:
        if "typed-untypeable" in witness:
            got = "signaled bug: "+quotedelim[3]
        else:
            got = "signaled bug: "+quotedelim[5]
    elif "typed-incorrectly" in witness:
        got = quotedelim[5]
    elif "did-not-type" in witness:
        got = "type error: "+quotedelim[5]
    elif "wrote the error message" in witness:
        if "typed-untypeable" in witness:
            got = "error: "+quotedelim[3]
        else:
            got = "error: "+quotedelim[5]
    elif "typed-untypeable" in witness:
        got = quotedelim[3]
    else:
        stderr.write('Warning: Unable to finish comment for "'+witness+'"\n')
        return ["Term "+term, "Is "+shouldbe]
    term     = re.sub(UML_PATTERN, '&\g<0>&', term)
    shouldbe = re.sub(UML_PATTERN, '&\g<0>&', shouldbe)
    got      = re.sub(UML_PATTERN, '&\g<0>&', got)
    return ["Test "+test,"Term "+term, "Is "+shouldbe, "Got "+got]

def prosite_to_re(pattern):
    """convert a valid Prosite pattern into an re string"""
    flg = (pattern[:2] == "[<")
    s = string.replace(pattern, "{", "[^")
    s = string.translate(s, _prosite_trans, "-.")
    # special case "[<" and ">]", if they exist
    if flg:
        i = string.index(s, "]")
        s = "(?:^|[" + s[2:i] + "])" + s[i+1:]
    if s[-2:] == "$]":
        i = string.rindex(s, "[")
        s = s[:i] + "(?:" + s[i:-2] + "]|$)"
    elif s[-3:] == "$]$":
        i = string.rindex(s, "[")
        s = s[:i] + "(?:" + s[i:-3] + "]|$)$"
    return s

        def scrapeHTML(self, htmlText):
                self.agentList = []
                scraper = BeautifulSoup()
                scraper.feed(htmlText)
                #~ print ">>>\n",

                list = scraper.fetch('a', {'href':'/$%'})
                #~ for l in list:
                        #~ print "AREF >>>", l

                #~ print "\n\n Second tests..."
                #~ print 'Fetch List...'
                for s in list:
                        s = str(s)
                        #~ print 's >>>', s
                        start = string.index(s, ">")
                        end = string.rindex(s, "<")
                        t = s[start+1:end]
                        #~ print "AREF...", t
                        self.agentList.append(str(t))
                        #~ alphabet = [
#~ 'Alpha','Beta','Gamma','Delta','Epsilon',
#~ 'Zeta','Eta','Theta','Iota','Kappa',
#~ 'Lambda','Mu','Nu','Xi','Omicron',
#~ 'Pi','Rho','Sigma','Tau','Upsilon',
#~ 'Phi','Chi','Psi','Omega'
#~ ]
                #~ for item in alphabet:
                        #~ self.agentList.append(item)
                return self.agentList

    def newProcess(self, sensor):
        # determine the executable
        mainFile = None
        for main in Sensor.VALID_MAINS:
            target = os.path.join(gettempdir(), "sonar", sensor.name, main)
            if os.path.exists(target):
                mainFile = main
                break

        # break if there is no main file
        if mainFile == None:
            print "missing main file for sensor %s" % (sensor.name)
            return

        # determine the executable (python, ..)
        executable = None
        main = None
        try:
            index = string.rindex(mainFile, ".")
            ending = mainFile[(index + 1) :]
            if ending == "py":
                executable = "python"
                main = "main.py"
            elif ending == "sh":
                executable = "bash"
                main = "main.sh"
            elif ending == "exe":
                executable = None
                main = "main.exe"
        except ValueError:
            executable = None
            main = None

        # create a new process
        try:
            path = os.path.join(gettempdir(), "sonar", sensor.name, main)

            # configure executable and main file
            if executable is None:
                executable = [path, sensor.name]
            else:
                executable = [executable, path, sensor.name]

            # check if the sensor configuration has parameters
            if sensor.settings.parameters is not None:
                paramLen = len(sensor.settings.parameters)
                if paramLen > 0:
                    print "sensor parameter exists, appending same as command line arguments"
                    for parameter in sensor.settings.parameters:
                        paramValue = parameter.key + "=" + parameter.value
                        executable.append(paramValue)

            process = Popen(executable, stdout=PIPE, bufsize=1, universal_newlines=True)

            print "PID %i" % (process.pid)
            return process
        except Exception, e:
            print "error starting process: %s" % (e)
            return None

def split_package(modulename):
    """Splits a module name into package, and module"""
    from string import rindex
    try:
	ix = rindex(modulename, '.')
	return modulename[:ix], modulename[ix+1:]
    except ValueError:
	return None, modulename

 def calcOutputFilename(self, inputFilename):
    try:
       posn = string.rindex(inputFilename, '.')
       cut = inputFilename[:posn]
    except:
       # no '.', so use the whole string
       cut = inputFilename
    return cut + self.fileExtension()

def wrap(text, width, font):
    """Wrap a line of text, returning a list of lines."""

    lines = []
    while text:
        if font.size(text)[0] <= width: return lines + [text]
        try:
            i = string.rindex(text, ' ')
            while font.size(text[:i])[0] > width:
                i = string.rindex(text, ' ', 0, i)
        except ValueError:
            i = len(text)-1
            while font.size(text[:i])[0] > width and i: i = i - 1
            if not i: raise ValueError, 'width %d too narrow' % width

        lines.append(text[:i])
        text = string.lstrip(text[i:])

    return lines

def _create_ob_from_factory(c, id, file, path):
    try:
        i = string.rindex(c, '.')
        m, c = c[:i], c[i+1:]
        c = getObject(m, c)
        f = c()
        ob = _wrap_ob(f(id, file), path)
        ob.__factory = f
        return ob
    except: pass

def _create_ob_from_function(c, id, file, path):
    try:
        i = string.rindex(c, '.')
        m, c = c[:i], c[i+1:]
        m = __import__(m, globals(), locals(), (c,))
        c = getattr(m, c)
        f = getattr(c, 'createSelf').im_func
        if f.func_code.co_varnames == ('id', 'file'):
            return _wrap_ob(f(id, file), path)
    except: pass

    def process_request(self, req):
        htdocs_dir = self.config.get('docs', 'htdocs_dir')

        # Handle processing of /pdf/somefilename requests
        if (string.find(req.path_info, "docs/pdf/") != -1):
            pdf_file_name = (string.split(req.path_info, '/')[3])
            req.send_file(htdocs_dir + '/pdf/' + pdf_file_name)
            return None

        elif (string.find(req.path_info, "docs/html/") != -1):
            html_file_name = req.path_info[string.rindex(req.path_info, 'docs/html/') + 10:]
            if html_file_name.endswith('.html'):
                html = codecs.open(htdocs_dir + '/html/' + html_file_name, 'r', 'utf-8', 'replace')
                content = re.compile(r'^.*<body >\s*<p>\s*(.*)<p>\s*<br>\s*<hr>\s*<address>.*$', re.MULTILINE | re.IGNORECASE | re.DOTALL).match(html.read())
                html.close()
                if content is not None:
                    req.hdf.set_unescaped('content', content.group(1))
                else:
                    req.hdf['content'] = 'No content'
                return 'htmldoc.cs', None
            else:
                req.send_file(htdocs_dir + '/html/' + html_file_name)
                return None

        elif (string.find(req.path_info, "docs/irc/") != -1):
            irc_file_name = (string.split(req.path_info, '/')[3])
            irc = codecs.open(htdocs_dir + '/irc/' + irc_file_name, 'r', 'utf-8', 'replace')
            content = ''
            try:
                for line in irc:
                    content += wiki_to_html(line, self.env, req)
            finally:
                irc.close()
            if content is not None:
                req.hdf.set_unescaped('content', content)
            else:
                req.hdf['content'] = 'No content'
            return 'ircdoc.cs', None

        # Handle the default case
        else:
            pdfs_dir = htdocs_dir + '/pdf/'
            pdfs = filter(os.listdir(pdfs_dir), "*.pdf")
            pdfs.sort()
            pdfs = map(lambda x: x.rstrip('.pdf'), pdfs)
            req.hdf['pdfs'] = pdfs

            irc_dir = htdocs_dir + '/irc/'
            irc = filter(os.listdir(irc_dir), "lifecyclemanager-dev.log.*")
            irc.sort()
            req.hdf['irc'] = irc

            add_stylesheet(req, 'hw/css/docs.css')
            return 'docs.cs', None

    def readValueFromHandle(self, handle):
        raw = self.readRawHandle(handle).strip()

        try:
            value = raw[string.rindex(raw, "Characteristic value/descriptor:") + 33:]
        except(ValueError):
            print "Error reading value from Handle: " + handle
            print "[ERROR] " + raw
            sys.exit()

        return value

    def readValueFromUUID(self, uuid):
        raw = self.readRawUUID(uuid).strip()

        try:
            value = raw[string.rindex(raw, "value:") + 7:]
        except(ValueError):
            print "Error reading value from UUID: " + str(uuid)
            print "[ERROR] " + raw
            sys.exit()

        return value

    def unsubscribe(self, kn_route_location, statushandler = None):
        """
            unsubscribe - Remove an entry from the dispatch table,
            then enqueue "route" request to submit to the server.

            unsubscribe(kn_route_location, [status-handler])

            Inputs: a route URI, a status handler.
            Output: none.

            What it does:
                1. Remove any corresponding dispatch table entries.
                2. Parse the route URI into source topic and route ID.
                   If parsing failed, skip steps 3 and 4.
                3. Build a "route" request using the source topic,
                   route ID, and empty string as the destination.
                4. Enqueue request.
        """

        parts = urlparse.urlparse(kn_route_location)

        end_of_topic = -1

        try:
            end_of_topic = string.rindex(parts[2], self._KN_ROUTES_)
        except ValueError:
            if statushandler:
                statushandler.onStatus(
                    {
                    "status" : "400 Bad Request",
                    "kn_payload" :
                    "Client will not delete a route without the magic '%s' substring."
                    % self._KN_ROUTES_
                    })
                return

        kn_id = unicode(
            urllib.unquote(parts[2][end_of_topic + len(self._KN_ROUTES_) : ]),
            "UTF-8", "replace")

        parts = parts[:2] + (parts[2][ : end_of_topic ],) + parts[3:]

        requestMessage = {
            "kn_from" : urlparse.urlunparse(parts),
            "kn_to" : "",
            "do_method" : "route",
            "kn_id" : kn_id,
            "kn_expires" : "+5"
            }

        requestMessage["kn_from"] = canonicalizeTopic(
            self.getServerURL(), requestMessage["kn_from"])

        self.enqueue(requestMessage, statushandler)

def getMajorMinor(deviceName, dmsetupLs):
    """
    Given output of dmsetup ls this will return
    themajor:minor (block name) of the device deviceName
    """

    startingIndex = string.rindex(dmsetupLs, deviceName) + len(deviceName)
    endingIndex = string.index(dmsetupLs[startingIndex:], "\n") + startingIndex
    # trim the preceding tab and ()'s
    newStr = dmsetupLs[startingIndex + 2: endingIndex - 1]
    return newStr

 def updatepos(self, i, j):
     if i >= j:
         return j
     rawdata = self.rawdata
     nlines = string.count(rawdata, "\n", i, j)
     if nlines:
         self.lineno = self.lineno + nlines
         pos = string.rindex(rawdata, "\n", i, j) # Should not fail
         self.offset = j-(pos+1)
     else:
         self.offset = self.offset + j-i
     return j

 def newProcess(self, name):
     # determine the executable
     mainFile = None
     for main in ProcessLoader.VALID_MAINS:
         target = os.path.join(tempfile.gettempdir(), 'relay', name, main)
         if os.path.exists(target):
             mainFile = main
             break
         
         
     # break if there is no main file
     if mainFile == None:
         print 'missing main file for sensor %s' % (name)
         return None
     
     # determine the executable (python, ..)
     executable = None
     main = None
     try:    
         index = string.rindex(mainFile, '.')
         ending = mainFile[(index + 1):]
         if ending == 'py':
             executable = 'python'
             main = 'main.py'
         elif ending == 'sh':
             executable = 'bash'
             main = 'main.sh'
         elif ending == 'exe':
             executable = None
             main = 'main.exe'
     except ValueError:
         executable = None
         main = None
         return None
     
     # create a new process 
     try:
         path = os.path.join(tempfile.gettempdir(), 'relay', name, main)
         cwd = os.path.join(tempfile.gettempdir(), 'relay', name)
         
         # configure executable and main file
         if executable is None:
             executable = [path, name]
         else:
             executable = [executable, path, name]
         
         process = Popen(executable, stdout=PIPE, bufsize=0, universal_newlines=True, cwd=cwd)
         
         print 'PID %i' % (process.pid)
         return process
     except Exception as e:
         print 'error starting process %s' % (e)
         return None

 def pull(self, sources_dir):
     #Download the list of sources.
     package_file_path = self.downloadFile(sources_dir, 'sha1-all')
     with open(package_file_path) as f:
         packages = f.readlines()
     for package in packages:
         i = string.rindex(package, '-')
         package_name = string.strip(package[:i])
         package_sha1 = string.strip(package[i+1:])
         package_path = self.downloadFile(sources_dir, package_name, package_sha1)
         if package_sha1 != self.getFileHash(package_path):
             raise Exception('Invalid sha1 for package %s' % package_name)