def remove_punct(text, marks=None, beginning_marks=None, trailing_marks=None):
    """
    Remove punctuation from ``text`` by replacing all instances of ``marks``,
    ``beginning_marks`` or ``trailing_marks`` with an empty string.
    Args:
        text (str): raw text
        marks (str): If specified, remove only the characters in this string,
            e.g. ``marks=',;:'`` removes commas, semi-colons, and colons.
        beginning_marks (str): If specified, remove only the characters in this
            string from the beginning of the text, e.g. ``marks='^'`` removes
            ``^`` from the beginning of text.
        trailing_marks (str): If specified, remove only the characters in this
            string from the end of the text, e.g. ``marks='%'`` removes ``%``
            from the beginning of text. If non the above is given, all punctuation
            marks are removed.
    Returns:
        str
    """
    # First off, replace dashes with white space: import-export banks --> import export banks
    text = re.sub('-', ' ', text, flags=re.UNICODE)
    text = re.sub('  ', ' ', text, flags=re.UNICODE).strip()
    if beginning_marks:
        text = re.sub('^[{}]+'.format(re.escape(beginning_marks)), '', text, flags=re.UNICODE)
    if trailing_marks:
        text = re.sub('$[{}]+'.format(re.escape(trailing_marks)), '', text, flags=re.UNICODE)
    if marks:
        return re.sub('[{}]+'.format(re.escape(marks)), '', text, flags=re.UNICODE)
    else:
        if isinstance(text, unicode_):
            return text.translate(PUNCT_TRANSLATE_UNICODE)
        else:
            return text.translate(None, PUNCT_TRANSLATE_BYTES)

def findAll(document, search_text, line_interval=None, case_sensitive=True, word=False):
    """
    Find all occurrences of a given search text (evt. regexp text).
    """
    match_pos = []
    if line_interval is None:
        line_interval = (1, document.numLines()+1)
    else:
        line_interval = (max(1, line_interval[0]), min(document.numLines()+1, line_interval[1]))

    flags = 0
    if not case_sensitive:
       flags = re.IGNORECASE

    if word:
        search_text = r'\b'+re.escape(search_text)+r'\b'
    else:
        search_text = re.escape(search_text)

    allMatches = re.compile(search_text, flags).finditer

    for line_num in range(*line_interval):
        line_text = document.lineText(line_num)
        match_pos.extend( ((line_num, m.start()+1, m.end()+1) for m in allMatches(line_text)) )

    return match_pos

    def cleanup(self):
        self.logger.info('Starting to clean up old Nuclide processes/files.')
        # TODO: Remove it after migration is complete.
        # For migration, stop the forever monitor processes of Nuclide server.
        # This does not stop existing Nuclide server processes themselves.
        # It just removes the monitor so that we can kill them on upgrade.
        for proc in ProcessInfo.get_processes(
                getpass.getuser(), '%s.*%s' %
                (re.escape('forever/bin/monitor'), re.escape('nuclide-main.js'))):
            self.logger.info('Stopping forever monitor process: %s' % proc)
            proc.stop()

        # Clean up multiple Nuclide processes on same port.
        # There should be no more than one on a given port.
        # TODO: log the error to analytics db.
        # { port1 => [proc1, proc2, ...], ...}
        server_proc_map = defaultdict(list)
        # Group the processes by port.
        for proc in NuclideServer.get_processes():
            port = int(proc.get_command_param('port'))
            server_proc_map[port].append(proc)
        for port in server_proc_map:
            if len(server_proc_map[port]) > 1:
                self.logger.warning(
                    'Multiple Nuclide processes on port %d. Something wrong. Clean them up...' %
                    port)
                for proc in server_proc_map[port]:
                    proc.stop()

        self.cleanup_certificates(CERTS_EXPIRATION_DAYS)
        self.logger.info('Finished cleaning up old Nuclide processes/files.')

 def ignore(self, path, exact=False):
     """Ignores a path."""
     if exact:
         path = "^{0}$".format(re.escape(path))
     else:
         path = re.escape(path)
     self._ignore.append(re.compile(path))

 def replace_section(self, rst, section_name, replacement, remove_header=False):
     if not len(replacement):
         replacement = u"\n"
     elif replacement[-1] != u"\n":
         replacement = u"%s\n" % replacement
     if remove_header:
         replacement = u"%s\n" % replacement
     else:
         replacement = u"\\1\n%s\n" % replacement
     regex = (ur"""(?msx)
     (\n
         %(section_name)s\n
         ([%(header_chars)s])\2[^\n]+\n
     ).*?\n
     (?=(?:
         ^[^\n]+\n
         \2\2\2
       |
         \Z
     ))
     """) % {
         'section_name': re.escape(section_name),
         'header_chars': re.escape('-#=.'),
     }
     return re.sub(regex, replacement, rst)

 def Filter(self, text, filter_type):
     '''
     Filters content of browser by various expressions (type of expression
     is defined by filter_type).
     '''
     if text == '':
         self.values = self.allvalues
     else:
         num = None
         if text.isdigit():
             num = int(text)
         if filter_type == 0:
             match = re.compile('.*%s.*' % re.escape(text), re.I)
         elif filter_type == 1:
             try:
                 match = re.compile(text, re.I)
             except:
                 raise FilterException('Failed to compile regexp')
         elif filter_type == 2:
             text = text.replace('*', '__SEARCH_ALL__')
             text = text.replace('?', '__SEARCH_ONE__')
             text = re.escape(text)
             text = text.replace('\\_\\_SEARCH\\_ALL\\_\\_', '.*')
             text = text.replace('\\_\\_SEARCH\\_ONE\\_\\_', '.')
             match = re.compile('.*%s.*' % text, re.I)
         else:
             raise Exception('Unsupported filter type %s!' % filter_type)
         self.values = [
             item for item in self.allvalues
             if Wammu.Utils.MatchesText(item, match, num)
         ]
     self.SetItemCount(len(self.values))
     self.RefreshView()
     self.ShowRow(0)

def isGoodResult(name, show, log=True, season=-1):
    """
    Use an automatically-created regex to make sure the result actually is the show it claims to be
    """

    all_show_names = allPossibleShowNames(show, season=season)
    showNames = map(sanitizeSceneName, all_show_names) + all_show_names
    showNames += map(unidecode, all_show_names)

    for curName in set(showNames):
        if not show.is_anime:
            escaped_name = re.sub('\\\\[\\s.-]', '\W+', re.escape(curName))
            if show.startyear:
                escaped_name += "(?:\W+" + str(show.startyear) + ")?"
            curRegex = '^' + escaped_name + '\W+(?:(?:S\d[\dE._ -])|(?:\d\d?x)|(?:\d{4}\W\d\d\W\d\d)|(?:(?:part|pt)[\._ -]?(\d|[ivx]))|Season\W+\d+\W+|E\d+\W+|(?:\d{1,3}.+\d{1,}[a-zA-Z]{2}\W+[a-zA-Z]{3,}\W+\d{4}.+))'
        else:
            escaped_name = re.sub('\\\\[\\s.-]', '[\W_]+', re.escape(curName))
            # FIXME: find a "automatically-created" regex for anime releases # test at http://regexr.com?2uon3
            curRegex = '^((\[.*?\])|(\d+[\.-]))*[ _\.]*' + escaped_name + '(([ ._-]+\d+)|([ ._-]+s\d{2})).*'

        if log:
            logger.log(u"Checking if show " + name + " matches " + curRegex, logger.DEBUG)

        match = re.search(curRegex, name, re.I)
        if match:
            logger.log(u"Matched " + curRegex + " to " + name, logger.DEBUG)
            return True

    if log:
        logger.log(
            u"Provider gave result " + name + " but that doesn't seem like a valid result for " + show.name + " so I'm ignoring it")
    return False

    def test_add_FailoverMORoute_smpps(self):
        rorder = '10'
        rtype = 'FailoverMORoute'
        cid1 = 'smppuser1'
        typed_cid1 = 'smpps(%s)' % cid1
        cid2 = 'smppuser2'
        typed_cid2 = 'smpps(%s)' % cid2
        fid = 'f1'
        _str_ = ['%s to 2 connectors:' % rtype, '\t- %s' % re.escape(typed_cid1), '\t- %s' % re.escape(typed_cid2)]

        # Add MORoute
        extraCommands = [{'command': 'order %s' % rorder},
                         {'command': 'type %s' % rtype},
                         {'command': 'connectors %s;%s' % (typed_cid1, typed_cid2)},
                         {'command': 'filters %s' % fid}]
        yield self.add_moroute('jcli : ', extraCommands)

        # Make asserts
        expectedList = _str_
        yield self._test('jcli : ', [{'command': 'morouter -s %s' % rorder, 'expect': expectedList}])
        expectedList = ['#Order Type                    Connector ID\(s\)                                  Filter\(s\)',
                        '#%s %s %s     <T>' % (
                        rorder.ljust(5), rtype.ljust(23), (re.escape(typed_cid1) + ', ' + re.escape(typed_cid2)).ljust(48)),
                        'Total MO Routes: 1']
        commands = [{'command': 'morouter -l', 'expect': expectedList}]
        yield self._test(r'jcli : ', commands)

def generate_manual_patterns_and_replacements():
    manual_replacement_file = open("lib/manual_replacement_library.txt", 'r')
    manual_mapping_0 = []
    manual_mapping_1 = []
    replacement_count=0
    for line in manual_replacement_file:
        #allow # to be a comment
        if(line[0]=='#' or line=='\n'):
            continue
        line_without_newline = remove_eol_pattern.sub('',line)
        line_split = line_without_newline.split("|")
        #An individual re can only hold 100 entities. So split if necessary.
        if(replacement_count<99):
            manual_mapping_0.append((line_split[0],line_split[1].decode('utf-8')))
        else:
            manual_mapping_1.append((line_split[0],line_split[1].decode('utf-8')))
        replacement_count+=1
        
    #multisub, but only done once for speed
    manual_pattern_0 = '|'.join('(%s)' % re.escape(p) for p, s in manual_mapping_0)
    substs_0 = [s for p, s in manual_mapping_0]
    manual_replacements_0 = lambda m: substs_0[m.lastindex - 1]
    manual_pattern_compiled_0 = re.compile(manual_pattern_0, re.UNICODE)
    
    manual_pattern_1 = '|'.join('(%s)' % re.escape(p) for p, s in manual_mapping_1)
    substs_1 = [s for p, s in manual_mapping_1]
    manual_replacement_1 = lambda n: substs_1[n.lastindex - 1]
    manual_pattern_compiled_1 = re.compile(manual_pattern_1, re.UNICODE)
    
    return {'patterns':(manual_pattern_compiled_0, manual_pattern_compiled_1),
            'replacements':(manual_replacements_0, manual_replacement_1)}

def replaceDeployFile(out_file,template_file,train_file,fix_layers=None):
    f=open(template_file,'rb');
    text=f.read()[:];
    f.close();
    

    text=text.replace('$TRAIN_TXT','"'+train_file+'"');
    if fix_layers is not None:
        start_excludes=[];
        for fix_layer_curr in fix_layers:
            starts = [match.start() for match in re.finditer(re.escape('name: "'+fix_layer_curr), text)]
            assert len(starts)==1;
            # start_excludes=starts[:];
            start_excludes.append(starts[0]);
        starts=[match.start() for match in re.finditer(re.escape('name: '), text)]
        starts=[idx for idx in starts if idx not in start_excludes];
        starts.sort();
        starts=starts[::-1];
        # starts=starts[1:];
        for start in starts:
            string_orig=text[start:];   
            string_orig=string_orig[:string_orig.index('\n')]
            # [:string_orig.rindex('"')+1]
            # print string_orig
            string_new=string_orig[:string_orig.rindex('"')]+'_fix"';
            # print string_new,string_orig
            text=text.replace(string_orig,string_new);


    f=open(out_file,'wb')
    f.write(text);
    f.close();    

def parse_route_template(template):
    rbuilder = ["^"]
    fbuilder = []
    position = 0
    schema = {}

    for match in template_var_re_finditer(template):
        param_name = match.group("name")
        param_type = match.group("type") or "id"
        # TODO: Handle KeyError, maybe we want to use a custom error here.
        param_formatchar, param_re, param_schema = _schema_map[param_type]
        schema[param_name] = param_schema

        rbuilder.append(re.escape(template[position:match.start()]))
        rbuilder.append(param_re.format(param_name))

        fbuilder.append(template[position:match.start()])
        fbuilder.append("{")
        fbuilder.append(param_name)
        fbuilder.append(":")
        fbuilder.append(param_formatchar)
        fbuilder.append("}")

        position = match.end()

    rbuilder.append(re.escape(template[position:]))
    rbuilder.append("$")
    fbuilder.append(template[position:])

    return (valid.Schema(schema),
            re.compile(u"".join(rbuilder)),
            u"".join(fbuilder).format)

def find_used_variables_in_text(variant, recipe_text, selectors=False):
    used_variables = set()
    recipe_lines = recipe_text.splitlines()
    for v in variant:
        all_res = []
        compiler_match = re.match(r'(.*?)_compiler(_version)?$', v)
        if compiler_match and not selectors:
            compiler_lang = compiler_match.group(1)
            compiler_regex = (
                r"\{\s*compiler\([\'\"]%s[\"\'][^\{]*?\}" % re.escape(compiler_lang)
            )
            all_res.append(compiler_regex)
            variant_lines = [line for line in recipe_lines if v in line or compiler_lang in line]
        else:
            variant_lines = [line for line in recipe_lines if v in line.replace('-', '_')]
        if not variant_lines:
            continue
        v_regex = re.escape(v)
        v_req_regex = '[-_]'.join(map(re.escape, v.split('_')))
        variant_regex = r"\{\s*(?:pin_[a-z]+\(\s*?['\"])?%s[^'\"]*?\}\}" % v_regex
        selector_regex = r"^[^#\[]*?\#?\s\[[^\]]*?(?<![_\w\d])%s[=\s<>!\]]" % v_regex
        conditional_regex = r"(?:^|[^\{])\{%\s*(?:el)?if\s*" + v_regex + r"\s*(?:[^%]*?)?%\}"
        # plain req name, no version spec.  Look for end of line after name, or comment or selector
        requirement_regex = r"^\s+\-\s+%s\s*(?:\s[\[#]|$)" % v_req_regex
        if selectors:
            all_res.extend([selector_regex])
        else:
            all_res.extend([variant_regex, requirement_regex, conditional_regex])
        # consolidate all re's into one big one for speedup
        all_res = r"|".join(all_res)
        if any(re.search(all_res, line) for line in variant_lines):
            used_variables.add(v)
    return used_variables

def patch_up_variable(origdata, data, origtype, var, ret):
    type = origtype
    var = re.sub(r"\s*=\s*[^;,\)]+", "", var)
    curlybracere = re.compile(r"\s*(\S+)\s*({})\s*(\S*)", re.MULTILINE)
    for var in var.split(","):
        var = var.strip()
        pat = r"%s\s*([^;{]*)%s\s*(%s)" % (re.escape(origtype), re.escape(var), _endpattern)
        end = re.search(pat, data)
        if end.group(2) == "[":
            type += re.search(r"([\[\]]+)", data[end.start():]).group(1)
        i = var.find("[]")
        if i != -1:
            type += var[i:]
            var = var[:i]
        if "<" in type and ">" in type:
            s = r"(%s.+%s)(const)?[^{};]*(%s)" % (type[:type.find("<")+1], type[type.find(">"):], var)
            regex = re.compile(s)
            match = None
            for m in regex.finditer(origdata):
                match = m
            type = match.group(1)
        match = curlybracere.search(var)
        if match:
            if match.group(3):
                var = match.group(3)
                type += " %s" % match.group(1)
            else:
                var = match.group(1)
        ret.append((type, var))

    def getValue(self, value, disable_quote):
        """Return the value of this parameter.

        params:
            value: text value
            disable_quote: is quote disabled?
        return value
        """
        # NULL value for any type
        if str(value).upper() == "NULL":
            if self.type.strip().lower() in ("text", "varchar", "character varying"):
                return value + "::text"
            elif self.type.replace(" ", "").lower() in ("text[]", "varchar[]"):
                return value + "::text[]"
            else:
                return value + "::" + self.type.strip()

        if re.search(r"ARRAY\[.*\]", value.upper()):
            value = re.sub("(?i)" + re.escape("NAN"), "'NAN'::double precision", value)
            value = re.sub("(?i)" + re.escape("Infinity"), "'Infinity'::double precision", value)
            if self.type.lower() != "text" and self.type.lower() != "varchar":
                return value + "::" + self.type
        # does the invoke need quote and does the value support quote itself?
        if disable_quote and not self.quote:
            return value
        if self.type.lower() in ("text", "varchar", "character varying") and value.lower() == "empty":
            return "''::text"  # empty string
        else:
            # return "'%s'::%s" % (value, self.type)
            # value = value.replace("ARRAY[","{")
            # value = value.replace("]","}")
            return "$_valString$%s$_valString$::%s" % (value, self.type)

    def test_realm_host_assignation(self):
        """ Test host realm assignation
        Test realms on each host

        :return: None
        """
        with pytest.raises(SystemExit):
            self.setup_with_file('cfg/realms/several_realms.cfg', 'cfg/realms/several_realms.ini')
        self.show_logs()
        assert not self.conf_is_correct

        self.assert_any_cfg_log_match(re.escape(
            "Configuration in hostgroup::in_realm2 is incorrect; "
        ))
        self.assert_any_cfg_log_match(re.escape(
            "host test_host3_hg_realm2 (realm: realm1) is not in the same realm than its hostgroup in_realm2 (realm: realm2)"
        ))

        # self.assert_any_cfg_log_match(re.escape(
        #     "hostgroup in_realm2 got the default realm but it has some hosts that are from different realms"
        # ))

        # Some error messages
        assert len(self.configuration_errors) == 3

        realm1 = self._arbiter.conf.realms.find_by_name('realm1')
        assert realm1 is not None
        realm2 = self._arbiter.conf.realms.find_by_name('realm2')
        assert realm2 is not None

        host = self._arbiter.conf.hosts.find_by_name('test_host_realm1')
        assert realm1.uuid == host.realm

        host = self._arbiter.conf.hosts.find_by_name('test_host_realm2')
        assert realm2.uuid == host.realm

    def search(self, q, user):
        required_visibility = self.required_visibility(user)
        query_filter = {"path": self.mongo_path_prefix, "effectiveVisibility": {"$in": tuple(required_visibility)}}
        if q.startswith("album:"):
            album = q[len("album:") :]
            query_filter["type"] = "album"
            query_filter = {
                "$and": [
                    query_filter,
                    {
                        "$or": [
                            {"name": {"$regex": re.compile(re.escape(album), re.IGNORECASE)}},
                            {"title": {"$regex": re.compile(re.escape(album), re.IGNORECASE)}},
                        ]
                    },
                ]
            }
        elif q.startswith("tag:"):
            _id = Es.id_by_name(q[len("tag:") :])
            if _id is None:
                return
            query_filter["type"] = {"$ne": "album"}
            query_filter["tags"] = _id
        else:
            # do a full-text search
            for result in db.command(
                "text", "fotos", search=q, filter=query_filter, limit=96  # dividable by 2, 3 and 4
            )["results"]:
                yield entity(result["obj"])
            return

        # search for album or tag
        for o in fcol.find(query_filter).sort("date", -1):
            yield entity(o)

    def run(self):
        """
        Starts the robot's action.
        """
        # regular expression to find the original template.
        # {{vfd}} does the same thing as {{Vfd}}, so both will be found.
        # The old syntax, {{msg:vfd}}, will also be found.
        # The group 'parameters' will either match the parameters, or an
        # empty string if there are none.

        replacements = []
        exceptions = {}

        for old, new in self.templates.iteritems():
            if not pywikibot.getSite().nocapitalize:
                pattern = '[' + re.escape(old[0].upper()) + re.escape(old[0].lower()) + ']' + re.escape(old[1:])
            else:
                pattern = re.escape(old)
            pattern = re.sub(r'_|\\ ', r'[_ ]', pattern)
            templateRegex = re.compile(r'\{\{ *([Tt]emplate:|[mM][sS][gG]:)?' + pattern + r'(?P<parameters>\s*\|.+?|) *}}', re.DOTALL)

            if self.remove:
                replacements.append((templateRegex, ''))
            elif self.subst:
                replacements.append((templateRegex, '{{subst:' + old + '\g<parameters>}}'))
                exceptions['inside-tags']=['ref']
            else:
                replacements.append((templateRegex, '{{' + new + '\g<parameters>}}'))

        replaceBot = replace.ReplaceRobot(self.generator, replacements, exceptions, acceptall = self.acceptAll, addedCat=self.addedCat, editSummary=self.editSummary)
        replaceBot.run()

  def SetJavaAssertsEnabled(self, enable):
    """Sets or removes the device java assertions property.

    Args:
      enable: If True the property will be set.

    Returns:
      True if the file was modified (reboot is required for it to take effect).
    """
    # First ensure the desired property is persisted.
    temp_props_file = tempfile.NamedTemporaryFile()
    properties = ''
    if self._adb.Pull(LOCAL_PROPERTIES_PATH, temp_props_file.name):
      properties = file(temp_props_file.name).read()
    re_search = re.compile(r'^\s*' + re.escape(JAVA_ASSERT_PROPERTY) +
                           r'\s*=\s*all\s*$', re.MULTILINE)
    if enable != bool(re.search(re_search, properties)):
      re_replace = re.compile(r'^\s*' + re.escape(JAVA_ASSERT_PROPERTY) +
                              r'\s*=\s*\w+\s*$', re.MULTILINE)
      properties = re.sub(re_replace, '', properties)
      if enable:
        properties += '\n%s=all\n' % JAVA_ASSERT_PROPERTY

      file(temp_props_file.name, 'w').write(properties)
      self._adb.Push(temp_props_file.name, LOCAL_PROPERTIES_PATH)

    # Next, check the current runtime value is what we need, and
    # if not, set it and report that a reboot is required.
    was_set = 'all' in self.RunShellCommand('getprop ' + JAVA_ASSERT_PROPERTY)
    if was_set == enable:
      return False

    self.RunShellCommand('setprop %s "%s"' % (JAVA_ASSERT_PROPERTY,
                                              enable and 'all' or ''))
    return True

    def wait_for_load_finished_url(self, url, *, timeout=None,
                                   load_status='success'):
        """Wait until a URL has finished loading."""
        __tracebackhide__ = (lambda e: e.errisinstance(
            testprocess.WaitForTimeout))

        if timeout is None:
            if 'CI' in os.environ:
                timeout = 15000
            else:
                timeout = 5000

        # We really need the same representation that the webview uses in its
        # __repr__
        qurl = QUrl(url)
        if not qurl.isValid():
            raise ValueError("Invalid URL {}: {}".format(url,
                                                         qurl.errorString()))
        url = utils.elide(qurl.toDisplayString(QUrl.EncodeUnicode), 100)
        assert url

        pattern = re.compile(
            r"(load status for <qutebrowser\.browser\..* "
            r"tab_id=\d+ url='{url}/?'>: LoadStatus\.{load_status}|fetch: "
            r"PyQt5\.QtCore\.QUrl\('{url}'\) -> .*)".format(
                load_status=re.escape(load_status), url=re.escape(url)))

        try:
            self.wait_for(message=pattern, timeout=timeout)
        except testprocess.WaitForTimeout:
            raise testprocess.WaitForTimeout("Timed out while waiting for {} "
                                             "to be loaded".format(url))

 def _adjustParameter(paramString, parameter, newValue):
     retVal = paramString
     match = re.search("%s=(?P<value>[^&]*)" % re.escape(parameter), paramString)
     if match:
         origValue = match.group("value")
         retVal = re.sub("%s=[^&]*" % re.escape(parameter), "%s=%s" % (parameter, newValue), paramString)
     return retVal