def from_html(main_row, subtext_row):
        ''' Constructs Story from HN site markup elements.
        Arguments are <tr> elements obtained with BeautifulSoup.
        '''
        link = main_row.find_all('td')[2].a
        vote_td = main_row.find_all('td')[1]
        subtext = subtext_row.find('td', {'class': 'subtext'})
        comments_link = subtext.find('a', href=regex(r'item\?id\=.+'))
        not_job = bool(comments_link)

        story = {'title': link.text, 'url': link['href']}
        if not_job:
            points = cast(int, subtext.find('span', id=regex(r'score_\d+')
                                            ).text.split()[0], default=0)
            comments_count = cast(int, comments_link.text.split()[0],
                                  default=0)
            story.update({
                'author': subtext.find('a', href=regex(r'user\?id\=.+')).text,
                'points': points,
                'time': list(subtext.strings)[-2].replace('|', '').strip(),
                'comments_count': comments_count,
                'comments_url': comments_link['href'],
                'upvote_url': vote_td.find('a', id=regex(r'up_\d+'))['href'],
            })
            url = story['comments_url']
        else:
            story['time'] = subtext.text
            url = story['url']

        story['id'] = int(url[url.find('=')+1:])
        return Story(**story)

 def __init__(self, name, pathPatterns, filePatterns, all_conditions = False):
   self.name = name
   self.allcond = all_conditions
   self.fpat = []
   self.ppat = []
   for pat in filePatterns:
     self.fpat.append(regex(pat))
   for pat in pathPatterns:
     self.ppat.append(regex(pat))

 def test0(self):
     r'''
     `_feed` and `_expected_expected` has been manually prepared reading
     `_tree_template`. `expected` is programmatically prepared from
     _tree_template.
     '''
     from ... import TestTree
     from .....util import name_shift
     from .....lib import SimplestFindBy, Str, StringIO
     from re import compile as regex, escape
     b = StringIO()
     _tree_template.feed(b, 'p1.py\n', '', '0')
     _tree_template.feed(b, 'p1+/\n', '', '1')
     feed = b.getvalue()
     expected = _tree_template.expected()
     expected = tuple(expected)
     abs_dir = join(self.tmp, 'a')
     content = _content_feed
     abs_dir = TestTree(abs_dir, feed, _content_feed)
     replace, actual = name_shift(abs_dir)
     for s in replace:
         content = regex(r'\b' + escape(s[0]) + r'\b').sub(s[1], content)
     cmp = []
     find = SimplestFindBy(lambda prefix, infix, y: y.endswith(r'.py'))
     for y in find(Str(abs_dir), ''):
         with abs_dir.joinpath(y).open(r'rt') as istream:
             cmp.append(istream.read() == content)
     expected = [True] * len(cmp), expected
     actual = cmp, actual
     self.assertEqual(expected, actual)

def main():
    "Detect all orphan files and entries"
    g = Graph()
    for manifest in [MANIFEST_PATH, MANIFEST_SYNTAX_PATH,
                     MANIFEST_TURTLE_LDPATCH_PATH,]:
        g.load(manifest, format="turtle")
        LOG.debug("loaded %s", manifest)
    for manifest in [MANIFEST_TURTLE_PATH,]:
        if exists(manifest):
            g.load(manifest, format="turtle")
            LOG.debug("loaded %s", manifest)

    # looking for entries that are not part of an mf:entry list
    for subj, _, _ in g.triples((None, MF.action, None)):
        if not list(g.triples((None, RDF.first, subj))):
            print subj

    # looking for files that are not referenced
    FILTER = regex(r'.*\.(ttl|nt|ldpatch)$')
    for foldername in (TESTSUITE_PATH, TURTLE_PATH):
        for filename in listdir(foldername):
            if not FILTER.match(filename):
                continue
            if filename.startswith("manifest"):
                continue
            iri = p2i(join(foldername, filename))
            if not list(g.triples((None, None, iri))):
                print iri

    # checking that all entries have the correct name
    for subj, _, obj in g.triples((None, MF.name, None)):
        if subj.rsplit("#",1)[1] != unicode(obj):
            print "WRONG NAME for ", subj

 def test(self):
     from ....lib import Path, RegexFileEditor
     from re import compile as regex
     edit = RegexFileEditor(((regex(r'\ba\b'), 'x'), (regex(r'\b0\b'), '9')))
     a, b = (Path(self.tmp).joinpath(x) for x in ('a', 'b'))
     with a.open(r'wt') as ostream:
         ostream.write(r'abc.a')
     with b.open(r'wt') as ostream:
         ostream.write(r'012.0')
     for x in a, b:
         edit(x)
     with a.open(r'rt') as istream:
         a = istream.read()
     with b.open(r'rt') as istream:
         b = istream.read()
     expected = r'abc.x', r'012.9'
     actual = a, b
     self.assertEqual(expected, actual)

def _ed(substs, abs_dir):
    from ..lib import RegexFileEditor, SimplestFindBy
    from re import compile as regex, escape
    olds = (y[0] for y in substs)
    news = tuple(y[1] for y in substs)
    oldpats = tuple(regex(r'\b' + escape(y) + r'\b') for y in olds)
    find = SimplestFindBy(lambda prefix, infix, y: y.endswith(r'.py'))
    for y in find(abs_dir, ''):
        RegexFileEditor(zip(oldpats, news))(abs_dir.joinpath(y))

def readConfig():
    global notificationTimeout, errorTimeout, socketTimeout, \
           checkInterval, sources

    currentSource = None
    file = open(getenv('HOME') + '/.config/catfriend', 'r')
    re = regex("^\s*(?:([a-zA-Z_]+)(?:\s+(\S+))?\s*)?(?:#.*)?$")

    checks = []
    for source in sources:
        checks.append(MailSource(source))

    while True:
        line = file.readline()
        if not line: break
        res = re.match(line)
        if not res:
            return line

        res = res.groups()
        if res[0] is None: continue

        if res[0] == "notificationTimeout":
            notificationTimeout = int(res[1])
        elif res[0] == "errorTimeout":
            errorTimeout = int(res[1])
        elif res[0] == "socketTimeout":
            socketTimeout = int(res[1])
        elif res[0] == "checkInterval":
            checkInterval = int(res[1])
        elif res[0] == "host" or res[0] == "imap":
            if currentSource:
                sources.append(currentSource)
            currentSource = MailSource(res[1])
        elif currentSource is None:
            return line
        elif not res[1]:
            if res[0] == "nossl":
                currentSource.noSsl = True
            else:
                return line
        elif res[0] == "id":
            currentSource.id = res[1]
        elif res[0] == "user":
            currentSource.user = res[1]
        elif res[0] == "password":
            currentSource.password = res[1]
        elif res[0] == "cert_file":
            # ignored
            currentSource.cert_file = res[1]
        else:
            return line

    sources.append(currentSource)

def get_filter():
    """a filter which takes out only filenames which probably contain media"""
    extensions = ['avi', 'mpg', 'mpeg', 'mp4', 'mkv', 'ogv',
                  'flv', 'ogg', 'mov', 'mp3', 'ac3', 'rm', 'ram',
                  'wmv', '3gp', 'aac', 'asf', 'h263', 'webm',
                  'm4a', '3g2', 'mj2']
    regexstring = r'\.('
    for extension in extensions:
        regexstring = regexstring + extension + '|'
    regexstring = regexstring[:-1] + ')$'
    return regex(regexstring).search

    def __prepare_fields(self):

        from re import match as regex

        regstr = '(^`)([a-z]*)'

        for i in range(len(self.select_dict['fields'])):

            field, field_as = self.__ensure_field_as(self.select_dict['fields'][i])

            if not regex(regstr, self.select_dict['fields'][i]):
                self.select_dict['fields'][i] = '`a`.'+field+' AS '+field_as

	def is_non_proposal_filibuster(self, message):
		""" Parses the message, determines if it is a filibustering non-proposal (D: or :D:)

			:param message: Message to parse.
			:type message: str
			:returns: True if the message is 'D:', ':D:', or 'nick: D:', etc.
		"""

		npf_matcher = regex(r'<[^>]+> (?:(\S+)[:,] )?(:)?D:')
		result = npf_matcher.match(message)

		return False if result == None else True

	def is_ignored_message(self, message):
		""" Parses the message, determines if it should be ignored (does not reset votecount, is not a proposal)

			:param message: Message to parse.
			:type message: strings
			:returns: True if the message should be ignored when legislating.
		"""

		jpq_matcher = regex(r'\*\*\* (Joins|Parts|Quits)')
		result = jpq_matcher.match(message)

		return False if result == None else True

    def __init__(self, banned_words=BANNED_WORDS):
        from tools import AudioEffect, HiddenTextEffect, ExplicitTextEffect, PhonemicEffect, \
            VoiceEffect

        self.effects = {cls: [] for cls in
                        (AudioEffect, HiddenTextEffect, ExplicitTextEffect, PhonemicEffect, VoiceEffect)}
        self.connection_time = datetime.now()
        self.last_attack = datetime.now()  # any user has to wait some time before attacking, after entering the chan
        self.last_message = datetime.now()
        self.timestamps = list()
        self.has_been_warned = False # User has been warned he shouldn't flood
        self._banned_words = [regex(word) for word in banned_words]
        self.is_shadowbanned = False # User has been shadowbanned

    def _retrieve_user_info(self, page="/"):
        """ Gets HN user info from given page.
        A page is either an URL or BeautifulSoup object.
        Returns True of False, depending on whether user info
        could be found.
        """
        if isinstance(page, basestring):
            page = self._fetch_page(page)

        top_table = page.find("table").find("table")
        user_td = top_table.find_all("td")[-1]
        user_span = user_td.find("span", {"class": "pagetop"})
        user_link = user_span.find("a", href=regex(r"user\?id\=.+"))
        if not user_link:
            return False

        name = user_link.text
        points = regex(r"\((\d+)\)").search(user_span.text).group(1)
        if not (name or points):
            return False

        self.user_name = name
        self.user_points = points
        return True

 def __init__(self, e, v = None):
     if isinstance(e, Exception) and v is None:
         self._t = e.__class__
         self._match = lambda s: s.startswith(str(e))
     elif isinstance(e, type) and issubclass(e, Exception) and \
          ((v is None) or isinstance(v, str)):
         self._t = e
         if v is None:
             self._match = lambda s: True
         else:
             v = regex("^(?:{0:s})$".format(v))
             self._match = lambda s: v.match(s) is not None
     else:
         raise Exception("usage: with expected(Exception[, \"regex\"]): "
                         "or with expected(Exception(\"text\")):")

	def __init__(self, line):
		if regex(r"\n$", line):
			line = line[:-1]
		
		parts = line.split()

		self.operator = ""
		self.label = ""
		self.location = -1
		self.location_name = ""

		if len(parts) == 1:
			self.operator = parts[0]
		elif len(parts) == 2:
			if parts[0] in operators:
				self.operator = parts[0]

				if is_location(parts[1]):
					self.location = int(parts[1])
				else:
					self.location_name = parts[1]
			else:
				self.operator = parts[1]
				
				if self.operator == "DAT":
					self.location_name = parts[0]
					valid_location_names.append(self.location_name)
				else:
					self.label = parts[0]
		elif len(parts) == 3:
			self.operator = parts[1]
			
			if self.operator == "DAT":
				self.location_name = parts[0]
				memory[self.location_name] = int(parts[2])
				
				valid_location_names.append(self.location_name)
			else:
				self.label = parts[0]
				
				if is_location(parts[2]):
					self.location = int(parts[2])
				else:
					self.location_name = parts[2]

def _prefix_stem_suffix(from_, to, names):
    from re import compile as regex, escape
    sfrom = _Str(from_)
    from_ = sfrom.split('%')
    if len(from_) != 2:
        raise ValueError(from_)
    sto = _Str(to)
    to2 = sto.split('%')
    len_to = len(to2)
    if 2 < len_to:
        raise ValueError(to)
    rx = regex(escape(from_[0]) + r'(.+)' + escape(from_[1]))
    if len_to == 1:
        for _ in names:
            yield '', to, ''
        return
    to_pos, to_lineno = to.loc
    prefix = to.create(to2[0], to_pos, to_lineno)
    for e in names:
        stem = rx.match(_Str(e)).groups(1)[0]
        suffix = to.create(to2[1], to_pos + len(stem), to_lineno)
        stem = to.create(stem, to_pos + sto.index('%'), to_lineno)
        yield prefix, stem, suffix

def load_graph():
    """Load manifests and reports into a single graph, and return it"""

    g = Graph()
    g.bind("earl", EARL[""])
    g.bind("dc", DC[""])
    g.bind("mf", MF[""])
    g.bind("doap", "http://usefulinc.com/ns/doap#")
    g.bind("foaf", "http://xmlns.com/foaf/0.1/")

    for manifest_filename, manifest_iri in ALL_MANIFESTS_PATH_BASE:
        g.load(manifest_filename, format="turtle", publicID=manifest_iri)
        LOG.debug("loaded %s", manifest_filename)

    FILTER = regex(r'.*\.ttl$')
    for filename in listdir(REPORTS_PATH):
        if not FILTER.match(filename):
            continue
        report_path = join(REPORTS_PATH, filename)
        g.load(report_path, format="turtle",
               publicID=TESTSUITE_NS["reports/" + filename])
        LOG.debug("loaded %s", report_path)

    return g

from json_tools import prepare, field_by_path, list_field_paths
from re import compile as regex
from multiprocessing import Pool
from os import walk, makedirs, remove
from json import load, dump, loads
from parser_settings import files_of_interest
from utils import get_answer
from bisect import insort_left
from special_cases import specialSections

root_dir = "./assets"
prefix = "./translations"
texts_prefix = "texts"
sub_file = normpath(join(prefix, "substitutions.json"))

glitchEmoteExtractor = regex("^([In]{,3}\s?[A-Za-z]+\.)\s+(.*)")
glitchIsHere = regex("^.*[gG]litch.*")


def defaultHandler(val, filename, path):
  sec = ""
  for pattern in specialSections:
    if pattern.match(filename, path):
      sec = pattern.name
      break
  return [(sec, val, filename, path)]
  
def glitchDescriptionSpecialHandler(val, filename, path):
  ## Handles glitch utterances, and separates it to emote part and text part,
  ## then saves the parts in new paths to database
  ## See details in textHandlers description

    def __init__(self, import_name, **kwargs):
        self.cssFiles = []
        self.jsFiles = []
        self.regex = {
            'color': regex('^#(([0-9a-fA-F]{3})|([0-9a-fA-F]{6}))$'),
            'date': regex('^20[0-9]{2}-[0-9]{2}-[0-9]{2}$'),
            'hex16': regex('^([0-9a-fA-F])*$'),
            'sha256': regex('^([0-9a-fA-F]){64}$')
        }
        self.dhGen = int(2)
        self.dhGroup = {
            'modp1': 'ffffffffffffffffc90fdaa22168c234c4c6628b80dc1cd129024e088'
                + 'a67cc74020bbea63b139b22514a08798e3404ddef9519b3cd3a431b302b0'
                + 'a6df25f14374fe1356d6d51c245e485b576625e7ec6f44c42e9a63a3620f'
                + 'fffffffffffffff',
            'modp2': 'ffffffffffffffffc90fdaa22168c234c4c6628b80dc1cd129024e088'
                + 'a67cc74020bbea63b139b22514a08798e3404ddef9519b3cd3a431b302b0'
                + 'a6df25f14374fe1356d6d51c245e485b576625e7ec6f44c42e9a637ed6b0'
                + 'bff5cb6f406b7edee386bfb5a899fa5ae9f24117c4b1fe649286651ece65'
                + '381ffffffffffffffff',
            'modp5': 'ffffffffffffffffc90fdaa22168c234c4c6628b80dc1cd129024e088'
                + 'a67cc74020bbea63b139b22514a08798e3404ddef9519b3cd3a431b302b0'
                + 'a6df25f14374fe1356d6d51c245e485b576625e7ec6f44c42e9a637ed6b0'
                + 'bff5cb6f406b7edee386bfb5a899fa5ae9f24117c4b1fe649286651ece45'
                + 'b3dc2007cb8a163bf0598da48361c55d39a69163fa8fd24cf5f83655d23d'
                + 'ca3ad961c62f356208552bb9ed529077096966d670c354e4abc9804f1746'
                + 'c08ca237327ffffffffffffffff',
            'modp14': 'ffffffffffffffffc90fdaa22168c234c4c6628b80dc1cd129024e08'
                + '8a67cc74020bbea63b139b22514a08798e3404ddef9519b3cd3a431b302b'
                + '0a6df25f14374fe1356d6d51c245e485b576625e7ec6f44c42e9a637ed6b'
                + '0bff5cb6f406b7edee386bfb5a899fa5ae9f24117c4b1fe649286651ece4'
                + '5b3dc2007cb8a163bf0598da48361c55d39a69163fa8fd24cf5f83655d23'
                + 'dca3ad961c62f356208552bb9ed529077096966d670c354e4abc9804f174'
                + '6c08ca18217c32905e462e36ce3be39e772c180e86039b2783a2ec07a28f'
                + 'b5c55df06f4c52c9de2bcbf6955817183995497cea956ae515d2261898fa'
                + '051015728e5a8aacaa68ffffffffffffffff',
            'modp15': 'ffffffffffffffffc90fdaa22168c234c4c6628b80dc1cd129024e08'
                + '8a67cc74020bbea63b139b22514a08798e3404ddef9519b3cd3a431b302b'
                + '0a6df25f14374fe1356d6d51c245e485b576625e7ec6f44c42e9a637ed6b'
                + '0bff5cb6f406b7edee386bfb5a899fa5ae9f24117c4b1fe649286651ece4'
                + '5b3dc2007cb8a163bf0598da48361c55d39a69163fa8fd24cf5f83655d23'
                + 'dca3ad961c62f356208552bb9ed529077096966d670c354e4abc9804f174'
                + '6c08ca18217c32905e462e36ce3be39e772c180e86039b2783a2ec07a28f'
                + 'b5c55df06f4c52c9de2bcbf6955817183995497cea956ae515d2261898fa'
                + '051015728e5a8aaac42dad33170d04507a33a85521abdf1cba64ecfb8504'
                + '58dbef0a8aea71575d060c7db3970f85a6e1e4c7abf5ae8cdb0933d71e8c'
                + '94e04a25619dcee3d2261ad2ee6bf12ffa06d98a0864d87602733ec86a64'
                + '521f2b18177b200cbbe117577a615d6c770988c0bad946e208e24fa074e5'
                + 'ab3143db5bfce0fd108e4b82d120a93ad2caffffffffffffffff',
            'modp16': 'ffffffffffffffffc90fdaa22168c234c4c6628b80dc1cd129024e08'
                + '8a67cc74020bbea63b139b22514a08798e3404ddef9519b3cd3a431b302b'
                + '0a6df25f14374fe1356d6d51c245e485b576625e7ec6f44c42e9a637ed6b'
                + '0bff5cb6f406b7edee386bfb5a899fa5ae9f24117c4b1fe649286651ece4'
                + '5b3dc2007cb8a163bf0598da48361c55d39a69163fa8fd24cf5f83655d23'
                + 'dca3ad961c62f356208552bb9ed529077096966d670c354e4abc9804f174'
                + '6c08ca18217c32905e462e36ce3be39e772c180e86039b2783a2ec07a28f'
                + 'b5c55df06f4c52c9de2bcbf6955817183995497cea956ae515d2261898fa'
                + '051015728e5a8aaac42dad33170d04507a33a85521abdf1cba64ecfb8504'
                + '58dbef0a8aea71575d060c7db3970f85a6e1e4c7abf5ae8cdb0933d71e8c'
                + '94e04a25619dcee3d2261ad2ee6bf12ffa06d98a0864d87602733ec86a64'
                + '521f2b18177b200cbbe117577a615d6c770988c0bad946e208e24fa074e5'
                + 'ab3143db5bfce0fd108e4b82d120a92108011a723c12a787e6d788719a10'
                + 'bdba5b2699c327186af4e23c1a946834b6150bda2583e9ca2ad44ce8dbbb'
                + 'c2db04de8ef92e8efc141fbecaa6287c59474e6bc05d99b2964fa090c3a2'
                + '233ba186515be7ed1f612970cee2d7afb81bdd762170481cd0069127d5b0'
                + '5aa993b4ea988d8fddc186ffb7dc90a6c08f4df435c934063199ffffffff'
                + 'ffffffff',
            'modp17': 'ffffffffffffffffc90fdaa22168c234c4c6628b80dc1cd129024e08'
                + '8a67cc74020bbea63b139b22514a08798e3404ddef9519b3cd3a431b302b'
                + '0a6df25f14374fe1356d6d51c245e485b576625e7ec6f44c42e9a637ed6b'
                + '0bff5cb6f406b7edee386bfb5a899fa5ae9f24117c4b1fe649286651ece4'
                + '5b3dc2007cb8a163bf0598da48361c55d39a69163fa8fd24cf5f83655d23'
                + 'dca3ad961c62f356208552bb9ed529077096966d670c354e4abc9804f174'
                + '6c08ca18217c32905e462e36ce3be39e772c180e86039b2783a2ec07a28f'
                + 'b5c55df06f4c52c9de2bcbf6955817183995497cea956ae515d2261898fa'
                + '051015728e5a8aaac42dad33170d04507a33a85521abdf1cba64ecfb8504'
                + '58dbef0a8aea71575d060c7db3970f85a6e1e4c7abf5ae8cdb0933d71e8c'
                + '94e04a25619dcee3d2261ad2ee6bf12ffa06d98a0864d87602733ec86a64'
                + '521f2b18177b200cbbe117577a615d6c770988c0bad946e208e24fa074e5'
                + 'ab3143db5bfce0fd108e4b82d120a92108011a723c12a787e6d788719a10'
                + 'bdba5b2699c327186af4e23c1a946834b6150bda2583e9ca2ad44ce8dbbb'
                + 'c2db04de8ef92e8efc141fbecaa6287c59474e6bc05d99b2964fa090c3a2'
                + '233ba186515be7ed1f612970cee2d7afb81bdd762170481cd0069127d5b0'
                + '5aa993b4ea988d8fddc186ffb7dc90a6c08f4df435c93402849236c3fab4'
                + 'd27c7026c1d4dcb2602646dec9751e763dba37bdf8ff9406ad9e530ee5db'
                + '382f413001aeb06a53ed9027d831179727b0865a8918da3edbebcf9b14ed'
                + '44ce6cbaced4bb1bdb7f1447e6cc254b332051512bd7af426fb8f401378c'
                + 'd2bf5983ca01c64b92ecf032ea15d1721d03f482d7ce6e74fef6d55e702f'
                + '46980c82b5a84031900b1c9e59e7c97fbec7e8f323a97a7e36cc88be0f1d'
                + '45b7ff585ac54bd407b22b4154aacc8f6d7ebf48e1d814cc5ed20f8037e0'
                + 'a79715eef29be32806a1d58bb7c5da76f550aa3d8a1fbff0eb19ccb1a313'
                + 'd55cda56c9ec2ef29632387fe8d76e3c0468043e8f663f4860ee12bf2d5b'
                + '0b7474d6e694f91e6dcc4024ffffffffffffffff',
            'modp18': 'ffffffffffffffffc90fdaa22168c234c4c6628b80dc1cd129024e08'
                + '8a67cc74020bbea63b139b22514a08798e3404ddef9519b3cd3a431b302b'
                + '0a6df25f14374fe1356d6d51c245e485b576625e7ec6f44c42e9a637ed6b'
                + '0bff5cb6f406b7edee386bfb5a899fa5ae9f24117c4b1fe649286651ece4'
                + '5b3dc2007cb8a163bf0598da48361c55d39a69163fa8fd24cf5f83655d23'
                + 'dca3ad961c62f356208552bb9ed529077096966d670c354e4abc9804f174'
                + '6c08ca18217c32905e462e36ce3be39e772c180e86039b2783a2ec07a28f'
#.........这里部分代码省略.........

def is_location(loc):
	if regex(r"^\d\d$", loc):
		return True
	else:
		return False