def create_test_from_file(fl, name, group, policy):
  txt = fl.read()
  fl.close()

  appdir = os.path.join(TESTS_DIR, group, name)
  if os.path.exists(appdir):
    if OVERWRITE:
      if not os.path.isdir(appdir):
        fatal("Unable to overwrite file: %s" % appdir)
      warn("Creating in existing directory: %s" % appdir)
    else:
      fatal("Not overwriting existing directory: %s" % appdir)
  prepare_dir(appdir)

  inputdir = os.path.join(appdir, 'source-input')
  if os.path.exists(inputdir):
    assert OVERWRITE
    if not os.path.isdir(inputdir):
      fatal("Unable to overwrite non-directory: %s" % inputdir)
  else:
    os.makedirs(inputdir)

  tgtfile = "%s.js" % name
  tgtpath = os.path.join(inputdir, tgtfile)
  tgtfl = open(tgtpath, 'w')

  tgtfl.write(txt)
  tgtfl.close()

def update_bills_2(congress, bill_type, bill_number, recordtext, changehash, newchangehash, force_update):
	"""Compares a THOMAS search result record to the hash file to see if anything
	changed, and if so, or if force_update == True, re-parses the bill or amendment."""
	
	key = bill_type + str(bill_number)
	rec = md5_base64(recordtext)

	if not force_update and key in changehash and changehash[key] == rec:
		newchangehash[key] = changehash[key]
		return
	
	if not force_update:
		warn("Detected Update to %d %s %d." % (congress, bill_type, bill_number))
	
	try:
		if bill_type == 'hz':
			#if (!ParseAmendment($bs, 'h', 'Z', $bn)) { return; }
			pass
		elif bill_type == 'sp':
			#if (!ParseAmendment($bs, 's', 'P', $bn)) { return; }
			pass
		else:
			parse_bill(congress, bill_type, bill_number)
	
		newchangehash[key] = rec
	except Exception as e:
		import traceback
		warn("Parsing bill %d %s %d: " % (congress, bill_type, bill_number) + unicode(e) + "\n" + traceback.format_exc())

def _runHelperWait(host):
    output = []
    while True:

        c = _getConnection(host)
        if not c:
            return None

        (stdin, stdout) = c

        line = stdout.readline().strip()
        if line == "~~~":
            break
        output += [line]

    try:
        rc = int(output[-1])
    except ValueError:
        util.warn("cannot parse exit code from helper on %s: %s" % (host.host, output[-1]))
        rc = 1

    util.debug(1, "exit code %d" % rc, prefix=host.host)

    for line in output:
        util.debug(2, "           > %s" % line, prefix=host.host)

    return (rc == 0, output[:-1])

 def _autobuild(self, data, expand_limit):
     children = []
     if os.path.islink(data):
         node_type = 'symlink'
     elif os.path.isdir(data):
         node_type = 'directory'
         try:
             children = os.listdir(data)
             children = map(lambda f,d=data: os.path.join(d,f),
                            children)
         except OSError:
             warn("Unable to list directory '%s'" % data)
             children = []
     else:
         node_type = 'file'
     self.configure(1,node_type,name = os.path.basename(data))
     if (self.properties ^ NP_ABSTRACT) & NP_ABSTRACT:
         self.treewidget.addNode(self)
     if self.properties & NP_ABSTRACT:
         childprops = (self.properties ^ NP_ABSTRACT) | NP_ROOT
     else:
         childprops = (self.properties|NP_ROOT) ^ NP_ROOT
     if self.state & NS_EXPANDED:
         childstate = self.state
     else:
         childstate = ((self.state | NS_PENDING_SHOW | NS_VISIBLE)
                       ^ (NS_PENDING_SHOW | NS_VISIBLE))
     for ch in children:
         treenode = FSTreeNode(self.treewidget,self,ch,
                               expand_limit-1,
                               props=childprops,
                               state=childstate)
         self.children.append(treenode)
     if children:
         self.state = self.state | NS_HAS_CHILDREN

    def translate(self):
        """Translate a section into something suitable for apache::vhost

        Only Directory, Files, Location, DirectoryMatch, FilesMatch,
        and LocationMatch are supported by apache::vhost, all mapping
        to the directories parameter.

        Return:
            A dictionary that can be added to the array for the
            directories parameter of apache::vhost

        """
        provider = self.name.lower()
        if provider not in ['directory', 'files', 'location',
                            'directorymatch', 'filesmatch', 'locationmatch']:
            util.warn("warning: could not translate section {}".format(self.name))
            return None
        result = {
            'provider' : provider,
            'path'     : self.path,
        }
        for child in self.directives:
            # Not supporting sections within sections
            if isinstance(child, DirectiveAST):
                result.update(child.translate())
        return result

def post_reply(reply,post):
  global badsubs
  global submissioncount
  global totalposted
  try:
    #TODO change name
    #possibly remove? not gonna be nsfw
    reply = "#####	\n\n######	\n\n####	\n"+reply+"^Parent ^commenter ^can [^toggle ^NSFW](/message/compose?to=autowikibot&subject=AutoWikibot NSFW toggle&message=%2Btoggle-nsfw+____id____) ^or[](#or) [^delete](/message/compose?to=autowikibot&subject=AutoWikibot Deletion&message=%2Bdelete+____id____)^. ^Will ^also ^delete ^on ^comment ^score ^of ^-1 ^or ^less. ^| [^(FAQs)](http://www.np.reddit.com/r/autowikibot/wiki/index) ^| [^Mods](http://www.np.reddit.com/r/autowikibot/comments/1x013o/for_moderators_switches_commands_and_css/) ^| [^Magic ^Words](http://www.np.reddit.com/r/autowikibot/comments/1ux484/ask_wikibot/)"
    a = post.reply('[#placeholder-awb]Comment is being processed... It will be automatically replaced by new text within a minute or will be deleted if that fails.')
    postsuccess = r.get_info(thing_id='t1_'+str(a.id)).edit(reply.replace('____id____',str(a.id)))
    if not postsuccess:
      raise Exception ('reply unsuccessful')
    totalposted = totalposted + 1
    submissioncount[str(post.submission.id)]+=1
    success("[OK] #%s "%totalposted)
    return True
  except Exception as e:
    warn("REPLY FAILED: %s @ %s"%(e,post.subreddit))
    if str(e).find('TOO_LONG') > -1:
      a.delete()
    elif str(e) == '403 Client Error: Forbidden' and str(post.subreddit) not in badsubs:
      badsubs = badsubs_page.content_md.strip().split()
      badsubs.append(str(post.subreddit))
      editsummary = 'added '+str(post.subreddit)
      save_changing_variables(editsummary)
    else:
      fail(e)
      a.delete()
    return False

    def __init__(self, cfgfile):

        self.types = {}
        cnt = 0

        if not os.path.exists(cfgfile):
            if Installing:
                return

            util.error("analysis configuration %s does not exist" % cfgfile)

        for line in open(cfgfile):
            cnt += 1
            line = line.strip()
            if not line or line.startswith("#"): 
                continue

            f = line.split()
            if len(f) < 2:
                util.warn("cannot parse line %d in %s" % (cnt, cfgfile))
                continue

            type = f[0]
            mechanism = f[1]
            descr = ""
            if len(f) > 2:
                descr = " ".join(f[2:])

            self.types[type] = (mechanism, descr)

def run_targetpages(debug=False, overwrite=False, refine=None, synonly=False, service=False, apps=None):
  results = RunResults('targetpages', overwrite)
  
  sites = get_lines(TARGETPAGE_FILE, comment='#')
  polnet = os.path.join(POLICY_DIR, 'network-isolation.policy')
  policies = {'network-isolation': [polnet]}
  for site in sites:
    # Limit to the given sites names, if provided.
    if apps is not None and site not in apps: continue

    # Extract the application name from the URL.
    app = None
    paramidx = site.find("?payload=")
    if paramidx > -1:
      plidx = paramidx + 9
      endidx = site.find("&", plidx)
      if endidx == -1: endidx = len(site)
      app = site[plidx:endidx]
      warn("Appname: %s" % app)
    
    url = 'http://' + site
    res = run_website(url, policies, debug=debug, overwrite=overwrite, refine=refine, synonly=synonly, service=service, appname=app)

    # Track successful results
    results.add(res)

    # Space the output.
    sys.stderr.write('\n')

  results.printSummary()

def makeLocalNetworks(path, silent=False):

    netcfg = config.Config.localnetscfg

    if not os.path.exists(netcfg):
        util.warn("list of local networks does not exist in %s" % netcfg)
        return

    if ( not silent ):
        util.output("generating local-networks.bro ...", False)

    out = open(os.path.join(path, "local-networks.bro"), "w")
    print >>out, "# Automatically generated. Do not edit.\n"

    netcfg = config.Config.localnetscfg

    if os.path.exists(netcfg):
        nets = readNetworks(netcfg)

        print >>out, "redef Site::local_nets = {"
        for (cidr, tag) in nets:
            print >>out, "\t%s," % cidr,
            if tag != "":
                print >>out, "\t# %s" % tag,
            print >>out
        print >>out, "};\n"

    if ( not silent ):
        util.output(" done.")

def isRunning(nodes, setcrashed=True):

    results = []
    cmds = []

    for node in nodes:
        pid = node.getPID()
        if not pid:
            results += [(node, False)]
            continue

        cmds += [(node, "check-pid", [str(pid)])]

    for (node, success, output) in execute.runHelperParallel(cmds):

        # If we cannot connect to the host at all, we filter it out because
        # the process might actually still be running but we can't tell.
        if output == None:
            if config.Config.cron == "0":
                util.warn("cannot connect to %s" % node.name)
            continue

        results += [(node, success)]

        if not success:
            if setcrashed:
                # Grmpf. It crashed.
                node.clearPID();
                node.setCrashed()

    return results

  def save_report(self):
    try:
      ts = time.time() 

      # Comment scores
      self.c.execute('''CREATE TABLE IF NOT EXISTS comment_scores
          (cid TEXT, subreddit TEXT, score INTEGER, ts INTEGER)''')
      self.c.execute('''CREATE INDEX IF NOT EXISTS cscores_time ON comment_scores(ts)''')

      for cid, score in self.score_map.iteritems():
        self.c.execute('''INSERT INTO comment_scores(cid, subreddit, score, ts) VALUES(?,?,?,?)''',
            (cid, self.subreddit_map[cid], score, ts))
      self.conn.commit()

      # Deleted comments
      self.c.execute('''CREATE TABLE IF NOT EXISTS deleted_comments
          (cid TEXT, subreddit TEXT, score INTEGER, ts INTEGER)''')
      self.c.execute('''CREATE INDEX IF NOT EXISTS cscores_time ON comment_scores(ts)''')

      for cols in self.del_list:
        # I'm sure this could be done better
        curcols = list(cols)
        curcols.append(ts)
        self.c.execute('''INSERT INTO deleted_comments(cid, subreddit, score, ts) VALUES(?,?,?,?)''', curcols)
      self.conn.commit()

    except Exception, e:
      warn(e)
      warn("Failed to write subreddit scores")

def makeLocalNetworks():

    netcfg = config.Config.localnetscfg

    if not os.path.exists(netcfg):
        if not config.Installing:
            util.warn("list of local networks does not exist in %s" % netcfg)
        return

    util.output("generating local-networks.bro ...", False)

    out = open(os.path.join(config.Config.policydirsiteinstallauto, "local-networks.bro"), "w")
    print >>out, "# Automatically generated. Do not edit.\n"

    netcfg = config.Config.localnetscfg

    if os.path.exists(netcfg):
        nets = readNetworks(netcfg)

        print >>out, "redef local_nets = {"
        for (cidr, tag) in nets:
            print >>out, "\t%s," % cidr,
            if tag != "":
                print >>out, "\t# %s" % tag,
            print >>out
        print >>out, "};\n"

    util.output(" done.")

    def _readConfig(self, file, allowstate = False):
        config = {}
        try:
            for line in open(file):

                line = line.strip()
                if not line or line.startswith("#"):
                    continue

                args = line.split("=", 1)
                if len(args) != 2:
                    util.error("%s: syntax error '%s'" % (file, line))

                (key, val) = args
                key = key.strip().lower()
                val = val.strip()

                if not allowstate and ".state." in key:
                    util.error("state variable '%s' not allowed in file: %s" % (key, file))

                # if the key already exists, just overwrite with new value
                config[key] = val

        except IOError, e:
            util.warn("cannot read '%s' (this is ok on first run)" % file)

def post_reply(reply,post):
  global badsubs
  global submissioncount
  global totalposted
  # This is a quick hack to fix the double list issue (git issue #12)
  # Please find the actual source of this bug, and delete this hack
  # It removes any sentences that are posted more than once
  lines = []
  for line in reply.split("\n"):
    if line not in lines:
      lines.append(line)
  reply = '\n'.join(lines)
  try:
    reply = "#####	\n\n######	\n\n####	\n"+reply+"\n^Parent ^commenter ^can [^toggle ^NSFW](http://www.np.reddit.com/message/compose?to=autowikiabot&subject=AutoWikibot NSFW toggle&message=%2Btoggle-nsfw+____id____) ^or[](#or) [^delete](http://www.np.reddit.com/message/compose?to=autowikiabot&subject=AutoWikibot Deletion&message=%2Bdelete+____id____)^. ^Will ^also ^delete ^on ^comment ^score ^of ^-1 ^or ^less. ^| [^(FAQs)](http://www.np.reddit.com/r/autowikiabot/wiki/index) ^|  [^Source](https://github.com/Timidger/autowikiabot-py)\n ^(Please note this bot is in testing. Any help would be greatly appreciated, even if it is just a bug report! Please checkout the) [^source ^code](https://github.com/Timidger/autowikiabot-py) ^(to submit bugs)"
    a = post.reply('[#placeholder-awb]Comment is being processed... It will be automatically replaced by new text within a minute or will be deleted if that fails.')
    postsuccess = r.get_info(thing_id='t1_'+str(a.id)).edit(reply.replace('____id____',str(a.id)))
    if not postsuccess:
      raise Exception ('reply unsuccessful')
    totalposted = totalposted + 1
    submissioncount[str(post.submission.id)]+=1
    success("[OK] #%s "%totalposted)
    return True
  except Exception as e:
    warn("REPLY FAILED: %s @ %s"%(e,post.subreddit))
    if str(e) == '(TOO_LONG) `this is too long (max: 15000.0)` on field `text`':
      a.delete()
    elif str(e) == '403 Client Error: Forbidden' and str(post.subreddit) not in badsubs:
      badsubs = badsubs_page.content_md.strip().split()
      badsubs.append(str(post.subreddit))
      editsummary = 'added '+str(post.subreddit)
      save_changing_variables(editsummary)
    else:
      fail(e)
      a.delete()
    return False

def btlinesearch(f, x0, fx0, g, dx, accept_ratio, shrink_factor, max_steps, verbose=False):
    '''
    Find a step size t such that f(x0 + t*dx) is within a factor
    accept_ratio of the linearized function value improvement.

    Args:
        f: the function
        x0: starting point for search
        fx0: the value f(x0). Will be computed if set to None.
        g: search direction, typically the gradient of f at x0
        dx: the largest possible step to take
        accept_ratio: termination criterion
        shrink_factor: how much to decrease the step every iteration
    '''
    if fx0 is None: fx0 = f(x0)
    t = 1.
    m = g.dot(dx)
    if accept_ratio != 0 and m > 0: util.warn('WARNING: %.10f not <= 0' % m)
    num_steps = 0
    while num_steps < max_steps:
        true_imp = f(x0 + t*dx) - fx0
        lin_imp = t*m
        if verbose: true_imp, lin_imp, accept_ratio
        if true_imp <= accept_ratio * lin_imp:
            break
        t *= shrink_factor
        num_steps += 1
    return x0 + t*dx, num_steps

    def _readNodes(self):
        self.nodelist = {}
        config = ConfigParser.SafeConfigParser()
        if not config.read(self.nodecfg) and not Installing:
            util.error("cannot read '%s'" % self.nodecfg)

        manager = False
        proxy = False
        standalone = False

        file = self.nodecfg

        counts = {}
        for sec in config.sections():

            node = Node(sec)
            self.nodelist[sec] = node

            for (key, val) in config.items(sec):
                if not key in Node._tags:
                    util.warn("%s: unknown key '%s' in section '%s'" % (file, key, sec))
                    continue

                if key == "type":
                    # We determine which types are valid by checking for having an
                    # option specifying which scripts to use for it.
                    cfg = "scripts-%s" % val 
                    if not cfg  in self.config:
                        util.error("%s: unknown type '%s' in section '%s'" % (file, val, sec))

                    self.nodelist[sec].scripts = self.config[cfg].split()

                    if val == "manager":
                        if manager:
                            util.error("only one manager can be defined")
                        manager = True

                    if val == "proxy":
                        proxy = True

                    if val == "standalone":
                        standalone = True

                node.__dict__[key] = val

            try:
                node.addr = socket.gethostbyname(node.host)
            except AttributeError:
                util.error("%s: no host given in section '%s'" % (file, sec))
            except socket.gaierror, e:
                util.error("%s: unknown host '%s' in section '%s' [%s]" % (file, node.host, sec, e.args[1]))

            # Each node gets a number unique across its type.
            type = self.nodelist[sec].type
            try: 
                counts[type] += 1
            except KeyError:
                counts[type] = 1

            node.count = counts[type]

 def invert(self, rtol, min_count_t):
     max_divs = 15
     
     min_count_t = max(min_count_t, 2)
     divs = discrete.divs(min_count_t)
     if divs > max_divs:
         raise exc.NumericalError("min. inversion time step count (%s) and corresponding min. divs (%s) too large; max. divs: %s" % (min_count_t, divs, max_divs))
     
     divs = max(divs-1, 0)
     last_res = None
     while True:
         count_t = discrete.steps(divs)
         res = self._integrate(rtol, count_t)
         
         diff = None
         if None not in (last_res, res):
             diff = abs(res - last_res)
             if diff <= rtol * abs(res):
                 break
         
         divs += 1
         if divs > max_divs:
             if diff is not None:
                 util.warn("max. inversion time divs (%s) exceeded; rtol: %s; latest step count: %s; latest difference: %s (current: %s; last: %s)" % (max_divs, rtol, count_t, diff, res, last_res), stacklevel=2)
                 break
             else:
                 msg = "max. inversion time divs (%s) exceeded; rtol: %s; latest step count: %s" % (max_divs, rtol, count_t)
                 if res is not None:
                     util.warn(msg, stacklevel=2)
                     break
                 else:
                     raise exc.NumericalError(msg)
         last_res = res
     
     return res

def cancel_request(conn):
    warn('canceling spot instance requests and terminating instances...')
    requests = conn.get_all_spot_instance_requests(load_request_ids())
    for r in requests:
        r.cancel()
    instance_ids = [r.instance_id for r in requests if r.instance_id is not None]
    if len(instance_ids) > 0:
        conn.terminate_instances(instance_ids)

    def checkBroVersion(self):
        if "broversion" not in self.state:
            return

        oldversion = self.state["broversion"]

        version = self._getBroVersion()
        if version != oldversion:
            util.warn("new bro version detected (run 'broctl install')")

def _updateHTTPStats():
    # Create meta file.
    if not os.path.exists(config.Config.statsdir):
        try:
            os.makedirs(config.Config.statsdir)
        except OSError, err:
            util.output("error creating directory: %s" % err)
            return

        util.warn("creating directory for stats file: %s" % config.Config.statsdir)