def test_get_template(tmpdir, mode):
    """Load template using relative or absolute path."""
    template_path = tmpdir.join('template.tmpl')
    template_path.write('value = {{ value }}')

    if mode == 'absolute':
        template = get_template(template_path.strpath)
    else:
        template = get_template('template.tmpl', template_path=tmpdir.strpath)

    assert template.render({'value': 1}) == 'value = 1'

  def write(self):
    template = get_template(self.repo.padTemplate)
    basename = self.repo.basename
    filename = basename + '.xml'

    pad = template.render({
      'name': self.repo.name,
      'type': self.repo.type,
      'basename': basename,
      'browser_name': self.browser_name,
      'browser_min_version': self.browser_min_version,
      'version': self.version,
      'release_date': self.release_date,
      'release_status': self.release_status,
      'os_support': ','.join(self.os_support),
      'language': ','.join(self.languages),
      'download_size': self.download_size,
      'download_url': self.download_url,
      'pad_url': urljoin(self.repo.padURL, filename),
    }).encode('utf-8')

    path = os.path.join(self.repo.padDirectory, filename)
    validate_pad(pad, path)

    with open(path, 'wb') as file:
      file.write(pad)

def sitekey_frame(environ, start_response):
    template_path, template_file = os.path.split(
        get_config().get('testpages', 'sitekeyFrameTemplate'),
    )
    template = get_template(template_file, template_path=template_path)

    http_path = request_path(environ)
    http_host = environ['HTTP_HOST']
    http_ua = environ['HTTP_USER_AGENT']

    key = M2Crypto.EVP.load_key(get_config().get('testpages', 'sitekeyPath'))
    key.sign_init()
    key.sign_update('\x00'.join([http_path, http_host, http_ua]))
    public_key = base64.b64encode(key.as_der())
    signature = base64.b64encode(key.final())

    start_response('200 OK', [
        ('Content-Type', 'text/html; charset=utf-8'),
        ('X-Adblock-Key', '%s_%s' % (public_key, signature)),
    ])
    return [template.render({
        'public_key': public_key,
        'signature': signature,
        'http_path': http_path,
        'http_host': http_host,
        'http_ua': http_ua,
    }).encode('utf-8')]

    def writeUpdateManifest(self):
        """
          Writes update.rdf file for the current build
        """
        baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
        if self.config.type == 'safari':
            manifestPath = os.path.join(baseDir, 'updates.plist')
            templateName = 'safariUpdateManifest'
        elif self.config.type == 'android':
            manifestPath = os.path.join(baseDir, 'updates.xml')
            templateName = 'androidUpdateManifest'
        else:
            return

        if not os.path.exists(baseDir):
            os.makedirs(baseDir)

        # ABP for Android used to have its own update manifest format. We need to
        # generate both that and the new one in the libadblockplus format as long
        # as a significant amount of users is on an old version.
        if self.config.type == 'android':
            newManifestPath = os.path.join(baseDir, 'update.json')
            writeAndroidUpdateManifest(newManifestPath, [{
                'basename': self.basename,
                'version': self.version,
                'updateURL': self.updateURL
            }])

        template = get_template(get_config().get('extensions', templateName))
        template.stream({'extensions': [self]}).dump(manifestPath)

  def updateIndex(self, versions):
    """
      Updates index page listing all existing versions
    """
    baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
    if not os.path.exists(baseDir):
      os.makedirs(baseDir)
    outputFile = "index.html"
    outputPath = os.path.join(baseDir, outputFile)

    links = []
    for version in versions:
      packageFile = self.basename + '-' + version + self.config.packageSuffix
      changelogFile = self.basename + '-' + version + '.changelog.xhtml'
      if not os.path.exists(os.path.join(baseDir, packageFile)):
        # Oops
        continue

      link = {
        'version': version,
        'download': packageFile,
        'mtime': os.path.getmtime(os.path.join(baseDir, packageFile)),
        'size': os.path.getsize(os.path.join(baseDir, packageFile))
      }
      if os.path.exists(os.path.join(baseDir, changelogFile)):
        link['changelog'] = changelogFile
      links.append(link)
    template = get_template(get_config().get('extensions', 'nightlyIndexPage'))
    template.stream({'config': self.config, 'links': links}).dump(outputPath)

def writeUpdateManifest(links):
  """
  writes an update manifest for all extensions and Android apps
  """

  extensions = {'gecko': [], 'android': [], 'safari': [], 'ie': []}
  for repo in Configuration.getRepositoryConfigurations():
    if repo.type not in extensions or not links.has_section(repo.repositoryName):
      continue
    data = readMetadata(repo, links.get(repo.repositoryName, 'version'))
    data['updateURL'] = links.get(repo.repositoryName, 'downloadURL')
    if data['updateURL'].startswith(repo.downloadsURL):
      data['updateURL'] += "?update"
    extensions[repo.type].append(data)

  if len(extensions['android']) > 1:
    print >>sys.stderr, 'Warning: more than one Android app defined, update manifest only works for one'

  for repoType in extensions.iterkeys():
    manifestPath = get_config().get('extensions', '%sUpdateManifestPath' % repoType)
    if repoType == 'ie':
      writeIEUpdateManifest(manifestPath, extensions[repoType])
    else:
      # ABP for Android used to have its own update manifest format. We need to
      # generate both that and the new one in the libadblockplus format as long
      # as a significant amount of users is on an old version.
      if repoType == 'android':
        newManifestPath = get_config().get("extensions",
                                           "androidNewUpdateManifestPath")
        writeAndroidUpdateManifest(newManifestPath, extensions[repoType])
      template = get_template(get_config().get('extensions', '%sUpdateManifest' % repoType))
      template.stream({'extensions': extensions[repoType]}).dump(manifestPath)

def saveReport(guid, reportData, isNew=False):
    cursor = get_db().cursor()
    screenshot = reportData.get('screenshot', None)
    if screenshot != None:
        reportData['hasscreenshot'] = 2 if reportData.get('screenshotEdited', False) else 1
        try:
            saveScreenshot(guid, screenshot)
        except (TypeError, UnicodeEncodeError):
            reportData['hasscreenshot'] = 0
        del reportData['screenshot']
    knownIssues = len(reportData.get('knownIssues', []))
    contact = getUserId(reportData.get('email', None)) if reportData.get('email', None) else None
    dumpstr = marshal.dumps(reportData)

    if contact != None and isNew:
        executeQuery(cursor, 'INSERT INTO #PFX#users (id, reports) VALUES (%s, 1) ON DUPLICATE KEY UPDATE reports = reports + 1', contact)
    executeQuery(cursor,
                 '''INSERT INTO #PFX#reports (guid, type, ctime, site, comment, status, contact, hasscreenshot, knownissues, dump)
                 VALUES (%(guid)s, %(type)s, FROM_UNIXTIME(%(ctime)s), %(site)s, %(comment)s, %(status)s, %(contact)s,
                 %(hasscreenshot)s, %(knownissues)s, _binary %(dump)s) ON DUPLICATE KEY
                 UPDATE type = %(type)s, site = %(site)s, comment = %(comment)s, status = %(status)s,
                 hasscreenshot = %(hasscreenshot)s, knownissues = %(knownissues)s, dump = _binary %(dump)s''',
                 {'guid': guid, 'type': reportData.get('type', None), 'ctime': reportData['time'], 'site': reportData.get('siteName', None),
                  'comment': reportData.get('comment', None), 'status': reportData.get('status', None), 'contact': contact,
                  'hasscreenshot': reportData.get('hasscreenshot', 0), 'knownissues': knownIssues, 'dump': dumpstr})
    if len(reportData['subscriptions']) > 0:
        for sn in reportData['subscriptions']:
            executeQuery(cursor, 'SELECT id FROM #PFX#subscriptions WHERE url = %s', sn['id'])
            id = cursor.fetchone()
            if id != None:
                def filterMatch(f):
                    return any(u == sn['id'] for u in f.get('subscriptions', []))
                hasMatches = any(filterMatch(f) for f in reportData.get('filters', []))
                executeQuery(cursor, 'INSERT IGNORE INTO #PFX#sublists (report, list, hasmatches) VALUES (%s, %s, %s)', (guid, id[0], hasMatches))

    get_db().commit()

    reportData['guid'] = guid
    if contact:
        # TODO: The mail anonymization should happen in the template, not here
        origEmail = reportData['email']
        email = reportData['email']
        email = re.sub(r' at ', r'@', email)
        email = re.sub(r' dot ', r'.', email)
        reportData['email'] = anonymizeMail(email)
        reportData['uid'] = contact

    file = os.path.join(get_config().get('reports', 'dataPath'), guid[0], guid[1], guid[2], guid[3], guid + '.html')
    dir = os.path.dirname(file)
    if not os.path.exists(dir):
        os.makedirs(dir)
    template = get_template(get_config().get('reports', 'webTemplate'))
    template.stream(reportData).dump(file, encoding='utf-8')

    if contact:
        reportData['email'] = origEmail

def _post_comments(ui, repo, config, refs):
    repo_name = posixpath.split(repo.url())[1]
    template = get_template('hg/template/issue_commit_comment.tmpl',
                            autoescape=False)
    for ref in refs:
        comment_text = template.render({
            'repository_name': repo_name,
            'changes': ref.commits,
            'format_description': _format_description,
        })
        with _trac_proxy(ui, config, 'getting issue {}'.format(ref.id)) as tp:
            attrs = tp.ticket.get(ref.id)[3]
            changes = {'_ts': attrs['_ts'], 'action': 'leave'}
            _update_issue(ui, config, ref.id, changes, comment_text)

def sitekey_frame(environ, start_response):
    template_path, template_file = os.path.split(get_config().get("testpages", "sitekeyFrameTemplate"))
    template = get_template(template_file, template_path=template_path)

    key = M2Crypto.EVP.load_key(get_config().get("testpages", "sitekeyPath"))
    key.sign_init()
    key.sign_update("\x00".join((request_path(environ), environ["HTTP_HOST"], environ["HTTP_USER_AGENT"])))

    public_key = base64.b64encode(key.as_der())
    signature = base64.b64encode(key.final())

    start_response(
        "200 OK", [("Content-Type", "text/html; charset=utf-8"), ("X-Adblock-Key", "%s_%s" % (public_key, signature))]
    )
    return [template.render({"public_key": public_key, "signature": signature}).encode("utf-8")]

def handleRequest(environ, start_response):
    if not environ.get('HTTP_X_ADBLOCK_PLUS'):
        return showError('Please use Adblock Plus to submit reports', start_response)

    if environ['REQUEST_METHOD'].upper() != 'POST' or not environ.get('CONTENT_TYPE', '').startswith('text/xml'):
        return showError('Unsupported request method', start_response)

    params = parse_qs(environ.get('QUERY_STRING', ''))

    requestVersion = params.get('version', ['0'])[0]
    if requestVersion != '1':
        return showError('Unsupported request version', start_response)

    guid = params.get('guid', [''])[0].lower()
    if not re.match(r'^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$', guid):
        return showError('Invalid or missing GUID', start_response)

    path = os.path.join(get_config().get('reports', 'dataPath'), guid + '.xml')
    if os.path.exists(path) or os.path.exists(path + '.tmp'):
        return showError('Duplicate GUID', start_response)

    try:
        request_size = int(environ['CONTENT_LENGTH'])
    except (KeyError, ValueError):
        return showError('Invalid or missing Content-Length header', start_response,
                         '411 Length Required')

    dir = os.path.dirname(path)
    if not os.path.exists(dir):
        os.makedirs(dir)
    try:
        file = open(path + '.tmp', 'wb')
        data = environ['wsgi.input'].read(request_size)
        file.write(data)
        file.close()

        knownIssues = knownIssuesParser.findMatches(data.splitlines(), params.get('lang', ['en-US'])[0])

        os.rename(path + '.tmp', path)
    except Exception as e:
        if os.path.isfile(path + '.tmp'):
            os.remove(path + '.tmp')
        raise e

    template = get_template(get_config().get('reports', 'submitResponseTemplate'))
    start_response('200 OK', [('Content-Type', 'application/xhtml+xml; charset=utf-8')])
    return [template.render({'url': get_config().get('reports', 'urlRoot') + guid, 'knownIssues': knownIssues}).encode('utf-8')]

def handleRequest(environ, start_response):
  setupStderr(environ['wsgi.errors'])

  params = parse_qs(environ.get('QUERY_STRING', ''))

  id = params.get('id', [''])[0].lower()
  if not re.match(r'^[\da-f]{32}$', id):
    return showError('Invalid or missing ID', start_response)

  user = getUser(id)
  if user == None:
    return showError('User not found', start_response)

  user['reportlist'] = getReportsForUser(id)

  template = get_template(get_config().get('reports', 'showUserTemplate'))
  start_response('200 OK', [('Content-Type', 'application/xhtml+xml; charset=utf-8')])
  return [template.render(user).encode('utf-8')]

  def writeChangelog(self, changes):
    """
      write the changelog file into the cloned repository
    """
    baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
    if not os.path.exists(baseDir):
      os.makedirs(baseDir)
    changelogFile = "%s-%s.changelog.xhtml" % (self.basename, self.version)
    changelogPath = os.path.join(baseDir, changelogFile)
    self.changelogURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '/' + changelogFile)

    template = get_template(get_config().get('extensions', 'changelogTemplate'))
    template.stream({'changes': changes}).dump(changelogPath)

    linkPath = os.path.join(baseDir, '00latest.changelog.xhtml')
    if hasattr(os, 'symlink'):
      if os.path.exists(linkPath):
        os.remove(linkPath)
      os.symlink(os.path.basename(changelogPath), linkPath)
    else:
      shutil.copyfile(changelogPath, linkPath)

def writeUpdateManifest(links):
  """
  writes an update manifest for all Gecko extensions and Android apps
  """

  extensions = {'gecko': [], 'android': []}
  for repo in Configuration.getRepositoryConfigurations():
    if repo.type not in extensions or not links.has_section(repo.repositoryName):
      continue
    data = readMetadata(repo, links.get(repo.repositoryName, 'version'))
    data['updateURL'] = links.get(repo.repositoryName, 'downloadURL')
    if data['updateURL'].startswith(repo.downloadsURL):
      data['updateURL'] += "?update"
    extensions[repo.type].append(data)

  if len(extensions['android']) > 1:
    print >>sys.stderr, 'Warning: more than one Android app defined, update manifest only works for one'

  for repoType in extensions.iterkeys():
    manifestPath = get_config().get('extensions', '%sUpdateManifestPath' % repoType)
    template = get_template(get_config().get('extensions', '%sUpdateManifest' % repoType))
    template.stream({'extensions': extensions[repoType]}).dump(manifestPath)

  def writeUpdateManifest(self):
    """
      Writes update.rdf file for the current build
    """
    baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
    if not os.path.exists(baseDir):
      os.makedirs(baseDir)
    if self.config.type == 'chrome' or self.config.type == 'opera':
      manifestPath = os.path.join(baseDir, "updates.xml")
      templateName = 'chromeUpdateManifest'
    elif self.config.type == 'safari':
      manifestPath = os.path.join(baseDir, "updates.plist")
      templateName = 'safariUpdateManifest'
    elif self.config.type == 'android':
      manifestPath = os.path.join(baseDir, "updates.xml")
      templateName = 'androidUpdateManifest'
    else:
      manifestPath = os.path.join(baseDir, "update.rdf")
      templateName = 'geckoUpdateManifest'

    template = get_template(get_config().get('extensions', templateName))
    template.stream({'extensions': [self]}).dump(manifestPath)

def updateDigests(dir):
  global currentTime
  
  subs = subscriptionParser.readSubscriptions()
  defname, defemail = parseaddr(get_config().get('reports', 'defaultSubscriptionRecipient'))

  subscriptions = {}
  emails = {}
  emails[defemail] = []
  for subscription in subs.values():
    for title, url, complete in subscription.variants:
      subscriptions[url] = subscription
    name, email = parseaddr(subscription.email)
    if email != '':
      emails[email] = []
      
  startTime = currentTime - get_config().getint('reports', 'digestDays') * 24*60*60
  for dbreport in getReports(startTime):
    report = {
      'guid': dbreport['guid'],
      'status': dbreport['status'],
      'url': get_config().get('reports', 'urlRoot') + dbreport['guid'] + '#secret=' + calculateReportSecret(dbreport['guid']),
      'site': dbreport['site'],
      'comment': dbreport['comment'],
      'type': dbreport['type'],
      'subscriptions': [],
      'contact': dbreport['contact'],
      'score': getUserUsefulnessScore(dbreport['contact']),
      'hasscreenshot': dbreport['hasscreenshot'],
      'knownIssues': dbreport['knownissues'],
      'time': dbreport['ctime'],
    }

    recipients = set()
    reportSubscriptions = getReportSubscriptions(dbreport['guid'])

    if dbreport['type'] == 'false positive' or dbreport['type'] == 'false negative':
      for subscription in reportSubscriptions:
        subscriptionID = subscription.get('url', 'unknown')
        # Send false negatives to all subscription authors, false positives
        # only to subscriptions with matching filters
        if subscriptionID in subscriptions and (dbreport['type'] == 'false negative' or subscription.get('hasmatches', 0) > 0):
          name, email = parseaddr(subscriptions[subscriptionID].email)
          if email and not email in recipients:
            recipients.add(email)
            emails[email].append(report)
          report['subscriptions'].append(getSubscriptionInfo(subscriptions[subscriptionID]))
    else:
      for subscription in reportSubscriptions:
        subscriptionID = subscription.get('url', 'unknown')
        report['subscriptions'].append(getSubscriptionInfo(subscriptions[subscriptionID]))
      recipients.add(defemail)
      emails[defemail].append(report)
      
  # Generate new digests
  digests = set()
  for email, reports in emails.iteritems():
    if len(reports) == 0:
      continue
    file = getDigestPath(dir, email)
    template = get_template(get_config().get('reports', 'htmlDigestTemplate'))
    template.stream({'email': email, 'reports': reports}).dump(file, encoding='utf-8')
    digests.add(file)
  
  # Remove not updated digests which are more then 2 weeks old
  for filename in os.listdir(dir):
    file = os.path.join(dir, filename)
    if os.path.isfile(file) and file not in digests and re.match(r'^[\da-f]{32}\.html$', filename) and os.stat(file).st_mtime < currentTime - 14*24*60*60:
      os.remove(file)

def writeSubscriptions(templateName, outputFile=None):
    subscriptions = subscriptionParser.readSubscriptions().values()
    template = get_template(get_config().get('subscriptions', templateName + 'Template'))
    if outputFile == None:
        outputFile = get_config().get('subscriptions', templateName + 'File')
    template.stream({'subscriptions': subscriptions}).dump(outputFile, encoding='utf-8')

  if not re.match(r'^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$', guid):
    return showError('Invalid or missing GUID', start_response)

  path = os.path.join(get_config().get('reports', 'dataPath'), guid + '.xml')
  if os.path.exists(path) or os.path.exists(path + '.tmp'):
    return showError('Duplicate GUID', start_response)

  dir = os.path.dirname(path)
  if not os.path.exists(dir):
    os.makedirs(dir)
  try:
    file = open(path + '.tmp', 'wb')
    iter = dataIterator(environ['wsgi.input'], file)
    knownIssues = knownIssuesParser.findMatches(iter, params.get('lang', ['en-US'])[0])
    file.close()

    os.rename(path + '.tmp', path);
  except Exception, e:
    if os.path.isfile(path + '.tmp'):
      os.remove(path + '.tmp')
    raise e

  template = get_template(get_config().get('reports', 'submitResponseTemplate'))
  start_response('200 OK', [('Content-Type', 'application/xhtml+xml; charset=utf-8')])
  return [template.render({'url': get_config().get('reports', 'urlRoot') + guid, 'knownIssues': knownIssues}).encode('utf-8')]

def showError(message, start_response):
  template = get_template(get_config().get('reports', 'errorTemplate'))
  start_response('400 Processing Error', [('Content-Type', 'application/xhtml+xml; charset=utf-8')])
  return [template.render({'message': message}).encode('utf-8')]

def updateDigests(dir):
    global currentTime

    subs = subscriptionParser.readSubscriptions()
    defname, defemail = parseaddr(get_config().get("reports", "defaultSubscriptionRecipient"))

    subscriptions = {}
    emails = {}
    emails[defemail] = []
    for subscription in subs.values():
        for title, url, complete in subscription.variants:
            subscriptions[url] = subscription
        name, email = parseaddr(subscription.email)
        if email != "":
            emails[email] = []

    startTime = currentTime - get_config().getint("reports", "digestDays") * 24 * 60 * 60
    for dbreport in getReports(startTime):
        report = {
            "guid": dbreport["guid"],
            "status": dbreport["status"],
            "url": get_config().get("reports", "urlRoot")
            + dbreport["guid"]
            + "#secret="
            + calculateReportSecret(dbreport["guid"]),
            "site": dbreport["site"],
            "comment": dbreport["comment"],
            "type": dbreport["type"],
            "subscriptions": [],
            "contact": dbreport["contact"],
            "score": getUserUsefulnessScore(dbreport["contact"]),
            "hasscreenshot": dbreport["hasscreenshot"],
            "knownIssues": dbreport["knownissues"],
            "time": dbreport["ctime"],
        }

        recipients = set()
        reportSubscriptions = getReportSubscriptions(dbreport["guid"])

        if dbreport["type"] == "false positive" or dbreport["type"] == "false negative":
            for subscription in reportSubscriptions:
                subscriptionID = subscription.get("url", "unknown")
                # Send false negatives to all subscription authors, false positives
                # only to subscriptions with matching filters
                if subscriptionID in subscriptions and (
                    dbreport["type"] == "false negative" or subscription.get("hasmatches", 0) > 0
                ):
                    name, email = parseaddr(subscriptions[subscriptionID].email)
                    if email and not email in recipients:
                        recipients.add(email)
                        emails[email].append(report)
                    report["subscriptions"].append(getSubscriptionInfo(subscriptions[subscriptionID]))
        else:
            for subscription in reportSubscriptions:
                subscriptionID = subscription.get("url", "unknown")
                report["subscriptions"].append(getSubscriptionInfo(subscriptions[subscriptionID]))
            recipients.add(defemail)
            emails[defemail].append(report)

    # Generate new digests
    digests = set()
    for email, reports in emails.iteritems():
        if len(reports) == 0:
            continue
        file = getDigestPath(dir, email)
        template = get_template(get_config().get("reports", "htmlDigestTemplate"))
        template.stream({"email": email, "reports": reports}).dump(file, encoding="utf-8")
        digests.add(file)

    # Remove not updated digests which are more then 2 weeks old
    for filename in os.listdir(dir):
        file = os.path.join(dir, filename)
        if (
            os.path.isfile(file)
            and file not in digests
            and re.match(r"^[\da-f]{32}\.html$", filename)
            and os.stat(file).st_mtime < currentTime - 14 * 24 * 60 * 60
        ):
            os.remove(file)

def showError(message, start_response):
    template = get_template(get_config().get("reports", "errorTemplate"))
    start_response("400 Processing Error", [("Content-Type", "application/xhtml+xml; charset=utf-8")])
    return [template.render({"message": message}).encode("utf-8")]

    for subscription in subscriptions:
        s = {'name': subscription.name, 'links': []}
        result.append(s)
        for key in ('homepage', 'forum', 'blog', 'faq', 'contact', 'changelog', 'policy'):
            url = getattr(subscription, key)
            if url != None:
                site = urlparse(url).netloc
                s['links'].append({
                    'url': url,
                    'title': key[0].upper() + key[1:],
                    'result': urls[url],
                    'siteResult': site in sites and sites[site],
                })
        for (title, url, complete) in subscription.variants:
            site = urlparse(url).netloc
            s['links'].append({
                'url': url,
                'title': title,
                'result': urls[url],
                'siteResult': site in sites and sites[site],
            })
    return result

if __name__ == '__main__':
    setupStderr()

    subscriptions = checkSubscriptions()
    outputFile = get_config().get('subscriptions', 'statusPage')
    template = get_template(get_config().get('subscriptions', 'statusTemplate'))
    template.stream({'subscriptions': subscriptions}).dump(outputFile, encoding='utf-8')