def sitekey_frame(environ, start_response):
    template_path, template_file = os.path.split(
        get_config().get('testpages', 'sitekeyFrameTemplate'),
    )
    template = get_template(template_file, template_path=template_path)

    http_path = request_path(environ)
    http_host = environ['HTTP_HOST']
    http_ua = environ['HTTP_USER_AGENT']

    key = M2Crypto.EVP.load_key(get_config().get('testpages', 'sitekeyPath'))
    key.sign_init()
    key.sign_update('\x00'.join([http_path, http_host, http_ua]))
    public_key = base64.b64encode(key.as_der())
    signature = base64.b64encode(key.final())

    start_response('200 OK', [
        ('Content-Type', 'text/html; charset=utf-8'),
        ('X-Adblock-Key', '%s_%s' % (public_key, signature)),
    ])
    return [template.render({
        'public_key': public_key,
        'signature': signature,
        'http_path': http_path,
        'http_host': http_host,
        'http_ua': http_ua,
    }).encode('utf-8')]

def writeUpdateManifest(links):
  """
  writes an update manifest for all extensions and Android apps
  """

  extensions = {'gecko': [], 'android': [], 'safari': [], 'ie': []}
  for repo in Configuration.getRepositoryConfigurations():
    if repo.type not in extensions or not links.has_section(repo.repositoryName):
      continue
    data = readMetadata(repo, links.get(repo.repositoryName, 'version'))
    data['updateURL'] = links.get(repo.repositoryName, 'downloadURL')
    if data['updateURL'].startswith(repo.downloadsURL):
      data['updateURL'] += "?update"
    extensions[repo.type].append(data)

  if len(extensions['android']) > 1:
    print >>sys.stderr, 'Warning: more than one Android app defined, update manifest only works for one'

  for repoType in extensions.iterkeys():
    manifestPath = get_config().get('extensions', '%sUpdateManifestPath' % repoType)
    if repoType == 'ie':
      writeIEUpdateManifest(manifestPath, extensions[repoType])
    else:
      # ABP for Android used to have its own update manifest format. We need to
      # generate both that and the new one in the libadblockplus format as long
      # as a significant amount of users is on an old version.
      if repoType == 'android':
        newManifestPath = get_config().get("extensions",
                                           "androidNewUpdateManifestPath")
        writeAndroidUpdateManifest(newManifestPath, extensions[repoType])
      template = get_template(get_config().get('extensions', '%sUpdateManifest' % repoType))
      template.stream({'extensions': extensions[repoType]}).dump(manifestPath)

def get_db():
    database = get_config().get('reports', 'database')
    dbuser = get_config().get('reports', 'dbuser')
    dbpasswd = get_config().get('reports', 'dbpassword')
    if os.name == 'nt':
        return MySQLdb.connect(user=dbuser, passwd=dbpasswd, db=database, use_unicode=True, charset='utf8', named_pipe=True)
    else:
        return MySQLdb.connect(user=dbuser, passwd=dbpasswd, db=database, use_unicode=True, charset='utf8')

def hook(ui, repo, node=None, **kwargs):
  ctx = repo[node]
  remote = [get_config().get('irchook', 'remote_command'),
            os.path.basename(repo.root), str(ctx.branch()), str(ctx.user()),
            str(ctx), str(ctx.description())]
  remote = ' '.join(map(lambda s: pipes.quote(s), remote))

  command = ['ssh', get_config().get('irchook', 'remote_host'), remote]
  subprocess.call(command)

def hook(ui=None, repo=None, **kwargs):
  setupStderr()

  root = repo.root if repo != None else get_config().get('hg', 'auth_repository')
  result = generate_data(root)

  with open(get_config().get('hg', 'auth_file'), 'wb') as file:
    for s in result:
      file.write(s)

def handleRequest(environ, start_response):
    setupStderr(environ["wsgi.errors"])

    if environ["REQUEST_METHOD"].upper() != "POST" or not environ.get("CONTENT_TYPE", "").startswith(
        "application/x-www-form-urlencoded"
    ):
        return showError("Unsupported request method", start_response)

    try:
        request_body_length = int(environ["CONTENT_LENGTH"])
    except:
        return showError("Invalid or missing Content-Length header", start_response)

    request_body = environ["wsgi.input"].read(request_body_length)
    params = {}
    for key, value in parse_qsl(request_body):
        params[key] = value.decode("utf-8")

    guid = params.get("guid", "").lower()
    if not re.match(r"^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$", guid):
        return showError("Invalid or missing report GUID", start_response)

    reportData = getReport(guid)

    if reportData == None:
        return showError("Report does not exist", start_response)

    secret = calculateReportSecret(guid)
    if params.get("secret", "") != secret and params.get("secret", "") != calculateReportSecret_compat(guid):
        return showError("Wrong secret value", start_response)

    reportData["status"] = params.get("status", "")
    if len(reportData["status"]) > 1024:
        reportData["status"] = reportData["status"][:1024]

    oldusefulness = reportData.get("usefulness", "0")
    reportData["usefulness"] = params.get("usefulness", "0")
    if "email" in reportData:
        updateUserUsefulness(getUserId(reportData["email"]), reportData["usefulness"], oldusefulness)

    saveReport(guid, reportData)

    if params.get("notify", "") and "email" in reportData:
        email = reportData["email"]
        email = re.sub(r" at ", r"@", email)
        email = re.sub(r" dot ", r".", email)
        if re.match(r"^[\w.%+-][email protected][\w.%+-]+(\.[\w.%+-]+)+", email):
            sendUpdateNotification(
                {"email": email, "url": get_config().get("reports", "urlRoot") + guid, "status": reportData["status"]}
            )

    newURL = get_config().get("reports", "urlRoot") + guid
    newURL += "?updated=" + str(int(random.uniform(0, 10000)))
    newURL += "#secret=" + secret
    start_response("302 Found", [("Location", newURL.encode("utf-8"))])
    return []

def saveReport(guid, reportData, isNew=False):
    cursor = get_db().cursor()
    screenshot = reportData.get('screenshot', None)
    if screenshot != None:
        reportData['hasscreenshot'] = 2 if reportData.get('screenshotEdited', False) else 1
        try:
            saveScreenshot(guid, screenshot)
        except (TypeError, UnicodeEncodeError):
            reportData['hasscreenshot'] = 0
        del reportData['screenshot']
    knownIssues = len(reportData.get('knownIssues', []))
    contact = getUserId(reportData.get('email', None)) if reportData.get('email', None) else None
    dumpstr = marshal.dumps(reportData)

    if contact != None and isNew:
        executeQuery(cursor, 'INSERT INTO #PFX#users (id, reports) VALUES (%s, 1) ON DUPLICATE KEY UPDATE reports = reports + 1', contact)
    executeQuery(cursor,
                 '''INSERT INTO #PFX#reports (guid, type, ctime, site, comment, status, contact, hasscreenshot, knownissues, dump)
                 VALUES (%(guid)s, %(type)s, FROM_UNIXTIME(%(ctime)s), %(site)s, %(comment)s, %(status)s, %(contact)s,
                 %(hasscreenshot)s, %(knownissues)s, _binary %(dump)s) ON DUPLICATE KEY
                 UPDATE type = %(type)s, site = %(site)s, comment = %(comment)s, status = %(status)s,
                 hasscreenshot = %(hasscreenshot)s, knownissues = %(knownissues)s, dump = _binary %(dump)s''',
                 {'guid': guid, 'type': reportData.get('type', None), 'ctime': reportData['time'], 'site': reportData.get('siteName', None),
                  'comment': reportData.get('comment', None), 'status': reportData.get('status', None), 'contact': contact,
                  'hasscreenshot': reportData.get('hasscreenshot', 0), 'knownissues': knownIssues, 'dump': dumpstr})
    if len(reportData['subscriptions']) > 0:
        for sn in reportData['subscriptions']:
            executeQuery(cursor, 'SELECT id FROM #PFX#subscriptions WHERE url = %s', sn['id'])
            id = cursor.fetchone()
            if id != None:
                def filterMatch(f):
                    return any(u == sn['id'] for u in f.get('subscriptions', []))
                hasMatches = any(filterMatch(f) for f in reportData.get('filters', []))
                executeQuery(cursor, 'INSERT IGNORE INTO #PFX#sublists (report, list, hasmatches) VALUES (%s, %s, %s)', (guid, id[0], hasMatches))

    get_db().commit()

    reportData['guid'] = guid
    if contact:
        # TODO: The mail anonymization should happen in the template, not here
        origEmail = reportData['email']
        email = reportData['email']
        email = re.sub(r' at ', r'@', email)
        email = re.sub(r' dot ', r'.', email)
        reportData['email'] = anonymizeMail(email)
        reportData['uid'] = contact

    file = os.path.join(get_config().get('reports', 'dataPath'), guid[0], guid[1], guid[2], guid[3], guid + '.html')
    dir = os.path.dirname(file)
    if not os.path.exists(dir):
        os.makedirs(dir)
    template = get_template(get_config().get('reports', 'webTemplate'))
    template.stream(reportData).dump(file, encoding='utf-8')

    if contact:
        reportData['email'] = origEmail

def handleRequest(environ, start_response):
  setupStderr(environ['wsgi.errors'])

  if environ['REQUEST_METHOD'].upper() != 'POST' or not environ.get('CONTENT_TYPE', '').startswith('application/x-www-form-urlencoded'):
    return showError('Unsupported request method', start_response)

  try:
    request_body_length = int(environ['CONTENT_LENGTH'])
  except:
    return showError('Invalid or missing Content-Length header', start_response)

  request_body = environ['wsgi.input'].read(request_body_length)
  params = {}
  for key, value in parse_qsl(request_body):
    params[key] = value.decode('utf-8')

  guid = params.get('guid', '').lower()
  if not re.match(r'^[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}$', guid):
    return showError('Invalid or missing report GUID', start_response)

  reportData = getReport(guid)    

  if reportData == None:
    return showError('Report does not exist', start_response)

  secret = calculateReportSecret(guid)
  if params.get('secret', '') != secret and params.get('secret', '') != calculateReportSecret_compat(guid):
    return showError('Wrong secret value', start_response)

  reportData['status'] = params.get('status', '')
  if len(reportData['status']) > 1024:
    reportData['status'] = reportData['status'][:1024]

  oldusefulness = reportData.get('usefulness', '0')
  reportData['usefulness'] = params.get('usefulness', '0')
  if ('email' in reportData):
    updateUserUsefulness(getUserId(reportData['email']), reportData['usefulness'], oldusefulness)

  saveReport(guid, reportData)

  if params.get('notify', '') and 'email' in reportData:
    email = reportData['email']
    email = re.sub(r' at ', r'@', email)
    email = re.sub(r' dot ', r'.', email)
    if re.match(r'^[\w.%+-][email protected][\w.%+-]+(\.[\w.%+-]+)+', email):
      sendUpdateNotification({
        'email': email,
        'url': get_config().get('reports', 'urlRoot') + guid,
        'status': reportData['status'],
      })

  newURL = get_config().get('reports', 'urlRoot') + guid
  newURL += '?updated=' + str(int(random.uniform(0, 10000)))
  newURL += '#secret=' + secret
  start_response('302 Found', [('Location', newURL.encode('utf-8'))])
  return []

def removeReport(guid):
    cursor = get_db().cursor()
    executeQuery(cursor, 'DELETE FROM #PFX#reports WHERE guid = %s', guid)
    get_db().commit()
    file = os.path.join(get_config().get('reports', 'dataPath'), guid[0], guid[1], guid[2], guid[3], guid + '.html')
    if os.path.isfile(file):
        os.remove(file)
    file = os.path.join(get_config().get('reports', 'dataPath'), guid[0], guid[1], guid[2], guid[3], guid + '.png')
    if os.path.isfile(file):
        os.remove(file)

def _get_db():
  database = get_config().get("crawler", "database")
  dbuser = get_config().get("crawler", "dbuser")
  dbpasswd = get_config().get("crawler", "dbpassword")
  if os.name == "nt":
    return MySQLdb.connect(user=dbuser, passwd=dbpasswd, db=database,
                           use_unicode=True, charset="utf8", named_pipe=True)
  else:
    return MySQLdb.connect(user=dbuser, passwd=dbpasswd, db=database,
                           use_unicode=True, charset="utf8")

    def build(self):
        """
          run the build command in the tempdir
        """
        baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
        if not os.path.exists(baseDir):
            os.makedirs(baseDir)
        outputFile = '%s-%s%s' % (self.basename, self.version, self.config.packageSuffix)
        self.path = os.path.join(baseDir, outputFile)
        self.updateURL = urlparse.urljoin(self.config.nightliesURL, self.basename + '/' + outputFile + '?update')

        if self.config.type == 'android':
            apkFile = open(self.path, 'wb')

            try:
                try:
                    port = get_config().get('extensions', 'androidBuildPort')
                except ConfigParser.NoOptionError:
                    port = '22'
                command = ['ssh', '-p', port, get_config().get('extensions', 'androidBuildHost')]
                command.extend(map(pipes.quote, [
                    '/home/android/bin/makedebugbuild.py', '--revision',
                    self.buildNum, '--version', self.version, '--stdout'
                ]))
                subprocess.check_call(command, stdout=apkFile, close_fds=True)
            except:
                # clear broken output if any
                if os.path.exists(self.path):
                    os.remove(self.path)
                raise
        else:
            env = os.environ
            spiderMonkeyBinary = self.config.spiderMonkeyBinary
            if spiderMonkeyBinary:
                env = dict(env, SPIDERMONKEY_BINARY=spiderMonkeyBinary)

            command = [os.path.join(self.tempdir, 'build.py'),
                       '-t', self.config.type, 'build', '-b', self.buildNum]
            if self.config.type != 'gecko':
                command.extend(['-k', self.config.keyFile])
            command.append(self.path)
            subprocess.check_call(command, env=env)

        if not os.path.exists(self.path):
            raise Exception("Build failed, output file hasn't been created")

        linkPath = os.path.join(baseDir, '00latest%s' % self.config.packageSuffix)
        if hasattr(os, 'symlink'):
            if os.path.exists(linkPath):
                os.remove(linkPath)
            os.symlink(os.path.basename(self.path), linkPath)
        else:
            shutil.copyfile(self.path, linkPath)

def hook(ui, repo, node=None, **kwargs):
    ctx = repo[node]
    commit_identifiers = ctx.bookmarks()
    if not commit_identifiers or ctx.branch() != 'default':
        commit_identifiers.append(ctx.branch())
    remote = [get_config().get('irchook', 'remote_command'),
              os.path.basename(repo.root), ','.join(commit_identifiers),
              str(ctx.user()), str(ctx), str(ctx.description())]
    remote = ' '.join(map(lambda s: pipes.quote(s), remote))

    command = ['ssh', get_config().get('irchook', 'remote_host'), remote]
    subprocess.call(command)

    def uploadToMozillaAddons(self):
        import urllib3

        header = {
            'alg': 'HS256',     # HMAC-SHA256
            'typ': 'JWT',
        }

        issued = int(time.time())
        payload = {
            'iss': get_config().get('extensions', 'amo_key'),
            'jti': random.random(),
            'iat': issued,
            'exp': issued + 60,
        }

        input = '{}.{}'.format(
            base64.b64encode(json.dumps(header)),
            base64.b64encode(json.dumps(payload))
        )

        signature = hmac.new(get_config().get('extensions', 'amo_secret'),
                             msg=input,
                             digestmod=hashlib.sha256).digest()
        token = '{}.{}'.format(input, base64.b64encode(signature))

        upload_url = ('https://addons.mozilla.org/api/v3/addons/{}/'
                      'versions/{}/').format(self.extensionID, self.version)

        with open(self.path, 'rb') as file:
            data, content_type = urllib3.filepost.encode_multipart_formdata({
                'upload': (
                    os.path.basename(self.path),
                    file.read(),
                    'application/x-xpinstall'
                )
            })

        request = urllib2.Request(upload_url, data=data)
        request.add_header('Content-Type', content_type)
        request.add_header('Authorization', 'JWT ' + token)
        request.get_method = lambda: 'PUT'

        try:
            urllib2.urlopen(request).close()
        except urllib2.HTTPError as e:
            try:
                logging.error(e.read())
            finally:
                e.close()
            raise

def sitekey_frame(environ, start_response):
    template_path, template_file = os.path.split(get_config().get("testpages", "sitekeyFrameTemplate"))
    template = get_template(template_file, template_path=template_path)

    key = M2Crypto.EVP.load_key(get_config().get("testpages", "sitekeyPath"))
    key.sign_init()
    key.sign_update("\x00".join((request_path(environ), environ["HTTP_HOST"], environ["HTTP_USER_AGENT"])))

    public_key = base64.b64encode(key.as_der())
    signature = base64.b64encode(key.final())

    start_response(
        "200 OK", [("Content-Type", "text/html; charset=utf-8"), ("X-Adblock-Key", "%s_%s" % (public_key, signature))]
    )
    return [template.render({"public_key": public_key, "signature": signature}).encode("utf-8")]

def load_notifications():
    repo = get_config().get('notifications', 'repository')
    command = ['hg', '-R', repo, 'archive', '-r', 'default', '-t', 'tar',
               '-p', '.', '-X', os.path.join(repo, '.hg_archival.txt'), '-']
    data = subprocess.check_output(command)

    notifications = []
    with tarfile.open(mode='r:', fileobj=StringIO(data)) as archive:
        for fileinfo in archive:
            name = fileinfo.name
            if name.startswith('./'):
                name = name[2:]

            if fileinfo.type == tarfile.REGTYPE:
                data = codecs.getreader('utf8')(archive.extractfile(fileinfo))
                try:
                    notification = _parse_notification(data, name)
                    if not 'inactive' in notification:
                        current_time = datetime.datetime.now()
                        start = notification.pop('start', current_time)
                        end = notification.pop('end', current_time)
                        if not start <= current_time <= end:
                            notification['inactive'] = True
                    notifications.append(notification)
                except:
                    traceback.print_exc()
    return notifications

def saveScreenshot(guid, screenshot):
    prefix = 'data:image/png;base64,'
    if not screenshot.startswith(prefix):
        raise TypeError('Screenshot is not a PNG image')
    data = base64.b64decode(screenshot[len(prefix):])
    file = os.path.join(get_config().get('reports', 'dataPath'), guid[0], guid[1], guid[2], guid[3], guid + '.png')
    dir = os.path.dirname(file)
    if not os.path.exists(dir):
        os.makedirs(dir)
    f = open(file, 'wb')
    f.write(data)
    f.close()
    if get_config().has_option('reports', 'pngOptimizerPath'):
        cmd = get_config().get('reports', 'pngOptimizerPath').split()
        cmd.append(file)
        subprocess.call(cmd)

  def updateIndex(self, versions):
    """
      Updates index page listing all existing versions
    """
    baseDir = os.path.join(self.config.nightliesDirectory, self.basename)
    if not os.path.exists(baseDir):
      os.makedirs(baseDir)
    outputFile = "index.html"
    outputPath = os.path.join(baseDir, outputFile)

    links = []
    for version in versions:
      packageFile = self.basename + '-' + version + self.config.packageSuffix
      changelogFile = self.basename + '-' + version + '.changelog.xhtml'
      if not os.path.exists(os.path.join(baseDir, packageFile)):
        # Oops
        continue

      link = {
        'version': version,
        'download': packageFile,
        'mtime': os.path.getmtime(os.path.join(baseDir, packageFile)),
        'size': os.path.getsize(os.path.join(baseDir, packageFile))
      }
      if os.path.exists(os.path.join(baseDir, changelogFile)):
        link['changelog'] = changelogFile
      links.append(link)
    template = get_template(get_config().get('extensions', 'nightlyIndexPage'))
    template.stream({'config': self.config, 'links': links}).dump(outputPath)

def verify_email(environ, start_response):
    config = get_config()
    params = dict(parse_qsl(environ.get('QUERY_STRING', '')))

    try:
        filename = config.get('submit_email', params['product'] + '_filename')
    except (KeyError, ConfigParser.NoOptionError):
        return send_simple_response(start_response, 400, 'Unknown product')

    email = params.get('email', '')
    signature = params.get('signature', '')
    if sign(config, email) != signature:
        return send_simple_response(
            start_response, 403,
            'Invalid signature in verification request.',
        )

    with open(filename, 'ab', 0) as file:
        fcntl.lockf(file, fcntl.LOCK_EX)
        try:
            print >>file, email
        finally:
            fcntl.lockf(file, fcntl.LOCK_UN)

    location = config.get('submit_email', 'successful_verification_redirect_location')
    location = location.format(lang=quote(params.get('lang') or 'en', ''))
    start_response('303 See Other', [('Location', location)])
    return []

def adblockbrowser_updates(environ, start_response):
    config = get_config()

    builds_dir = config.get('extensions', 'downloadsDirectory')
    builds_url = config.get('extensions', 'downloadsURL').rstrip('/')

    return _handle_request(environ, start_response, builds_dir, builds_url)

def handleRequest(environ, start_response):
  setupStderr(environ['wsgi.errors'])

  start_response('200 OK', [('Content-Type', 'text/plain; charset=utf-8')])
  if environ['REQUEST_METHOD'].upper() != 'POST' or not environ.get('CONTENT_TYPE', '').startswith('application/x-www-form-urlencoded'):
    return 'Unsupported request method'

  try:
    request_body_length = int(environ['CONTENT_LENGTH'])
  except:
    return 'Invalid or missing Content-Length header'

  request_body = environ['wsgi.input'].read(request_body_length)
  params = {}
  for key, value in parse_qsl(request_body):
    params[key] = value.decode('utf-8').strip()

  if not 'name' in params or params['name'] == '':
    return 'No name entered'
  if not 'email' in params or params['email'] == '':
    return 'No email address entered'
  if not 'subject' in params or params['subject'] == '':
    return 'No subject entered'
  if not 'message' in params or params['message'] == '':
    return 'No message entered'

  if not re.match(r'^\w[\w.+!-][email protected]\w[\w.-]+\.[a-zA-Z]{2,6}$', params['email']):
    return 'Invalid email address'

  sendMail(get_config().get('formmail', 'template'), params)
  return 'Message sent'