def xpi_build_from_model(rev_pk, mod_codes={}, att_codes={}, hashtag=None, tqueued=None):
    """ Get object and build xpi
    """
    if not hashtag:
        log.critical("No hashtag provided")
        return
    tstart = time.time()
    if tqueued:
        tinqueue = (tstart - tqueued) * 1000
        statsd.timing('xpi.build.queued', tinqueue)
        log.info('[xpi:%s] Addon job picked from queue (%dms)' % (hashtag, tinqueue))
    revision = PackageRevision.objects.get(pk=rev_pk)
    log.debug('[xpi:%s] Building %s' % (hashtag, revision))
    # prepare changed modules and attachments
    modules = []
    attachments = []
    for mod in revision.modules.all():
        if str(mod.pk) in mod_codes:
            mod.code = mod_codes[str(mod.pk)]
            modules.append(mod)
    for att in revision.attachments.all():
        if str(att.pk) in att_codes:
            att.code = att_codes[str(att.pk)]
            attachments.append(att)
    revision.build_xpi(
            modules=modules,
            attachments=attachments,
            hashtag=hashtag,
            tstart=tstart)

def get_download(r, hashtag, filename):
    """
    Download XPI (it has to be ready)
    """
    if not validator.is_valid('alphanum', hashtag):
        log.warning('[security] Wrong hashtag provided')
        return HttpResponseForbidden("{'error': 'Wrong hashtag'}")
    path = os.path.join(settings.XPI_TARGETDIR, '%s.xpi' % hashtag)
    log.info('[xpi:%s] Downloading Addon from %s' % (filename, path))

    tend = time.time()
    tkey = xpi_utils.get_queued_cache_key(hashtag, r)
    tqueued = cache.get(tkey)
    if tqueued:
        ttotal = (tend - tqueued) * 1000
        statsd.timing('xpi.build.total', ttotal)
        total = '%dms' % ttotal
    else:
        total = 'n/a'

    log.info('[xpi:%s] Downloading Add-on (%s)' % (hashtag, total))

    response = serve(r, path, '/', show_indexes=False)
    response['Content-Disposition'] = ('attachment; '
            'filename="%s.xpi"' % filename)
    return response

def get_zip(request, hashtag, filename):
    """
    Download zip (it has to be ready)
    """
    if not validator.is_valid("alphanum", hashtag):
        log.warning("[security] Wrong hashtag provided")
        return HttpResponseForbidden("{'error': 'Wrong hashtag'}")
    path = os.path.join(settings.XPI_TARGETDIR, "%s.zip" % hashtag)
    log.info("[zip:%s] Downloading Addon from %s" % (filename, path))

    tend = time.time()
    tkey = _get_zip_cache_key(request, hashtag)
    tqueued = cache.get(tkey)
    if tqueued:
        ttotal = (tend - tqueued) * 1000
        statsd.timing("zip.total", ttotal)
        total = "%dms" % ttotal
    else:
        total = "n/a"

    log.info("[zip:%s] Downloading Add-on (%s)" % (hashtag, total))

    response = serve(request, path, "/", show_indexes=False)
    response["Content-Disposition"] = "attachment; " 'filename="%s.zip"' % filename
    return response

def _rebuild_kb_chunk(data):
    """Re-render a chunk of documents.

    Note: Don't use host components when making redirects to wiki pages; those
    redirects won't be auto-pruned when they're 404s.

    """
    log.info('Rebuilding %s documents.' % len(data))

    pin_this_thread()  # Stick to master.

    messages = []
    start = time.time()
    for pk in data:
        message = None
        try:
            document = Document.objects.get(pk=pk)

            # If we know a redirect link to be broken (i.e. if it looks like a
            # link to a document but the document isn't there), log an error:
            url = document.redirect_url()
            if (url and points_to_document_view(url) and
                    not document.redirect_document()):
                log.warn('Invalid redirect document: %d' % pk)

            html = document.parse_and_calculate_links()
            if document.html != html:
                # We are calling update here to so we only update the html
                # column instead of all of them. This bypasses post_save
                # signal handlers like the one that triggers reindexing.
                # See bug 797038 and bug 797352.
                Document.objects.filter(pk=pk).update(html=html)
                statsd.incr('wiki.rebuild_chunk.change')
            else:
                statsd.incr('wiki.rebuild_chunk.nochange')
        except Document.DoesNotExist:
            message = 'Missing document: %d' % pk
        except Revision.DoesNotExist:
            message = 'Missing revision for document: %d' % pk
        except ValidationError as e:
            message = 'ValidationError for %d: %s' % (pk, e.messages[0])
        except SlugCollision:
            message = 'SlugCollision: %d' % pk
        except TitleCollision:
            message = 'TitleCollision: %d' % pk

        if message:
            log.debug(message)
            messages.append(message)
    d = time.time() - start
    statsd.timing('wiki.rebuild_chunk', int(round(d * 1000)))

    if messages:
        subject = ('[%s] Exceptions raised in _rebuild_kb_chunk()' %
                   settings.PLATFORM_NAME)
        mail_admins(subject=subject, message='\n'.join(messages))
    if not transaction.get_connection().in_atomic_block:
        transaction.commit()

    unpin_this_thread()  # Not all tasks need to do use the master.

    def handle_submission(self, frame, on_complete):
        """ Handles a submission popped off the dead letter queue.

        Pushes a failure response to XQueue to notify students of the issue.

        """
        submission = frame["submission"]
        submission_id = submission['xqueue_header']['submission_id']
        log.info("Pulled submission #%d off of dead letter queue", submission_id)
        statsd.incr('bux_grader_framework.submissions.dead_lettered')

        # Note time spent in grader
        elapsed_time = int((time.time() - frame["received_time"])*1000.0)
        statsd.timing('bux_grader_framework.total_time_spent', elapsed_time)
        log.info("Submission #%d evaluated in %0.3fms",
                 submission_id, elapsed_time)

        # Check evaluator for extra context to add to fail message.
        hints = ''
        if 'fail_hints' in dir(self.evaluator):
            hints = self.evaluator.fail_hints()

        # Post response to XQueue.
        message = FAIL_RESPONSE.substitute(reason=hints)
        result, success = safe_multi_call(self.xqueue.push_failure,
                                          args=(message, submission),
                                          max_attempts=5,
                                          delay=5)

        # Notifies queue to ack / nack message.
        on_complete(success)

    def submit(self, tags):
        for k in list(self.data.keys()):
            statsd.timing(self.prefix + "." + k, self.data.pop(k),
                          tags=tags, sample_rate=sample_rate)

        if settings.DEBUG:
            assert not self.starts, ('Timer(s) %r were started but never '
                                     'stopped' % self.starts)

 def _do_search(self):
     if not self._results_cache:
         qs = self._build_query()
         es = elasticutils.get_es()
         hits = es.search(qs, settings.ES_INDEX, self.type._meta.app_label)
         self._results_cache = results = SearchResults(self.type, hits)
         statsd.timing('search', results.took)
         log.debug('[%s] %s' % (results.took, qs))
     return self._results_cache

 def raw(self):
     qs = self._build_query()
     es = elasticutils.get_es()
     try:
         hits = es.search(qs, settings.ES_INDEX, self.type._meta.db_table)
     except Exception:
         log.error(qs)
         raise
     statsd.timing('search', hits['took'])
     log.debug('[%s] %s' % (hits['took'], qs))
     return hits

 def raw(self):
     qs = self._build_query()
     es = elasticutils.get_es()
     try:
         with statsd.timer('search.es.timer') as timer:
             hits = es.search(qs, self.index, self.type._meta.db_table)
     except Exception:
         log.error(qs)
         raise
     statsd.timing('search.es.took', hits['took'])
     log.debug('[%s] [%s] %s' % (hits['took'], timer.ms, qs))
     return hits

def zip_source(pk, hashtag, tqueued=None, **kw):
    if not hashtag:
        log.critical("[zip] No hashtag provided")
        return
    tstart = time.time()
    if tqueued:
        tinqueue = (tstart - tqueued) * 1000
        statsd.timing('zip.queued', tinqueue)
        log.info('[zip:%s] Addon job picked from queue (%dms)' % (hashtag, tinqueue))
    log.debug("[zip:%s] Compressing" % pk)
    PackageRevision.objects.get(pk=pk).zip_source(hashtag=hashtag, tstart=tstart)
    log.debug("[zip:%s] Compressed" % pk)

    def on_postrun(self, sender, **kw):
        # sender is the task object. task_id in here.
        pending = self.redis.hincrby(self.pending, sender.name, -1)
        # Clamp pending at 0. Tasks could be coming in before we started
        # tracking.
        if pending < 0:
            self.redis.hset(self.pending, sender.name, 0)
        self.redis.hincrby(self.run, sender.name, 1)

        start = self.redis.hget(self.timer, kw['task_id'])
        if start:
            t = (time.time() - float(start)) * 1000
            statsd.timing('tasks.%s' % sender.name, int(t))

def migrate_helpfulvotes(start_id, end_id):
    """Transfer helpfulvotes from old to new version."""

    if not waffle.switch_is_active('migrate-helpfulvotes'):
        raise  # Celery emails the failed IDs so we know to which to rerun.

    start = time.time()

    pin_this_thread()  # Pin to master

    transaction.enter_transaction_management()
    transaction.managed(True)
    try:
        cursor = connection.cursor()
        cursor.execute("""INSERT INTO `wiki_helpfulvote`
            (revision_id, helpful, created,
            creator_id, anonymous_id, user_agent)
            SELECT COALESCE(
                    (SELECT id FROM `wiki_revision`
                        WHERE `document_id` = wiki_helpfulvoteold.document_id
                            AND `is_approved`=1 AND
                            (`reviewed` <= wiki_helpfulvoteold.created
                                OR `reviewed` IS NULL)
                        ORDER BY CASE WHEN `reviewed`
                            IS NULL THEN 1 ELSE 0 END,
                                  `wiki_revision`.`created` DESC LIMIT 1),
                    (SELECT id FROM `wiki_revision`
                        WHERE `document_id` = wiki_helpfulvoteold.document_id
                            AND (`reviewed` <= wiki_helpfulvoteold.created
                                OR `reviewed` IS NULL)
                        ORDER BY CASE WHEN `reviewed`
                            IS NULL THEN 1 ELSE 0 END,
                                `wiki_revision`.`created`  DESC LIMIT 1),
                    (SELECT id FROM `wiki_revision`
                        WHERE `document_id` = wiki_helpfulvoteold.document_id
                        ORDER BY `created` ASC LIMIT 1)),
                helpful, created, creator_id, anonymous_id, user_agent
            FROM `wiki_helpfulvoteold` WHERE id >= %s AND id < %s""",
            [start_id, end_id])
        transaction.commit()
    except:
        transaction.rollback()
        raise

    transaction.leave_transaction_management()

    unpin_this_thread()

    d = time.time() - start
    statsd.timing('wiki.migrate_helpfulvotes', int(round(d * 1000)))

def _rebuild_kb_chunk(data, **kwargs):
    """Re-render a chunk of documents.

    Note: Don't use host components when making redirects to wiki pages; those
    redirects won't be auto-pruned when they're 404s.

    """
    log.info('Rebuilding %s documents.' % len(data))

    pin_this_thread()  # Stick to master.

    messages = []
    start = time.time()
    for pk in data:
        message = None
        try:
            document = Document.objects.get(pk=pk)

            # If we know a redirect link to be broken (i.e. if it looks like a
            # link to a document but the document isn't there), delete it:
            url = document.redirect_url()
            if (url and points_to_document_view(url) and
                not document.redirect_document()):
                document.delete()
            else:
                document.html = document.current_revision.content_parsed
                document.save()
        except Document.DoesNotExist:
            message = 'Missing document: %d' % pk
        except ValidationError as e:
            message = 'ValidationError for %d: %s' % (pk, e.messages[0])
        except SlugCollision:
            message = 'SlugCollision: %d' % pk
        except TitleCollision:
            message = 'TitleCollision: %d' % pk

        if message:
            log.debug(message)
            messages.append(message)
    d = time.time() - start
    statsd.timing('wiki.rebuild_chunk', int(round(d * 1000)))

    if messages:
        subject = ('[%s] Exceptions raised in _rebuild_kb_chunk()' %
                   settings.PLATFORM_NAME)
        mail_admins(subject=subject, message='\n'.join(messages))
    transaction.commit_unless_managed()

    unpin_this_thread()  # Not all tasks need to do use the master.

 def raw(self):
     """
     Builds query and passes to ElasticSearch, then returns the raw format
     returned.
     """
     qs = self._build_query()
     es = get_es()
     try:
         hits = es.search(qs, settings.ES_INDEX, self.type._meta.db_table)
     except Exception:
         log.error(qs)
         raise
     if statsd:
         statsd.timing("search", hits["took"])
     log.debug("[%s] %s" % (hits["took"], qs))
     return hits

def check_machine(jenkins, client, machine, error_counts):
    try:
        server = client.servers.get(machine.external_id)
    except:
        print "Unable to get server detail, will retry"
        traceback.print_exc()
        return

    if server.status == 'ACTIVE':
        ip = utils.get_public_ip(server)
        if not ip and 'os-floating-ips' in utils.get_extensions(client):
            utils.add_public_ip(server)
            ip = utils.get_public_ip(server)
        if not ip:
            raise Exception("Unable to find public ip of server")

        machine.ip = ip
        print "Machine %s is running, testing ssh" % machine.id
        if utils.ssh_connect(ip, 'jenkins'):
            if statsd:
                dt = int((time.time() - machine.state_time) * 1000)
                key = 'devstack.launch.%s' % machine.base_image.provider.name
                statsd.timing(key, dt)
                statsd.incr(key)
            print "Adding machine %s to Jenkins" % machine.id
            create_jenkins_node(jenkins, machine)
            print "Machine %s is ready" % machine.id
            machine.state = vmdatabase.READY
            return
    elif not server.status.startswith('BUILD'):
        count = error_counts.get(machine.id, 0)
        count += 1
        error_counts[machine.id] = count
        print "Machine %s is in error %s (%s/5)" % (machine.id,
                                                    server.status,
                                                    count)
        if count >= 5:
            if statsd:
                statsd.incr('devstack.error.%s' %
                            machine.base_image.provider.name)
            raise Exception("Too many errors querying machine %s" % machine.id)
    else:
        if time.time() - machine.state_time >= ABANDON_TIMEOUT:
            if statsd:
                statsd.incr('devstack.timeout.%s' %
                            machine.base_image.provider.name)
            raise Exception("Waited too long for machine %s" % machine.id)

 def raw(self):
     """
     Builds query and passes to ElasticSearch, then returns the raw format
     returned.
     """
     qs = self._build_query()
     es = get_es()
     index = (settings.ES_INDEXES.get(self.type)
              or settings.ES_INDEXES['default'])
     try:
         hits = es.search(qs, index, self.type._meta.db_table)
     except Exception:
         log.error(qs)
         raise
     if statsd:
         statsd.timing('search', hits['took'])
     log.debug('[%s] %s' % (hits['took'], qs))
     return hits

def main():
    db = vmdatabase.VMDatabase()

    machine = db.getMachineByJenkinsName(NODE_NAME)
    if machine.state != vmdatabase.HOLD:
        machine.state = vmdatabase.DELETE

    try:
        utils.update_stats(machine.base_image.provider)

        if UPSTREAM_BUILD_URL:
            fd = urllib.urlopen(UPSTREAM_BUILD_URL+'api/json')
            data = json.load(fd)
            result = data['result']
            if statsd and result == 'SUCCESS':
                dt = int(data['duration'])

                key = 'devstack.job.%s' % UPSTREAM_JOB_NAME
                statsd.timing(key+'.runtime', dt)
                statsd.incr(key+'.builds')

                key += '.%s' % UPSTREAM_BRANCH
                statsd.timing(key+'.runtime', dt)
                statsd.incr(key+'.builds')

                key += '.%s' % machine.base_image.provider.name
                statsd.timing(key+'.runtime', dt)
                statsd.incr(key+'.builds')
    except:
        print "Error getting build information"
        traceback.print_exc()

 def _record_time(self, request):
     if hasattr(request, '_start_time'):
         ms = int((time.time() - request._start_time) * 1000)
         data = dict(module=request._view_module, name=request._view_name,
                     method=request.method)
         statsd.timing('view.{module}.{name}.{method}'.format(**data), ms)
         statsd.timing('view.{module}.{method}'.format(**data), ms)
         statsd.timing('view.{method}'.format(**data), ms)

    def raw(self):
        """
        Builds query and passes to ElasticSearch, then returns the raw format
        returned.
        """
        qs = self._build_query()
        es = get_es()
        index = (settings.ES_INDEXES.get(self.type)
                 or settings.ES_INDEXES['default'])

        retries = getattr(settings, 'ES_RETRY', 0)
        retry_wait = getattr(settings, "ES_RETRY_INTERVAL", 0)

        try:
            args = [qs, index, self.type._meta.db_table]
            hits = self.retry_on_timeout(es.search, args, retries, retry_wait)
        except Exception:
            log.error(qs)
            raise
        if statsd:
            statsd.timing('search', hits['took'])
        log.debug('[%s] %s' % (hits['took'], qs))
        return hits

    def deleteNode(self, session, node):
        # Delete a node
        start_time = time.time()
        node.state = nodedb.DELETE
        self.updateStats(session, node.provider_name)
        provider = self.config.providers[node.provider_name]
        target = self.config.targets[node.target_name]
        manager = self.getProviderManager(provider)

        if target.jenkins_url:
            jenkins = self.getJenkinsManager(target)
            jenkins_name = node.nodename
            if jenkins.nodeExists(jenkins_name):
                jenkins.deleteNode(jenkins_name)
            self.log.info("Deleted jenkins node id: %s" % node.id)

        if node.external_id:
            try:
                server = manager.getServer(node.external_id)
                self.log.debug('Deleting server %s for node id: %s' %
                               (node.external_id,
                                node.id))
                manager.cleanupServer(server['id'])
            except provider_manager.NotFound:
                pass

        node.delete()
        self.log.info("Deleted node id: %s" % node.id)

        if statsd:
            dt = int((time.time() - start_time) * 1000)
            key = 'nodepool.delete.%s.%s.%s' % (node.image_name,
                                                node.provider_name,
                                                node.target_name)
            statsd.timing(key, dt)
            statsd.incr(key)
        self.updateStats(session, node.provider_name)