def test_do_list_usage_exists_with_instance(self):
        fake_request = self.mox.CreateMockAnything()
        fake_request.GET = {"instance": INSTANCE_ID_1}
        results = self.mox.CreateMockAnything()
        models.InstanceExists.objects.filter(instance=INSTANCE_ID_1).AndReturn(results)
        usage = self.mox.CreateMockAnything()
        usage.instance = INSTANCE_ID_1
        usage.launched_at = utils.decimal_utc()
        usage.deleted_at = usage.launched_at + 10
        usage.instance_type_id = 1
        usage.message_id = "someid"
        usage.status = "pending"
        results.__iter__().AndReturn([usage].__iter__())
        self.mox.ReplayAll()

        resp = stacky_server.do_list_usage_exists(fake_request)
        self.assertEqual(resp.status_code, 200)
        resp_json = json.loads(resp.content)
        self.assertEqual(len(resp_json), 2)
        self.assertEqual(
            resp_json[0], ["UUID", "Launched At", "Deleted At", "Instance Type Id", "Message ID", "Status"]
        )
        self.assertEqual(resp_json[1][0], INSTANCE_ID_1)
        launch_time_str = dt.dt_from_decimal(usage.launched_at)
        self.assertEqual(resp_json[1][1], str(launch_time_str))
        delete_time_str = dt.dt_from_decimal(usage.deleted_at)
        self.assertEqual(resp_json[1][2], str(delete_time_str))
        self.mox.VerifyAll()

def _verify_for_delete(exist, delete=None, delete_type="InstanceDeletes"):

    if not delete and exist.delete:
        # We know we have a delete and we have it's id
        delete = exist.delete
    elif not delete:
        if exist.deleted_at:
            # We received this exists before the delete, go find it
            deletes = models.InstanceDeletes.find(exist.instance, dt.dt_from_decimal(exist.launched_at))
            if deletes.count() == 1:
                delete = deletes[0]
            else:
                query = {"instance": exist.instance, "launched_at": exist.launched_at}
                raise NotFound(delete_type, query)
        else:
            # We don't know if this is supposed to have a delete or not.
            # Thus, we need to check if we have a delete for this instance.
            # We need to be careful though, since we could be verifying an
            # exist event that we got before the delete. So, we restrict the
            # search to only deletes before this exist's audit period ended.
            # If we find any, we fail validation
            launched_at = dt.dt_from_decimal(exist.launched_at)
            deleted_at_max = dt.dt_from_decimal(exist.audit_period_ending)
            deletes = models.InstanceDeletes.find(exist.instance, launched_at, deleted_at_max)
            if deletes.count() > 0:
                reason = "Found %s for non-delete exist" % delete_type
                raise VerificationException(reason)

    if delete:
        if not base_verifier._verify_date_field(delete.launched_at, exist.launched_at, same_second=True):
            raise FieldMismatch("launched_at", exist.launched_at, delete.launched_at, exist.instance)

        if not base_verifier._verify_date_field(delete.deleted_at, exist.deleted_at, same_second=True):
            raise FieldMismatch("deleted_at", exist.deleted_at, delete.deleted_at, exist.instance)

def _audit_launches_to_exists(launches, exists, beginning):
    fails = []
    for (instance, launches) in launches.items():
        if instance in exists:
            for expected in launches:
                found = False
                for actual in exists[instance]:
                    if int(expected['launched_at']) == \
                            int(actual['launched_at']):
                    # HACK (apmelton): Truncate the decimal because we may not
                    #    have the milliseconds.
                        found = True

                if not found:
                    rec = False
                    if reconciler:
                        args = (expected['id'], beginning)
                        rec = reconciler.missing_exists_for_instance(*args)
                    launched_at = dt.dt_from_decimal(expected['launched_at'])
                    msg = "Couldn't find exists for launch (%s, %s)"
                    msg = msg % (instance, launched_at)
                    cell, compute = cell_and_compute(instance, launched_at)
                    fails.append(['Launch', expected['id'], msg,
                                  'Y' if rec else 'N', cell, compute])
        else:
            rec = False
            if reconciler:
                args = (launches[0]['id'], beginning)
                rec = reconciler.missing_exists_for_instance(*args)
            msg = "No exists for instance (%s)" % instance
            launched_at = dt.dt_from_decimal(launches[0]['launched_at'])
            cell, compute = cell_and_compute(instance, launched_at)
            fails.append(['-', msg, 'Y' if rec else 'N',
                          cell, compute])
    return fails

    def test_verify_for_launch_late_ambiguous_usage(self):
        exist = self.mox.CreateMockAnything()
        exist.usage = None
        exist.instance = INSTANCE_ID_1
        launched_at = decimal.Decimal('1.1')
        exist.launched_at = launched_at
        exist.instance_type_id = 2
        results = self.mox.CreateMockAnything()
        models.InstanceUsage.objects.filter(
            instance=INSTANCE_ID_1).AndReturn(results)
        results.count().AndReturn(1)
        models.InstanceUsage.find(
            INSTANCE_ID_1, dt.dt_from_decimal(launched_at)).AndReturn(results)
        results.count().AndReturn(2)
        self.mox.ReplayAll()

        with self.assertRaises(AmbiguousResults) as ar:
            nova_verifier._verify_for_launch(exist)
        exception = ar.exception
        self.assertEqual(exception.object_type, 'InstanceUsage')
        search_params = {'instance': INSTANCE_ID_1,
                         'launched_at': decimal.Decimal('1.1')}
        self.assertEqual(exception.search_params, search_params)

        self.mox.VerifyAll()

    def test_do_request(self):
        fake_request = self.mox.CreateMockAnything()
        fake_request.GET = {"request_id": REQUEST_ID_1}
        raw = self._create_raw()
        results = self.mox.CreateMockAnything()
        models.RawData.objects.filter(request_id=REQUEST_ID_1).AndReturn(results)
        results.order_by("when").AndReturn(results)
        results.__iter__().AndReturn([raw].__iter__())
        self.mox.ReplayAll()

        resp = stacky_server.do_request(fake_request)

        self.assertEqual(resp.status_code, 200)
        json_resp = json.loads(resp.content)
        self.assertEqual(len(json_resp), 2)
        self.assertEqual(json_resp[0], ["#", "?", "When", "Deployment", "Event", "Host", "State", "State'", "Task'"])
        self.assertEqual(json_resp[1][0], 1)
        self.assertEqual(json_resp[1][1], u" ")
        self.assertEqual(json_resp[1][2], str(dt.dt_from_decimal(raw.when)))
        self.assertEqual(json_resp[1][3], u"deployment")
        self.assertEqual(json_resp[1][4], u"test.start")
        self.assertEqual(json_resp[1][5], u"example.com")
        self.assertEqual(json_resp[1][6], u"active")
        self.assertEqual(json_resp[1][7], None)
        self.assertEqual(json_resp[1][8], None)
        self.mox.VerifyAll()

def _verify_for_launch(exist):
    if exist.usage:
        launch = exist.usage
    else:
        if models.InstanceUsage.objects\
                 .filter(instance=exist.instance).count() > 0:
            launches = _find_launch(exist.instance,
                                    dt.dt_from_decimal(exist.launched_at))
            count = launches.count()
            query = {
                'instance': exist.instance,
                'launched_at': exist.launched_at
            }
            if count > 1:
                raise AmbiguousResults('InstanceUsage', query)
            elif count == 0:
                raise NotFound('InstanceUsage', query)
            launch = launches[0]
        else:
            raise NotFound('InstanceUsage', {'instance': exist.instance})

    if not _verify_date_field(launch.launched_at, exist.launched_at,
                              same_second=True):
        raise FieldMismatch('launched_at', exist.launched_at,
                            launch.launched_at)

    if launch.instance_type_id != exist.instance_type_id:
        raise FieldMismatch('instance_type_id', exist.instance_type_id,
                            launch.instance_type_id)

def _verify_with_reconciled_data(exist):
    if not exist.launched_at:
        raise VerificationException("Exists without a launched_at")

    query = models.InstanceReconcile.objects.filter(instance=exist.instance)
    if query.count() > 0:
        recs = models.InstanceReconcile.find(exist.instance,
                                             dt.dt_from_decimal((
                                             exist.launched_at)))
        search_query = {'instance': exist.instance,
                        'launched_at': exist.launched_at}
        count = recs.count()
        if count > 1:
            raise AmbiguousResults('InstanceReconcile', search_query)
        elif count == 0:
            raise NotFound('InstanceReconcile', search_query)
        reconcile = recs[0]
    else:
        raise NotFound('InstanceReconcile', {'instance': exist.instance})

    _verify_for_launch(exist, launch=reconcile,
                       launch_type="InstanceReconcile")
    delete = None
    if reconcile.deleted_at is not None:
        delete = reconcile
    _verify_for_delete(exist, delete=delete, delete_type="InstanceReconcile")

def _post_process_raw_data(rows, highlight=None):
    for row in rows:
        if "error" in row.routing_key:
            row.is_error = True
        if highlight and row.id == int(highlight):
            row.highlight = True
        row.fwhen = dt.dt_from_decimal(row.when)

    def test_do_list_usage_launches_with_instance(self):
        fake_request = self.mox.CreateMockAnything()
        fake_request.GET = {'instance': INSTANCE_ID_1}
        results = self.mox.CreateMockAnything()
        models.InstanceUsage.objects.filter(instance=INSTANCE_ID_1)\
                                    .AndReturn(results)
        usage = self.mox.CreateMockAnything()
        usage.instance = INSTANCE_ID_1
        usage.launched_at = utils.decimal_utc()
        usage.instance_type_id = 1
        results.__iter__().AndReturn([usage].__iter__())
        self.mox.ReplayAll()

        resp = stacky_server.do_list_usage_launches(fake_request)
        self.assertEqual(resp.status_code, 200)
        resp_json = json.loads(resp.content)
        self.assertEqual(len(resp_json), 2)
        self.assertEqual(resp_json[0], ["UUID", "Launched At",
                                        "Instance Type Id"])
        self.assertEqual(resp_json[1][0], INSTANCE_ID_1)
        time_str = dt.dt_from_decimal(usage.launched_at)
        self.assertEqual(resp_json[1][1], str(time_str))
        self.assertEqual(resp_json[1][2], 1)

        self.mox.VerifyAll()

 def test_verify_with_reconciled_data_deleted(self):
     exists = self.mox.CreateMockAnything()
     exists.instance = INSTANCE_ID_1
     launched_at = decimal.Decimal('1.1')
     deleted_at = decimal.Decimal('2.1')
     exists.launched_at = launched_at
     exists.deleted_at = deleted_at
     results = self.mox.CreateMockAnything()
     models.InstanceReconcile.objects.filter(instance=INSTANCE_ID_1)\
                                     .AndReturn(results)
     results.count().AndReturn(1)
     launched_at = dt.dt_from_decimal(decimal.Decimal('1.1'))
     recs = self.mox.CreateMockAnything()
     models.InstanceReconcile.find(INSTANCE_ID_1, launched_at).AndReturn(recs)
     recs.count().AndReturn(1)
     reconcile = self.mox.CreateMockAnything()
     reconcile.deleted_at = deleted_at
     recs[0].AndReturn(reconcile)
     self.mox.StubOutWithMock(nova_verifier, '_verify_for_launch')
     nova_verifier._verify_for_launch(exists, launch=reconcile,
                                   launch_type='InstanceReconcile')
     self.mox.StubOutWithMock(nova_verifier, '_verify_for_delete')
     nova_verifier._verify_for_delete(exists, delete=reconcile,
                                   delete_type='InstanceReconcile')
     self.mox.ReplayAll()
     nova_verifier._verify_with_reconciled_data(exists)
     self.mox.VerifyAll()

def _convert_model(model, extra_values_func=None):
    model_dict = model_to_dict(model)
    for key in model_dict:
        if isinstance(model_dict[key], decimal.Decimal):
            model_dict[key] = str(dt.dt_from_decimal(model_dict[key]))
    if extra_values_func:
        model_dict.update(extra_values_func(model))
    return model_dict

    def test_verify_for_delete_non_delete(self):
        launched_at = decimal.Decimal('1.1')
        deleted_at = decimal.Decimal('1.1')
        exist = self.mox.CreateMockAnything()
        exist.delete = None
        exist.instance = INSTANCE_ID_1
        exist.launched_at = launched_at
        exist.deleted_at = None
        exist.audit_period_ending = deleted_at
        results = self.mox.CreateMockAnything()
        models.InstanceDeletes.find(
            INSTANCE_ID_1, dt.dt_from_decimal(launched_at),
            dt.dt_from_decimal(deleted_at)).AndReturn(results)
        results.count().AndReturn(0)

        self.mox.ReplayAll()

        nova_verifier._verify_for_delete(exist)
        self.mox.VerifyAll()

def _verify_for_delete(exist):

    delete = None
    if exist.delete:
        # We know we have a delete and we have it's id
        delete = exist.delete
    else:
        if exist.deleted_at:
            # We received this exists before the delete, go find it
            deletes = _find_delete(exist.instance,
                                   dt.dt_from_decimal(exist.launched_at))
            if deletes.count() == 1:
                delete = deletes[0]
            else:
                query = {
                    'instance': exist.instance,
                    'launched_at': exist.launched_at
                }
                raise NotFound('InstanceDelete', query)
        else:
            # We don't know if this is supposed to have a delete or not.
            # Thus, we need to check if we have a delete for this instance.
            # We need to be careful though, since we could be verifying an
            # exist event that we got before the delete. So, we restrict the
            # search to only deletes before the time this exist was sent.
            # If we find any, we fail validation
            deletes = _find_delete(exist.instance,
                                   dt.dt_from_decimal(exist.launched_at),
                                   dt.dt_from_decimal(exist.raw.when))
            if deletes.count() > 0:
                reason = 'Found InstanceDeletes for non-delete exist'
                raise VerificationException(reason)

    if delete:
        if not _verify_date_field(delete.launched_at, exist.launched_at,
                                  same_second=True):
            raise FieldMismatch('launched_at', exist.launched_at,
                                delete.launched_at)

        if not _verify_date_field(delete.deleted_at, exist.deleted_at,
                                  same_second=True):
            raise FieldMismatch('deleted_at', exist.deleted_at,
                                delete.deleted_at)

def _post_process_raw_data(rows, highlight=None):
    deployments = models.Deployment.objects.all()
    dep_dict = dict((dep.id, dep) for dep in deployments)
    for row in rows:
        if "error" in row.routing_key:
            row.is_error = True
        if highlight and row.id == int(highlight):
            row.highlight = True
        row.fwhen = dt.dt_from_decimal(row.when)
        row.deployment = dep_dict[row.deployment_id]

    def test_do_list_usage_deletes(self):
        fake_request = self.mox.CreateMockAnything()
        fake_request.GET = {}
        results = self.mox.CreateMockAnything()
        models.InstanceDeletes.objects.all().AndReturn(results)
        usage = self.mox.CreateMockAnything()
        usage.instance = INSTANCE_ID_1
        usage.launched_at = utils.decimal_utc()
        usage.deleted_at = usage.launched_at + 10
        results.__iter__().AndReturn([usage].__iter__())
        self.mox.ReplayAll()

        resp = stacky_server.do_list_usage_deletes(fake_request)
        self.assertEqual(resp.status_code, 200)
        resp_json = json.loads(resp.content)
        self.assertEqual(len(resp_json), 2)
        self.assertEqual(resp_json[0], ["UUID", "Launched At", "Deleted At"])
        self.assertEqual(resp_json[1][0], INSTANCE_ID_1)
        launch_time_str = dt.dt_from_decimal(usage.launched_at)
        self.assertEqual(resp_json[1][1], str(launch_time_str))
        delete_time_str = dt.dt_from_decimal(usage.deleted_at)
        self.assertEqual(resp_json[1][2], str(delete_time_str))
        self.mox.VerifyAll()

    def test_verify_for_delete_non_delete_found_deletes(self):
        exist = self.mox.CreateMockAnything()
        exist.delete = None
        exist.instance = INSTANCE_ID_1
        launched_at = decimal.Decimal('1.1')
        deleted_at = decimal.Decimal('1.3')
        exist.launched_at = launched_at
        exist.deleted_at = None
        exist.audit_period_ending = deleted_at
        results = self.mox.CreateMockAnything()
        models.InstanceDeletes.find(
            INSTANCE_ID_1, dt.dt_from_decimal(launched_at),
            dt.dt_from_decimal(deleted_at)).AndReturn(results)
        results.count().AndReturn(1)

        self.mox.ReplayAll()

        try:
            nova_verifier._verify_for_delete(exist)
            self.fail()
        except VerificationException, ve:
            msg = 'Found InstanceDeletes for non-delete exist'
            self.assertEqual(ve.reason, msg)

 def _assert_on_show(self, values, raw):
     self.assertEqual(len(values), 12)
     self.assertEqual(values[0], ["Key", "Value"])
     self.assertEqual(values[1], ["#", raw.id])
     self.assertEqual(values[2], ["When", str(dt.dt_from_decimal(raw.when))])
     self.assertEqual(values[3], ["Deployment", raw.deployment.name])
     self.assertEqual(values[4], ["Category", raw.routing_key])
     self.assertEqual(values[5], ["Publisher", raw.publisher])
     self.assertEqual(values[6], ["State", raw.state])
     self.assertEqual(values[7], ["Event", raw.event])
     self.assertEqual(values[8], ["Service", raw.service])
     self.assertEqual(values[9], ["Host", raw.host])
     self.assertEqual(values[10], ["UUID", raw.instance])
     self.assertEqual(values[11], ["Req ID", raw.request_id])

    def test_verify_delete_when_no_delete(self):
        exist = self.mox.CreateMockAnything()
        exist.delete = None
        exist.uuid = IMAGE_UUID_1
        exist.deleted_at = None
        audit_period_ending = decimal.Decimal("1.2")
        exist.audit_period_ending = audit_period_ending

        results = self.mox.CreateMockAnything()
        models.ImageDeletes.find(IMAGE_UUID_1, dt.dt_from_decimal(audit_period_ending)).AndReturn(results)
        results.count().AndReturn(0)

        self.mox.ReplayAll()

        glance_verifier._verify_for_delete(exist)
        self.mox.VerifyAll()

def _verify_for_launch(exist, launch=None, launch_type="InstanceUsage"):

    if not launch and exist.usage:
        launch = exist.usage
    elif not launch:
        if models.InstanceUsage.objects.filter(instance=exist.instance).count() > 0:
            launches = models.InstanceUsage.find(exist.instance, dt.dt_from_decimal(exist.launched_at))
            count = launches.count()
            query = {"instance": exist.instance, "launched_at": exist.launched_at}
            if count > 1:
                raise AmbiguousResults(launch_type, query)
            elif count == 0:
                raise NotFound(launch_type, query)
            launch = launches[0]
        else:
            raise NotFound(launch_type, {"instance": exist.instance})

    _verify_field_mismatch(exist, launch)

    def test_verify_delete_found_delete_when_exist_deleted_at_is_none(self):
        exist = self.mox.CreateMockAnything()
        exist.delete = None
        exist.uuid = IMAGE_UUID_1
        audit_period_ending = decimal.Decimal("1.3")
        exist.deleted_at = None
        exist.audit_period_ending = audit_period_ending
        results = self.mox.CreateMockAnything()
        models.ImageDeletes.find(IMAGE_UUID_1, dt.dt_from_decimal(audit_period_ending)).AndReturn(results)
        results.count().AndReturn(1)

        self.mox.ReplayAll()

        with self.assertRaises(VerificationException) as ve:
            glance_verifier._verify_for_delete(exist)
        exception = ve.exception
        self.assertEqual(exception.reason, "Found ImageDeletes for non-delete exist")

        self.mox.VerifyAll()