def _inject_instances(trigger, rate_per_trigger, duration, payload=None, max_throughput=False):
    payload = payload or {}

    start = date_utils.get_datetime_utc_now()
    elapsed = 0.0
    count = 0

    dispatcher = TriggerDispatcher()
    while elapsed < duration:
        # print('Dispatching trigger %s at time %s', trigger, date_utils.get_datetime_utc_now())
        dispatcher.dispatch(trigger, payload)

        if rate_per_trigger:
            # NOTE: We decrease sleep delay for 56% to take into account overhead / delay because
            # of the call to dispatchet.dispatch method.
            delta = random.expovariate(rate_per_trigger)
            eventlet.sleep(delta * 0.56)

        elapsed = (date_utils.get_datetime_utc_now() - start).seconds
        count += 1

    actual_rate = int(count / elapsed)

    print('%s: Emitted %d triggers in %d seconds (actual rate=%s triggers / second)' %
          (trigger, count, elapsed, actual_rate))

    # NOTE: Due to the overhead of dispatcher.dispatch call, we allow for 10% of deviation from
    # requested rate before warning
    if rate_per_trigger and (actual_rate < (rate_per_trigger * 0.9)):
        print('')
        print('Warning, requested rate was %s triggers / second, but only achieved %s '
              'triggers / second' % (rate_per_trigger, actual_rate))
        print('Too increase the throuput you will likely need to run multiple instances of '
              'this script in parallel.')

 def test_get_marker_from_db(self):
     marker_dt = date_utils.get_datetime_utc_now() - datetime.timedelta(minutes=5)
     marker_db = DumperMarkerDB(marker=isotime.format(marker_dt, offset=False),
                                updated_at=date_utils.get_datetime_utc_now())
     DumperMarker.add_or_update(marker_db)
     exec_exporter = ExecutionsExporter(None, None)
     export_marker = exec_exporter._get_export_marker_from_db()
     self.assertEqual(export_marker, date_utils.add_utc_tz(marker_dt))

 def test_token_post_set_ttl(self):
     timestamp = date_utils.add_utc_tz(date_utils.get_datetime_utc_now())
     response = self.app.post_json(TOKEN_V1_PATH, {'ttl': 60}, expect_errors=False)
     expected_expiry = date_utils.get_datetime_utc_now() + datetime.timedelta(seconds=60)
     self.assertEqual(response.status_int, 201)
     actual_expiry = isotime.parse(response.json['expiry'])
     self.assertLess(timestamp, actual_expiry)
     self.assertLess(actual_expiry, expected_expiry)

 def _test_token_post(self, path=TOKEN_V1_PATH):
     ttl = cfg.CONF.auth.token_ttl
     timestamp = date_utils.get_datetime_utc_now()
     response = self.app.post_json(path, {}, expect_errors=False)
     expected_expiry = date_utils.get_datetime_utc_now() + datetime.timedelta(seconds=ttl)
     expected_expiry = date_utils.add_utc_tz(expected_expiry)
     self.assertEqual(response.status_int, 201)
     self.assertIsNotNone(response.json['token'])
     self.assertEqual(response.json['user'], USERNAME)
     actual_expiry = isotime.parse(response.json['expiry'])
     self.assertLess(timestamp, actual_expiry)
     self.assertLess(actual_expiry, expected_expiry)

    def test_update_canceled_liveaction(self):
        liveaction_db = LiveActionDB()
        liveaction_db.status = 'initializing'
        liveaction_db.start_timestamp = get_datetime_utc_now()
        liveaction_db.action = ResourceReference(
            name=ActionDBUtilsTestCase.action_db.name,
            pack=ActionDBUtilsTestCase.action_db.pack).ref
        params = {
            'actionstr': 'foo',
            'some_key_that_aint_exist_in_action_or_runner': 'bar',
            'runnerint': 555
        }
        liveaction_db.parameters = params
        liveaction_db = LiveAction.add_or_update(liveaction_db)
        origliveaction_db = copy.copy(liveaction_db)

        # Update by id.
        newliveaction_db = action_db_utils.update_liveaction_status(
            status='running', liveaction_id=liveaction_db.id)

        # Verify id didn't change.
        self.assertEqual(origliveaction_db.id, newliveaction_db.id)
        self.assertEqual(newliveaction_db.status, 'running')

        # Verify that state is published.
        self.assertTrue(LiveActionPublisher.publish_state.called)
        LiveActionPublisher.publish_state.assert_called_once_with(newliveaction_db, 'running')

        # Cancel liveaction.
        now = get_datetime_utc_now()
        status = 'canceled'
        newliveaction_db = action_db_utils.update_liveaction_status(
            status=status, end_timestamp=now, liveaction_id=liveaction_db.id)
        self.assertEqual(origliveaction_db.id, newliveaction_db.id)
        self.assertEqual(newliveaction_db.status, status)
        self.assertEqual(newliveaction_db.end_timestamp, now)

        # Since liveaction has already been canceled, check that anymore update of
        # status, result, context, and end timestamp are not processed.
        now = get_datetime_utc_now()
        status = 'succeeded'
        result = 'Work is done.'
        context = {'third_party_id': uuid.uuid4().hex}
        newliveaction_db = action_db_utils.update_liveaction_status(
            status=status, result=result, context=context, end_timestamp=now,
            liveaction_id=liveaction_db.id)

        self.assertEqual(origliveaction_db.id, newliveaction_db.id)
        self.assertEqual(newliveaction_db.status, 'canceled')
        self.assertNotEqual(newliveaction_db.result, result)
        self.assertNotEqual(newliveaction_db.context, context)
        self.assertNotEqual(newliveaction_db.end_timestamp, now)

 def test_execution_update(self):
     liveaction = self.MODELS['liveactions']['liveaction1.yaml']
     executions_util.create_execution_object(liveaction)
     liveaction.status = 'running'
     pre_update_timestamp = date_utils.get_datetime_utc_now()
     executions_util.update_execution(liveaction)
     post_update_timestamp = date_utils.get_datetime_utc_now()
     execution = self._get_action_execution(liveaction__id=str(liveaction.id),
                                            raise_exception=True)
     self.assertEquals(len(execution.log), 2)
     self.assertEquals(execution.log[1]['status'], liveaction.status)
     self.assertGreater(execution.log[1]['timestamp'], pre_update_timestamp)
     self.assertLess(execution.log[1]['timestamp'], post_update_timestamp)

def _inject_instances(trigger, rate_per_trigger, duration, payload={}):
    start = date_utils.get_datetime_utc_now()
    elapsed = 0.0
    count = 0

    dispatcher = TriggerDispatcher()
    while elapsed < duration:
        # print('Dispatching trigger %s at time %s', trigger, date_utils.get_datetime_utc_now())
        dispatcher.dispatch(trigger, payload)
        delta = random.expovariate(rate_per_trigger)
        eventlet.sleep(delta)
        elapsed = (date_utils.get_datetime_utc_now() - start).seconds / 60.0
        count += 1

    print("%s: Emitted %d triggers in %d seconds" % (trigger, count, elapsed))

    def _get_api_client(self):
        """
        Retrieve API client instance.
        """
        token_expire = self._token_expire <= get_datetime_utc_now()

        if not self._client or token_expire:
            self._logger.audit('Creating new Client object.')
            ttl = (24 * 60 * 60)
            self._token_expire = get_datetime_utc_now() + timedelta(seconds=ttl)
            temporary_token = create_token(username=self._api_username, ttl=ttl)
            api_url = get_full_public_api_url()
            self._client = Client(api_url=api_url, token=temporary_token.token)

        return self._client

    def test_update_same_liveaction_status(self):
        liveaction_db = LiveActionDB()
        liveaction_db.status = 'requested'
        liveaction_db.start_timestamp = get_datetime_utc_now()
        liveaction_db.action = ResourceReference(
            name=ActionDBUtilsTestCase.action_db.name,
            pack=ActionDBUtilsTestCase.action_db.pack).ref
        params = {
            'actionstr': 'foo',
            'some_key_that_aint_exist_in_action_or_runner': 'bar',
            'runnerint': 555
        }
        liveaction_db.parameters = params
        liveaction_db = LiveAction.add_or_update(liveaction_db)
        origliveaction_db = copy.copy(liveaction_db)

        # Update by id.
        newliveaction_db = action_db_utils.update_liveaction_status(
            status='requested', liveaction_id=liveaction_db.id)

        # Verify id didn't change.
        self.assertEqual(origliveaction_db.id, newliveaction_db.id)
        self.assertEqual(newliveaction_db.status, 'requested')

        # Verify that state is not published.
        self.assertFalse(LiveActionPublisher.publish_state.called)

    def _purge_action_executions(self):
        """
        Purge action executions and corresponding live action, stdout and stderr object which match
        the criteria defined in the config.
        """
        LOG.info('Performing garbage collection for action executions and related objects')

        utc_now = get_datetime_utc_now()
        timestamp = (utc_now - datetime.timedelta(days=self._action_executions_ttl))

        # Another sanity check to make sure we don't delete new executions
        if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)):
            raise ValueError('Calculated timestamp would violate the minimum TTL constraint')

        timestamp_str = isotime.format(dt=timestamp)
        LOG.info('Deleting action executions older than: %s' % (timestamp_str))

        assert timestamp < utc_now

        try:
            purge_executions(logger=LOG, timestamp=timestamp)
        except Exception as e:
            LOG.exception('Failed to delete executions: %s' % (six.text_type(e)))

        return True

    def test_what_comes_in_goes_out(self):
        field = ComplexDateTimeField()

        date = date_utils.get_datetime_utc_now()
        us = field._datetime_to_microseconds_since_epoch(date)
        result = field._microseconds_since_epoch_to_datetime(us)
        self.assertEqual(date, result)

    def process(self, instance):
        trigger = instance['trigger']
        payload = instance['payload']

        trigger_instance = None
        try:
            trigger_instance = container_utils.create_trigger_instance(
                trigger,
                payload or {},
                date_utils.get_datetime_utc_now(),
                raise_on_no_trigger=True)
        except:
            # We got a trigger ref but we were unable to create a trigger instance.
            # This could be because a trigger object wasn't found in db for the ref.
            LOG.exception('Failed to create trigger_instance %s.', instance)
            return

        if trigger_instance:
            try:
                self.rules_engine.handle_trigger_instance(trigger_instance)
            except:
                # This could be a large message but at least in case of an exception
                # we get to see more context.
                # Beyond this point code cannot really handle the exception anyway so
                # eating up the exception.
                LOG.exception('Failed to handle trigger_instance %s.', instance)
                return

 def _get_execution_db_model(self, status=action_constants.LIVEACTION_STATUS_REQUESTED):
     start_timestamp = date_utils.get_datetime_utc_now()
     action_ref = ResourceReference(name='test_action', pack='test_pack').ref
     parameters = None
     live_action_db = LiveActionDB(status=status, start_timestamp=start_timestamp,
                                   action=action_ref, parameters=parameters)
     return action.LiveAction.add_or_update(live_action_db, publish=False)

 def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=False):
     # Include timestamp in the name.
     filename = filename.format(ts=str(date_utils.get_datetime_utc_now()).replace(' ', '_'),
                                pid=os.getpid())
     super(FormatNamedFileHandler, self).__init__(filename, mode=mode, maxBytes=maxBytes,
                                                  backupCount=backupCount, encoding=encoding,
                                                  delay=delay)

    def setup_action_models(cls):
        action_db = ActionDB()
        action_db.name = 'action-1'
        action_db.description = 'awesomeness'
        action_db.enabled = True
        action_db.pack = 'wolfpack'
        action_db.ref = ResourceReference(name=action_db.name, pack=action_db.pack).ref
        action_db.entry_point = ''
        action_db.runner_type = {'name': 'test-runner'}
        action_db.parameters = {
            'actionstr': {'type': 'string', 'position': 1, 'required': True},
            'actionint': {'type': 'number', 'default': 10, 'position': 0},
            'runnerdummy': {'type': 'string', 'default': 'actiondummy'}
        }
        ActionDBUtilsTestCase.action_db = Action.add_or_update(action_db)

        liveaction_db = LiveActionDB()
        liveaction_db.status = 'initializing'
        liveaction_db.start_timestamp = get_datetime_utc_now()
        liveaction_db.action = ActionDBUtilsTestCase.action_db.ref
        params = {
            'actionstr': 'foo',
            'some_key_that_aint_exist_in_action_or_runner': 'bar',
            'runnerint': 555
        }
        liveaction_db.parameters = params
        ActionDBUtilsTestCase.liveaction_db = LiveAction.add_or_update(liveaction_db)

    def test_notify_triggers_end_timestamp_none(self):
        liveaction_db = LiveActionDB(action='core.local')
        liveaction_db.id = bson.ObjectId()
        liveaction_db.description = ''
        liveaction_db.status = 'succeeded'
        liveaction_db.parameters = {}
        on_success = NotificationSubSchema(message='Action succeeded.')
        on_failure = NotificationSubSchema(message='Action failed.')
        liveaction_db.notify = NotificationSchema(on_success=on_success,
                                                  on_failure=on_failure)
        liveaction_db.start_timestamp = date_utils.get_datetime_utc_now()

        # This tests for end_timestamp being set to None, which can happen when a policy cancels
        # a request.
        # The assertions within "MockDispatcher.dispatch" will validate that the underlying code
        # handles this properly, so all we need to do is keep the call to "notifier.process" below
        liveaction_db.end_timestamp = None
        LiveAction.add_or_update(liveaction_db)

        execution = MOCK_EXECUTION
        execution.liveaction = vars(LiveActionAPI.from_model(liveaction_db))
        execution.status = liveaction_db.status

        dispatcher = NotifierTestCase.MockDispatcher(self)
        notifier = Notifier(connection=None, queues=[], trigger_dispatcher=dispatcher)
        notifier.process(execution)

    def _get_next_execution(self):
        """
        Sort execution requests by FIFO and priority and get the latest, highest priority item from
        the queue and pop it off.
        """
        query = {
            'scheduled_start_timestamp__lte': date.get_datetime_utc_now(),
            'handling': False,
            'limit': 1,
            'order_by': [
                '+scheduled_start_timestamp',
            ]
        }

        execution_queue_item_db = ActionExecutionSchedulingQueue.query(**query).first()

        if not execution_queue_item_db:
            return None

        # Mark that this scheduler process is currently handling (processing) that request
        # NOTE: This operation is atomic (CAS)
        execution_queue_item_db.handling = True

        try:
            ActionExecutionSchedulingQueue.add_or_update(execution_queue_item_db, publish=False)
            return execution_queue_item_db
        except db_exc.StackStormDBObjectWriteConflictError:
            LOG.info('Execution queue item handled by another scheduler: %s',
                     execution_queue_item_db.id)

        return None

    def test_liveaction_gets_deleted(self):
        now = date_utils.get_datetime_utc_now()
        start_ts = now - timedelta(days=15)
        end_ts = now - timedelta(days=14)

        liveaction_model = copy.deepcopy(self.models['liveactions']['liveaction4.yaml'])
        liveaction_model['start_timestamp'] = start_ts
        liveaction_model['end_timestamp'] = end_ts
        liveaction_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        liveaction = LiveAction.add_or_update(liveaction_model)

        # Write one execution before cut-off threshold
        exec_model = copy.deepcopy(self.models['executions']['execution1.yaml'])
        exec_model['start_timestamp'] = start_ts
        exec_model['end_timestamp'] = end_ts
        exec_model['status'] = action_constants.LIVEACTION_STATUS_SUCCEEDED
        exec_model['id'] = bson.ObjectId()
        exec_model['liveaction']['id'] = str(liveaction.id)
        ActionExecution.add_or_update(exec_model)

        liveactions = LiveAction.get_all()
        executions = ActionExecution.get_all()
        self.assertEqual(len(liveactions), 1)
        self.assertEqual(len(executions), 1)

        purge_executions(logger=LOG, timestamp=now - timedelta(days=10))

        liveactions = LiveAction.get_all()
        executions = ActionExecution.get_all()
        self.assertEqual(len(executions), 0)
        self.assertEqual(len(liveactions), 0)

    def _do_cancel(self, runner, runnertype_db, action_db, liveaction_db):
        try:
            extra = {"runner": runner}
            LOG.debug("Performing cancel for runner: %s", (runner.runner_id), extra=extra)

            runner.cancel()

            liveaction_db = update_liveaction_status(
                status=action_constants.LIVEACTION_STATUS_CANCELED,
                end_timestamp=date_utils.get_datetime_utc_now(),
                liveaction_db=liveaction_db,
            )

            executions.update_execution(liveaction_db)

            LOG.debug("Performing post_run for runner: %s", runner.runner_id)
            result = {"error": "Execution canceled by user."}
            runner.post_run(status=liveaction_db.status, result=result)
            runner.container_service = None
        except:
            _, ex, tb = sys.exc_info()
            # include the error message and traceback to try and provide some hints.
            result = {"error": str(ex), "traceback": "".join(traceback.format_tb(tb, 20))}
            LOG.exception("Failed to cancel action %s." % (liveaction_db.id), extra=result)
        finally:
            # Always clean-up the auth_token
            status = liveaction_db.status
            self._clean_up_auth_token(runner=runner, status=status)

        return liveaction_db

    def test_datastore_token_timeout(self):
        datastore_service = SensorDatastoreService(logger=mock.Mock(),
                                                   pack_name='core',
                                                   class_name='TestSensor',
                                                   api_username='sensor_service')

        mock_api_client = mock.Mock()
        kvp1 = KeyValuePair()
        kvp1.name = 'test1'
        kvp1.value = 'bar'
        mock_api_client.keys.get_by_id.return_value = kvp1

        token_expire_time = get_datetime_utc_now() - timedelta(seconds=5)
        datastore_service._client = mock_api_client
        datastore_service._token_expire = token_expire_time

        self._set_mock_api_client(mock_api_client)

        with mock.patch(
            'st2common.services.datastore.Client',
            return_value=mock_api_client
        ) as datastore_client:
            value = datastore_service.get_value(name='test1', local=False)
            self.assertTrue(datastore_client.called)
            self.assertEqual(value, kvp1.value)
            self.assertGreater(datastore_service._token_expire, token_expire_time)