def test_resume(self):
        # Launch the workflow execution.
        liveaction = LiveActionDB(action=WF1_NAME, parameters=ACTION_PARAMS)
        liveaction, execution = action_service.request(liveaction)
        liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING)

        mistral_context = liveaction.context.get('mistral', None)
        self.assertIsNotNone(mistral_context)
        self.assertEqual(mistral_context['execution_id'], WF1_EXEC.get('id'))
        self.assertEqual(mistral_context['workflow_name'], WF1_EXEC.get('workflow_name'))

        # Pause the workflow execution.
        requester = cfg.CONF.system_user.user
        liveaction, execution = action_service.request_pause(liveaction, requester)
        executions.ExecutionManager.update.assert_called_with(WF1_EXEC.get('id'), 'PAUSED')
        liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSING)

        # Manually update the liveaction from pausing to paused. The paused state
        # is usually updated by the mistral querier.
        action_service.update_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED)
        liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_PAUSED)

        # Resume the workflow execution.
        liveaction, execution = action_service.request_resume(liveaction, requester)
        executions.ExecutionManager.update.assert_called_with(WF1_EXEC.get('id'), 'RUNNING')
        liveaction = self._wait_on_status(liveaction, action_constants.LIVEACTION_STATUS_RUNNING)

    def test_over_threshold(self):
        policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency')
        self.assertGreater(policy_db.parameters['threshold'], 0)

        for i in range(0, policy_db.parameters['threshold']):
            liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
            action_service.request(liveaction)

        scheduled = LiveAction.get_all()
        self.assertEqual(len(scheduled), policy_db.parameters['threshold'])
        for liveaction in scheduled:
            self.assertIn(liveaction.status, SCHEDULED_STATES)

        # Execution is expected to be delayed since concurrency threshold is reached.
        liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
        liveaction, _ = action_service.request(liveaction)
        liveaction = LiveAction.get_by_id(str(liveaction.id))
        self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)

        # Mark one of the execution as completed.
        action_service.update_status(
            scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)

        # Execution is expected to be rescheduled.
        liveaction = LiveAction.get_by_id(str(liveaction.id))
        self.assertIn(liveaction.status, SCHEDULED_STATES)

    def test_over_threshold(self):
        policy_db = Policy.get_by_ref("wolfpack.action-1.concurrency.attr")
        self.assertGreater(policy_db.parameters["threshold"], 0)
        self.assertIn("actionstr", policy_db.parameters["attributes"])

        for i in range(0, policy_db.parameters["threshold"]):
            liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "fu"})
            action_service.request(liveaction)

        scheduled = LiveAction.get_all()
        self.assertEqual(len(scheduled), policy_db.parameters["threshold"])
        for liveaction in scheduled:
            self.assertIn(liveaction.status, SCHEDULED_STATES)

        # Execution is expected to be delayed since concurrency threshold is reached.
        liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "fu"})
        liveaction, _ = action_service.request(liveaction)
        delayed = LiveAction.get_by_id(str(liveaction.id))
        self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED)

        # Execution is expected to be scheduled since concurrency threshold is not reached.
        # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
        liveaction = LiveActionDB(action="wolfpack.action-1", parameters={"actionstr": "bar"})
        liveaction, _ = action_service.request(liveaction)
        liveaction = LiveAction.get_by_id(str(liveaction.id))
        self.assertIn(liveaction.status, SCHEDULED_STATES)

        # Mark one of the execution as completed.
        action_service.update_status(scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)

        # Execution is expected to be rescheduled.
        liveaction = LiveAction.get_by_id(str(delayed.id))
        self.assertIn(liveaction.status, SCHEDULED_STATES)

def recover_delayed_executions():
    coordinator = coordination.get_coordinator()
    dt_now = date_utils.get_datetime_utc_now()
    dt_delta = datetime.timedelta(seconds=cfg.CONF.scheduler.delayed_execution_recovery)
    dt_timeout = dt_now - dt_delta

    with coordinator.get_lock('st2-rescheduling-delayed-executions'):
        liveactions = LiveAction.query(status=action_constants.LIVEACTION_STATUS_DELAYED,
                                       start_timestamp__lte=dt_timeout,
                                       order_by=['start_timestamp'])

        if not liveactions:
            return

        LOG.info('There are %d liveactions that have been delayed for longer than %d seconds.',
                 len(liveactions), cfg.CONF.scheduler.delayed_execution_recovery)

        # Update status to requested and publish status for each liveactions.
        rescheduled = 0
        for instance in liveactions:
            try:
                action_service.update_status(instance,
                                             action_constants.LIVEACTION_STATUS_REQUESTED,
                                             publish=True)
                rescheduled += 1
            except:
                LOG.exception('Unable to reschedule liveaction. <LiveAction.id=%s>', instance.id)

        LOG.info('Rescheduled %d out of %d delayed liveactions.', len(liveactions), rescheduled)

    def test_retry_policy_applied_on_workflow_failure(self):
        wf_name = 'sequential'
        wf_ac_ref = TEST_PACK + '.' + wf_name
        wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, wf_name + '.yaml')
        lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
        lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
        lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
        self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING, lv_ac_db.result)

        # Ensure there is only one execution recorded.
        self.assertEqual(len(lv_db_access.LiveAction.query(action=wf_ac_ref)), 1)

        # Identify the records for the workflow and task.
        wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0]
        t1_ex_db = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))[0]
        t1_lv_ac_db = lv_db_access.LiveAction.query(task_execution=str(t1_ex_db.id))[0]
        t1_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(t1_ex_db.id))[0]

        # Manually set the status to fail.
        ac_svc.update_status(t1_lv_ac_db, ac_const.LIVEACTION_STATUS_FAILED)
        t1_lv_ac_db = lv_db_access.LiveAction.query(task_execution=str(t1_ex_db.id))[0]
        t1_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(t1_ex_db.id))[0]
        self.assertEqual(t1_ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
        notifier.get_notifier().process(t1_ac_ex_db)
        workflows.get_engine().process(t1_ac_ex_db)

        # Assert the main workflow is completed.
        ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(ac_ex_db.id))
        self.assertEqual(ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
        notifier.get_notifier().process(ac_ex_db)

        # Ensure execution is retried.
        self.assertEqual(len(lv_db_access.LiveAction.query(action=wf_ac_ref)), 2)

    def test_over_threshold_delay_executions(self):
        policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
        self.assertGreater(policy_db.parameters['threshold'], 0)
        self.assertIn('actionstr', policy_db.parameters['attributes'])

        for i in range(0, policy_db.parameters['threshold']):
            liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
            action_service.request(liveaction)

        scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
        self.assertEqual(len(scheduled), policy_db.parameters['threshold'])

        # Execution is expected to be delayed since concurrency threshold is reached.
        liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
        liveaction, _ = action_service.request(liveaction)
        delayed = LiveAction.get_by_id(str(liveaction.id))
        self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED)

        # Execution is expected to be scheduled since concurrency threshold is not reached.
        # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
        liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'})
        liveaction, _ = action_service.request(liveaction)
        liveaction = LiveAction.get_by_id(str(liveaction.id))
        self.assertIn(liveaction.status, SCHEDULED_STATES)

        # Mark one of the execution as completed.
        action_service.update_status(
            scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)

        # Execution is expected to be rescheduled.
        liveaction = LiveAction.get_by_id(str(delayed.id))
        self.assertIn(liveaction.status, SCHEDULED_STATES)

    def _apply_after(self, target):
        # Schedule the oldest delayed executions.
        filters = self._get_filters(target)
        filters["status"] = action_constants.LIVEACTION_STATUS_DELAYED
        requests = action_access.LiveAction.query(order_by=["start_timestamp"], limit=1, **filters)

        if requests:
            action_service.update_status(requests[0], action_constants.LIVEACTION_STATUS_REQUESTED, publish=True)

    def test_over_threshold_delay_executions(self):
        policy_db = Policy.get_by_ref('wolfpack.action-1.concurrency.attr')
        self.assertGreater(policy_db.parameters['threshold'], 0)
        self.assertIn('actionstr', policy_db.parameters['attributes'])

        for i in range(0, policy_db.parameters['threshold']):
            liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
            action_service.request(liveaction)

        scheduled = [item for item in LiveAction.get_all() if item.status in SCHEDULED_STATES]
        self.assertEqual(len(scheduled), policy_db.parameters['threshold'])

        # Assert the correct number of published states and action executions. This is to avoid
        # duplicate executions caused by accidental publishing of state in the concurrency policies.
        # num_state_changes = len(scheduled) * len(['requested', 'scheduled', 'running'])
        expected_num_exec = len(scheduled)
        expected_num_pubs = expected_num_exec * 3
        self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
        self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)

        # Execution is expected to be delayed since concurrency threshold is reached.
        liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'fu'})
        liveaction, _ = action_service.request(liveaction)
        expected_num_pubs += 1  # Tally requested state.

        # Assert the action is delayed.
        delayed = LiveAction.get_by_id(str(liveaction.id))
        self.assertEqual(delayed.status, action_constants.LIVEACTION_STATUS_DELAYED)
        self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
        self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)

        # Execution is expected to be scheduled since concurrency threshold is not reached.
        # The execution with actionstr "fu" is over the threshold but actionstr "bar" is not.
        liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'bar'})
        liveaction, _ = action_service.request(liveaction)
        expected_num_exec += 1  # This request is expected to be executed.
        expected_num_pubs += 3  # Tally requested, scheduled, and running states.

        liveaction = LiveAction.get_by_id(str(liveaction.id))
        self.assertIn(liveaction.status, SCHEDULED_STATES)
        self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
        self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)

        # Mark one of the execution as completed.
        action_service.update_status(
            scheduled[0], action_constants.LIVEACTION_STATUS_SUCCEEDED, publish=True)
        expected_num_pubs += 1  # Tally succeeded state.

        # Once capacity freed up, the delayed execution is published as requested again.
        expected_num_exec += 1  # The delayed request is expected to be executed.
        expected_num_pubs += 3  # Tally requested, scheduled, and running state.

        # Execution is expected to be rescheduled.
        liveaction = LiveAction.get_by_id(str(delayed.id))
        self.assertIn(liveaction.status, SCHEDULED_STATES)
        self.assertEqual(expected_num_pubs, LiveActionPublisher.publish_state.call_count)
        self.assertEqual(expected_num_exec, runner.MockActionRunner.run.call_count)

    def _apply_after(self, target):
        # Schedule the oldest delayed executions.
        requests = action_access.LiveAction.query(action=target.action,
                                                  status=action_constants.LIVEACTION_STATUS_DELAYED,
                                                  order_by=['start_timestamp'], limit=1)

        if requests:
            action_service.update_status(
                requests[0], action_constants.LIVEACTION_STATUS_REQUESTED, publish=True)

    def test_request_cancellation_uncancelable_state(self):
        request, execution = self._submit_request()
        self.assertIsNotNone(execution)
        self.assertEqual(execution.id, request.id)
        self.assertEqual(execution.status, action_constants.LIVEACTION_STATUS_REQUESTED)

        # Update execution status to FAILED.
        action_service.update_status(execution, action_constants.LIVEACTION_STATUS_FAILED, False)
        execution = action_db.get_liveaction_by_id(execution.id)
        self.assertEqual(execution.status, action_constants.LIVEACTION_STATUS_FAILED)

        # Request cancellation.
        self.assertRaises(Exception, action_service.request_cancellation, execution)

    def test_request_cancellation(self):
        request, execution = self._submit_request()
        self.assertIsNotNone(execution)
        self.assertEqual(execution.id, request.id)
        self.assertEqual(execution.status, action_constants.LIVEACTION_STATUS_REQUESTED)

        # Update execution status to RUNNING.
        action_service.update_status(execution, action_constants.LIVEACTION_STATUS_RUNNING, False)
        execution = action_db.get_liveaction_by_id(execution.id)
        self.assertEqual(execution.status, action_constants.LIVEACTION_STATUS_RUNNING)

        # Request cancellation.
        execution = self._submit_cancellation(execution)
        self.assertEqual(execution.status, action_constants.LIVEACTION_STATUS_CANCELING)

    def resume(self):
        # Restore runner and action parameters since they are not provided on resume.
        runner_parameters, action_parameters = param_utils.render_final_params(
            self.runner_type.runner_parameters,
            self.action.parameters,
            self.liveaction.parameters,
            self.liveaction.context
        )

        # Assign runner parameters needed for pre-run.
        if runner_parameters:
            self.runner_parameters = runner_parameters

        # Restore chain holder if it is not initialized.
        if not self.chain_holder:
            self.pre_run()

        # Change the status of the liveaction from resuming to running.
        self.liveaction = action_service.update_status(
            self.liveaction,
            action_constants.LIVEACTION_STATUS_RUNNING,
            publish=False
        )

        # Run the action chain.
        return self._run_chain(action_parameters, resuming=True)

    def _apply_before(self, target):
        # Get the count of scheduled instances of the action.
        scheduled = action_access.LiveAction.count(
            action=target.action, status=action_constants.LIVEACTION_STATUS_SCHEDULED)

        # Get the count of running instances of the action.
        running = action_access.LiveAction.count(
            action=target.action, status=action_constants.LIVEACTION_STATUS_RUNNING)

        count = scheduled + running

        # Mark the execution as scheduled if threshold is not reached or delayed otherwise.
        if count < self.threshold:
            LOG.debug('There are %s instances of %s in scheduled or running status. '
                      'Threshold of %s is not reached. Action execution will be scheduled.',
                      count, target.action, self._policy_ref)
            status = action_constants.LIVEACTION_STATUS_SCHEDULED
        else:
            action = 'delayed' if self.policy_action == 'delay' else 'canceled'
            LOG.debug('There are %s instances of %s in scheduled or running status. '
                      'Threshold of %s is reached. Action execution will be %s.',
                      count, target.action, self._policy_ref, action)
            status = self._get_status_for_policy_action(action=self.policy_action)

        # Update the status in the database. Publish status for cancellation so the
        # appropriate runner can cancel the execution. Other statuses are not published
        # because they will be picked up by the worker(s) to be processed again,
        # leading to duplicate action executions.
        publish = (status == action_constants.LIVEACTION_STATUS_CANCELING)
        target = action_service.update_status(target, status, publish=publish)

        return target

    def _apply_before(self, target):
        # Get the count of scheduled and running instances of the action.
        filters = self._get_filters(target)

        # Get the count of scheduled instances of the action.
        filters['status'] = action_constants.LIVEACTION_STATUS_SCHEDULED
        scheduled = action_access.LiveAction.count(**filters)

        # Get the count of running instances of the action.
        filters['status'] = action_constants.LIVEACTION_STATUS_RUNNING
        running = action_access.LiveAction.count(**filters)

        count = scheduled + running

        # Mark the execution as scheduled if threshold is not reached or delayed otherwise.
        if count < self.threshold:
            LOG.debug('There are %s instances of %s in scheduled or running status. '
                      'Threshold of %s is not reached. Action execution will be scheduled.',
                      count, target.action, self._policy_ref)
            status = action_constants.LIVEACTION_STATUS_SCHEDULED
        else:
            action = 'delayed' if self.policy_action == 'delay' else 'canceled'
            LOG.debug('There are %s instances of %s in scheduled or running status. '
                      'Threshold of %s is reached. Action execution will be %s.',
                      count, target.action, self._policy_ref, action)
            status = self._get_status_for_policy_action(action=self.policy_action)

        # Update the status in the database but do not publish.
        target = action_service.update_status(target, status, publish=False)

        return target

    def _update_to_scheduled(liveaction_db, execution_queue_item_db):
        liveaction_id = str(liveaction_db.id)
        queue_item_id = str(execution_queue_item_db.id)

        extra = {
            'liveaction_id': liveaction_id,
            'liveaction_status': liveaction_db.status,
            'queue_item_id': queue_item_id
        }

        # Update liveaction status to "scheduled".
        LOG.info('Liveaction (%s) Status Update to Scheduled 1: %s (%s)',
                liveaction_id, liveaction_db.status, queue_item_id, extra=extra)

        if liveaction_db.status in [action_constants.LIVEACTION_STATUS_REQUESTED,
                                    action_constants.LIVEACTION_STATUS_DELAYED]:
            liveaction_db = action_service.update_status(
                liveaction_db, action_constants.LIVEACTION_STATUS_SCHEDULED, publish=False)

        # Publish the "scheduled" status here manually. Otherwise, there could be a
        # race condition with the update of the action_execution_db if the execution
        # of the liveaction completes first.
        LiveAction.publish_status(liveaction_db)

        extra['liveaction_status'] = liveaction_db.status

        # Delete execution queue entry only after status is published.
        ActionExecutionSchedulingQueue.delete(execution_queue_item_db)
        LOG.info('Liveaction (%s) Status Update to Scheduled 2: %s (%s)',
                liveaction_id, liveaction_db.status, queue_item_id)

    def _apply_before(self, target):
        # Get the count of scheduled instances of the action.
        scheduled = action_access.LiveAction.count(
            action=target.action, status=action_constants.LIVEACTION_STATUS_SCHEDULED)

        # Get the count of running instances of the action.
        running = action_access.LiveAction.count(
            action=target.action, status=action_constants.LIVEACTION_STATUS_RUNNING)

        count = scheduled + running

        # Mark the execution as scheduled if threshold is not reached or delayed otherwise.
        if count < self.threshold:
            LOG.debug('There are %s instances of %s in scheduled or running status. '
                      'Threshold of %s is not reached. Action execution will be scheduled.',
                      count, target.action, self._policy_ref)
            status = action_constants.LIVEACTION_STATUS_SCHEDULED
        else:
            LOG.debug('There are %s instances of %s in scheduled or running status. '
                      'Threshold of %s is reached. Action execution will be delayed.',
                      count, target.action, self._policy_ref)
            status = action_constants.LIVEACTION_STATUS_DELAYED

        # Update the status in the database but do not publish.
        target = action_service.update_status(target, status, publish=False)

        return target

    def test_cancel_delayed_execution(self):
        liveaction = LiveActionDB(action='wolfpack.action-1', parameters={'actionstr': 'foo'})
        liveaction, _ = action_service.request(liveaction)
        liveaction = LiveAction.get_by_id(str(liveaction.id))
        self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_REQUESTED)

        # Manually update the liveaction from requested to delayed to mock concurrency policy.
        action_service.update_status(liveaction, action_constants.LIVEACTION_STATUS_DELAYED)
        liveaction = LiveAction.get_by_id(str(liveaction.id))
        self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_DELAYED)

        # Cancel execution.
        action_service.request_cancellation(liveaction, cfg.CONF.system_user.user)

        # Cancel is only called when liveaction is still in running state.
        # Otherwise, the cancellation is only a state change.
        self.assertFalse(runners.ActionRunner.cancel.called)
        liveaction = LiveAction.get_by_id(str(liveaction.id))
        self.assertEqual(liveaction.status, action_constants.LIVEACTION_STATUS_CANCELED)

    def _invoke_action(self, action_db, runnertype_db, params, context=None,
                       additional_contexts=None):
        """
        Schedule an action execution.

        :type action_exec_spec: :class:`ActionExecutionSpecDB`

        :param params: Partially rendered parameters to execute the action with.
        :type params: ``dict``

        :rtype: :class:`LiveActionDB` on successful scheduling, None otherwise.
        """
        action_ref = action_db.ref
        runnertype_db = action_utils.get_runnertype_by_name(action_db.runner_type['name'])

        liveaction_db = LiveActionDB(action=action_ref, context=context, parameters=params)

        try:
            liveaction_db.parameters = self.get_resolved_parameters(
                runnertype_db=runnertype_db,
                action_db=action_db,
                params=liveaction_db.parameters,
                context=liveaction_db.context,
                additional_contexts=additional_contexts)
        except param_exc.ParamException as e:
            # We still need to create a request, so liveaction_db is assigned an ID
            liveaction_db, execution_db = action_service.create_request(liveaction_db)

            # By this point the execution is already in the DB therefore need to mark it failed.
            _, e, tb = sys.exc_info()
            action_service.update_status(
                liveaction=liveaction_db,
                new_status=action_constants.LIVEACTION_STATUS_FAILED,
                result={'error': six.text_type(e),
                        'traceback': ''.join(traceback.format_tb(tb, 20))})

            # Might be a good idea to return the actual ActionExecution rather than bubble up
            # the exception.
            raise validation_exc.ValueValidationException(six.text_type(e))

        liveaction_db, execution_db = action_service.request(liveaction_db)

        return execution_db

    def _schedule_execution(self, liveaction, user=None):
        # Initialize execution context if it does not exist.
        if not hasattr(liveaction, 'context'):
            liveaction.context = dict()

        liveaction.context['user'] = user
        LOG.debug('User is: %s' % liveaction.context['user'])

        # Retrieve other st2 context from request header.
        if 'st2-context' in pecan.request.headers and pecan.request.headers['st2-context']:
            context = jsonify.try_loads(pecan.request.headers['st2-context'])
            if not isinstance(context, dict):
                raise ValueError('Unable to convert st2-context from the headers into JSON.')
            liveaction.context.update(context)

        # Schedule the action execution.
        liveaction_db = LiveActionAPI.to_model(liveaction)
        liveaction_db, actionexecution_db = action_service.create_request(liveaction_db)

        action_db = action_utils.get_action_by_ref(liveaction_db.action)
        runnertype_db = action_utils.get_runnertype_by_name(action_db.runner_type['name'])

        try:
            liveaction_db.parameters = param_utils.render_live_params(
                runnertype_db.runner_parameters, action_db.parameters, liveaction_db.parameters,
                liveaction_db.context)
        except ParamException:
            # By this point the execution is already in the DB therefore need to mark it failed.
            _, e, tb = sys.exc_info()
            action_service.update_status(
                liveaction=liveaction_db,
                new_status=LIVEACTION_STATUS_FAILED,
                result={'error': str(e), 'traceback': ''.join(traceback.format_tb(tb, 20))})
            # Might be a good idea to return the actual ActionExecution rather than bubble up
            # the execption.
            raise ValueValidationException(str(e))

        liveaction_db = LiveAction.add_or_update(liveaction_db, publish=False)

        _, actionexecution_db = action_service.publish_request(liveaction_db, actionexecution_db)
        from_model_kwargs = self._get_from_model_kwargs_for_request(request=pecan.request)
        return ActionExecutionAPI.from_model(actionexecution_db, from_model_kwargs)

    def test_no_retry_policy_applied_on_task_failure(self):
        wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, 'subworkflow.yaml')
        lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
        lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
        lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
        self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_RUNNING, lv_ac_db.result)

        # Identify the records for the main workflow.
        wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(ac_ex_db.id))[0]
        tk_ex_dbs = wf_db_access.TaskExecution.query(workflow_execution=str(wf_ex_db.id))
        self.assertEqual(len(tk_ex_dbs), 1)

        # Identify the records for the tasks.
        t1_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(tk_ex_dbs[0].id))[0]
        t1_wf_ex_db = wf_db_access.WorkflowExecution.query(action_execution=str(t1_ac_ex_db.id))[0]
        self.assertEqual(t1_ac_ex_db.status, ac_const.LIVEACTION_STATUS_RUNNING)
        self.assertEqual(t1_wf_ex_db.status, wf_statuses.RUNNING)

        # Ensure there is only one execution for the task.
        tk_ac_ref = TEST_PACK + '.' + 'sequential'
        self.assertEqual(len(lv_db_access.LiveAction.query(action=tk_ac_ref)), 1)

        # Fail the subtask of the subworkflow.
        t1_t1_ex_db = wf_db_access.TaskExecution.query(workflow_execution=str(t1_wf_ex_db.id))[0]
        t1_t1_lv_ac_db = lv_db_access.LiveAction.query(task_execution=str(t1_t1_ex_db.id))[0]
        ac_svc.update_status(t1_t1_lv_ac_db, ac_const.LIVEACTION_STATUS_FAILED)
        t1_t1_ac_ex_db = ex_db_access.ActionExecution.query(task_execution=str(t1_t1_ex_db.id))[0]
        self.assertEqual(t1_t1_ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
        notifier.get_notifier().process(t1_t1_ac_ex_db)
        workflows.get_engine().process(t1_t1_ac_ex_db)

        # Ensure the task execution is not retried.
        self.assertEqual(len(lv_db_access.LiveAction.query(action=tk_ac_ref)), 1)

        # Process the failure of the subworkflow.
        t1_ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(t1_ac_ex_db.id))
        self.assertEqual(t1_ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)
        workflows.get_engine().process(t1_ac_ex_db)

        # Assert the main workflow is completed.
        ac_ex_db = ex_db_access.ActionExecution.get_by_id(str(ac_ex_db.id))
        self.assertEqual(ac_ex_db.status, ac_const.LIVEACTION_STATUS_FAILED)