def test_scheduler_doesnt_handel_calls_the_failed_on_update(self):
        def stop_thread_groups():
            [tg.stop() for tg in self.tgs]

        self.tgs = [scheduler.setup(), scheduler.setup()]
        self.addCleanup(stop_thread_groups)

        method_args = {'name': 'task', 'id': '321'}

        scheduler.schedule_call(
            None,
            TARGET_METHOD_NAME,
            DELAY,
            **method_args
        )

        time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
        calls = db_api.get_delayed_calls_to_start(time_filter)

        eventlet.sleep(WAIT)

        # If the scheduler does handel calls that failed on update
        # NotFoundException will raise.
        db_api.get_delayed_call(calls[0].id)

        db_api.delete_delayed_call(calls[0].id)

    def test_scheduler_call_target_method_with_correct_auth(self, method):
        method.side_effect = self.target_check_context_method

        default_context = base.get_context(default=True)
        auth_context.set_ctx(default_context)
        default_project_id = (
            default_context.project_id
        )

        scheduler.schedule_call(
            None,
            TARGET_METHOD_PATH,
            DELAY,
            **{'expected_project_id': default_project_id}
        )

        second_context = base.get_context(default=False)
        auth_context.set_ctx(second_context)
        second_project_id = (
            second_context.project_id
        )

        scheduler.schedule_call(
            None,
            TARGET_METHOD_PATH,
            DELAY,
            **{'expected_project_id': second_project_id}
        )

        self.assertNotEqual(default_project_id, second_project_id)

        for _ in range(2):
            self.assertTrue(self.queue.get())

    def test_scheduler_delete_calls(self, method):
        def stop_thread_groups():
            [tg.stop() for tg in self.tgs]

        self.tgs = [scheduler.setup(), scheduler.setup()]
        self.addCleanup(stop_thread_groups)

        method_args = {'name': 'task', 'id': '321'}

        scheduler.schedule_call(
            None,
            TARGET_METHOD_NAME,
            DELAY,
            **method_args
        )

        time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
        calls = db_api.get_delayed_calls_to_start(time_filter)

        self._assert_single_item(calls, target_method_name=TARGET_METHOD_NAME)

        eventlet.sleep(WAIT)

        self.assertRaises(exc.NotFoundException,
                          db_api.get_delayed_call,
                          calls[0].id
                          )

    def before_task_start(self, task_ex, task_spec):
        super(WaitBeforePolicy, self).before_task_start(task_ex, task_spec)

        context_key = "wait_before_policy"

        runtime_context = _ensure_context_has_key(task_ex.runtime_context, context_key)

        task_ex.runtime_context = runtime_context

        policy_context = runtime_context[context_key]

        if policy_context.get("skip"):
            # Unset state 'DELAYED'.
            wf_trace.info(task_ex, "Task '%s' [%s -> %s]" % (task_ex.name, states.DELAYED, states.RUNNING))

            task_ex.state = states.RUNNING

            return

        if task_ex.state != states.IDLE:
            policy_context.update({"skip": True})
            _log_task_delay(task_ex, self.delay)

            task_ex.state = states.DELAYED

            scheduler.schedule_call(None, _RUN_EXISTING_TASK_PATH, self.delay, task_ex_id=task_ex.id)

def _schedule_run_workflow(task_ex, task_spec, wf_input, index):
    parent_wf_ex = task_ex.workflow_execution
    parent_wf_spec = spec_parser.get_workflow_spec(parent_wf_ex.spec)

    wf_spec_name = task_spec.get_workflow_name()

    wf_def = e_utils.resolve_workflow_definition(
        parent_wf_ex.workflow_name,
        parent_wf_spec.get_name(),
        wf_spec_name
    )

    wf_spec = spec_parser.get_workflow_spec(wf_def.spec)

    wf_params = {
        'task_execution_id': task_ex.id,
        'with_items_index': index
    }

    if 'env' in parent_wf_ex.params:
        wf_params['env'] = parent_wf_ex.params['env']

    for k, v in wf_input.items():
        if k not in wf_spec.get_input():
            wf_params[k] = v
            del wf_input[k]

    scheduler.schedule_call(
        None,
        'mistral.engine.task_handler.run_workflow',
        0,
        wf_name=wf_def.name,
        wf_input=wf_input,
        wf_params=wf_params
    )

def schedule_on_action_complete(action_ex, delay=0):
    """Schedules task completion check.

    This method provides transactional decoupling of action completion from
    task completion check. It's needed in non-locking model in order to
    avoid 'phantom read' phenomena when reading state of multiple actions
    to see if a task is completed. Just starting a separate transaction
    without using scheduler is not safe due to concurrency window that we'll
    have in this case (time between transactions) whereas scheduler is a
    special component that is designed to be resistant to failures.

    :param action_ex: Action execution.
    :param delay: Minimum amount of time before task completion check
        should be made.
    """

    # Optimization to avoid opening a new transaction if it's not needed.
    if not action_ex.task_execution.spec.get('with-items'):
        _on_action_complete(action_ex)

        return

    key = 'th_on_a_c-%s' % action_ex.task_execution_id

    scheduler.schedule_call(
        None,
        _SCHEDULED_ON_ACTION_COMPLETE_PATH,
        delay,
        key=key,
        action_ex_id=action_ex.id,
        wf_action=isinstance(action_ex, models.WorkflowExecution)
    )

def _schedule_send_result_to_parent_workflow(wf_ex):
    scheduler.schedule_call(
        None,
        'mistral.engine.workflow_handler.send_result_to_parent_workflow',
        0,
        wf_ex_id=wf_ex.id
    )

def _schedule_refresh_task_state(task_ex, delay=0):
    """Schedules task preconditions check.

    This method provides transactional decoupling of task preconditions
    check from events that can potentially satisfy those preconditions.

    It's needed in non-locking model in order to avoid 'phantom read'
    phenomena when reading state of multiple tasks to see if a task that
    depends on them can start. Just starting a separate transaction
    without using scheduler is not safe due to concurrency window that
    we'll have in this case (time between transactions) whereas scheduler
    is a special component that is designed to be resistant to failures.

    :param task_ex: Task execution.
    :param delay: Delay.
    """
    key = 'th_c_t_s_a-%s' % task_ex.id

    scheduler.schedule_call(
        None,
        _REFRESH_TASK_STATE_PATH,
        delay,
        key=key,
        task_ex_id=task_ex.id
    )

    def schedule(self, input_dict, target, index=0, desc=''):
        assert not self.action_ex

        # Assign the action execution ID here to minimize database calls.
        # Otherwise, the input property of the action execution DB object needs
        # to be updated with the action execution ID after the action execution
        # DB object is created.
        action_ex_id = utils.generate_unicode_uuid()

        self._insert_action_context(action_ex_id, input_dict)

        self._create_action_execution(
            self._prepare_input(input_dict),
            self._prepare_runtime_context(index),
            desc=desc,
            action_ex_id=action_ex_id
        )

        scheduler.schedule_call(
            None,
            _RUN_EXISTING_ACTION_PATH,
            0,
            action_ex_id=self.action_ex.id,
            target=target
        )

    def test_scheduler_doesnt_handle_calls_the_failed_on_update(
            self,
            update_delayed_call):
        def update_call_failed(id, values, query_filter):
            self.queue.put("item")
            return None, 0

        update_delayed_call.side_effect = update_call_failed

        scheduler.schedule_call(
            None,
            TARGET_METHOD_PATH,
            DELAY,
            **{'name': 'task', 'id': '321'}
        )

        calls = db_api.get_delayed_calls_to_start(get_time_delay())

        self.queue.get()
        eventlet.sleep(1)

        update_delayed_call.assert_called_with(
            id=calls[0].id,
            values=mock.ANY,
            query_filter=mock.ANY
        )
        # If the scheduler does handel calls that failed on update
        # DBEntityNotFoundException will raise.
        db_api.get_delayed_call(calls[0].id)
        db_api.delete_delayed_call(calls[0].id)

    def test_scheduler_without_factory(self, method):
        method_args = {'name': 'task', 'id': '321'}

        scheduler.schedule_call(
            None,
            FACTORY_METHOD_PATH,
            DELAY,
            **method_args
        )

        time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
        calls = db_api.get_delayed_calls_to_start(time_filter)

        call = self._assert_single_item(
            calls,
            target_method_name=FACTORY_METHOD_PATH
        )

        self.assertIn('name', call['method_arguments'])

        eventlet.sleep(WAIT)

        method.assert_called_once_with(name='task', id='321')

        time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
        calls = db_api.get_delayed_calls_to_start(time_filter)

        self.assertEqual(0, len(calls))

    def test_scheduler_multi_instance(self, method):
        def stop_thread_groups():
            [tg.stop() for tg in self.tgs]

        self.tgs = [scheduler.setup(), scheduler.setup()]
        self.addCleanup(stop_thread_groups)

        method_args = {'name': 'task', 'id': '321'}

        scheduler.schedule_call(
            None,
            TARGET_METHOD_PATH,
            DELAY,
            **method_args
        )

        time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
        calls = db_api.get_delayed_calls_to_start(time_filter)

        self._assert_single_item(calls, target_method_name=TARGET_METHOD_PATH)

        eventlet.sleep(WAIT)

        method.assert_called_once_with(name='task', id='321')

        time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
        calls = db_api.get_delayed_calls_to_start(time_filter)

        self.assertEqual(0, len(calls))

    def after_task_complete(self, task_ex, task_spec):
        super(WaitAfterPolicy, self).after_task_complete(task_ex, task_spec)

        context_key = 'wait_after_policy'

        runtime_context = _ensure_context_has_key(
            task_ex.runtime_context,
            context_key
        )

        task_ex.runtime_context = runtime_context

        policy_context = runtime_context[context_key]
        if policy_context.get('skip'):
            # Skip, already processed.
            return

        policy_context.update({'skip': True})

        _log_task_delay(task_ex, self.delay)

        state = task_ex.state
        # Set task state to 'DELAYED'.
        task_ex.state = states.RUNNING_DELAYED

        # Schedule to change task state to RUNNING again.
        scheduler.schedule_call(
            _ENGINE_CLIENT_PATH,
            'on_task_state_change',
            self.delay,
            state=state,
            task_ex_id=task_ex.id,
        )

def _schedule_run_action(task_ex, task_spec, action_input, index):
    wf_ex = task_ex.workflow_execution
    wf_spec = spec_parser.get_workflow_spec(wf_ex.spec)

    action_spec_name = task_spec.get_action_name()

    action_def = action_handler.resolve_definition(
        action_spec_name,
        task_ex,
        wf_spec
    )

    action_ex = action_handler.create_action_execution(
        action_def, action_input, task_ex, index
    )

    target = expr.evaluate_recursively(
        task_spec.get_target(),
        utils.merge_dicts(
            copy.deepcopy(action_input),
            copy.copy(task_ex.in_context)
        )
    )

    scheduler.schedule_call(
        None,
        'mistral.engine.action_handler.run_existing_action',
        0,
        action_ex_id=action_ex.id,
        target=target
    )

    def test_scheduler_with_factory(self, factory):
        target_method_name = 'run_something'
        factory.return_value = type(
            'something',
            (object,),
            {
                target_method_name:
                    mock.MagicMock(side_effect=self.target_method)
            }
        )

        scheduler.schedule_call(
            TARGET_METHOD_PATH,
            target_method_name,
            DELAY,
            **{'name': 'task', 'id': '123'}
        )

        calls = db_api.get_delayed_calls_to_start(get_time_delay())
        call = self._assert_single_item(
            calls,
            target_method_name=target_method_name
        )
        self.assertIn('name', call['method_arguments'])

        self.queue.get()
        factory().run_something.assert_called_once_with(name='task', id='123')

        calls = db_api.get_delayed_calls_to_start(get_time_delay())
        self.assertEqual(0, len(calls))

    def test_scheduler_with_custom_batch_size(self):
        self.scheduler.stop()

        number_delayed_calls = 5
        processed_calls_at_time = []
        real_delete_calls_method = scheduler.Scheduler.delete_calls

        @staticmethod
        def delete_calls_counter(delayed_calls):
            real_delete_calls_method(delayed_calls)

            for _ in range(len(delayed_calls)):
                self.queue.put("item")
            processed_calls_at_time.append(len(delayed_calls))

        scheduler.Scheduler.delete_calls = delete_calls_counter

        # Create 5 delayed calls
        for i in range(number_delayed_calls):
            scheduler.schedule_call(
                None,
                TARGET_METHOD_PATH,
                0,
                **{'name': 'task', 'id': i}
            )

        # Start scheduler which process 2 calls at a time
        self.scheduler = scheduler.Scheduler(0, 1, 2)
        self.scheduler.start()

        # Wait when all of calls will be processed
        for _ in range(number_delayed_calls):
            self.queue.get()

        self.assertListEqual([1, 2, 2], sorted(processed_calls_at_time))

 def _schedule_send_result_to_parent_workflow(self):
     scheduler.schedule_call(
         None,
         _SEND_RESULT_TO_PARENT_WORKFLOW_PATH,
         0,
         wf_ex_id=self.wf_ex.id
     )

    def test_scheduler_with_factory(self, factory):
        target_method = 'run_something'
        method_args = {'name': 'task', 'id': '123'}

        scheduler.schedule_call(
            FACTORY_METHOD_PATH,
            target_method,
            DELAY,
            **method_args
        )

        calls = db_api.get_delayed_calls_to_start(
            datetime.datetime.now() + datetime.timedelta(seconds=2)
        )

        call = self._assert_single_item(
            calls,
            target_method_name=target_method
        )

        self.assertIn('name', call['method_arguments'])

        eventlet.sleep(WAIT)

        factory().run_something.assert_called_once_with(name='task', id='123')

        time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
        calls = db_api.get_delayed_calls_to_start(time_filter)

        self.assertEqual(0, len(calls))

    def after_task_complete(self, task_ex, task_spec):
        """Possible Cases:

        1. state = SUCCESS
           if continue_on is not specified,
           no need to move to next iteration;
           if current:count achieve retry:count then policy
           breaks the loop (regardless on continue-on condition);
           otherwise - check continue_on condition and if
           it is True - schedule the next iteration,
           otherwise policy breaks the loop.
        2. retry:count = 5, current:count = 2, state = ERROR,
           state = IDLE/DELAYED, current:count = 3
        3. retry:count = 5, current:count = 4, state = ERROR
        Iterations complete therefore state = #{state}, current:count = 4.
        """
        super(RetryPolicy, self).after_task_complete(task_ex, task_spec)

        context_key = "retry_task_policy"

        runtime_context = _ensure_context_has_key(task_ex.runtime_context, context_key)

        continue_on_evaluation = expressions.evaluate(
            self._continue_on_clause, data_flow.evaluate_task_outbound_context(task_ex)
        )

        task_ex.runtime_context = runtime_context

        state = task_ex.state

        if not states.is_completed(state):
            return

        policy_context = runtime_context[context_key]

        retry_no = 0

        if "retry_no" in policy_context:
            retry_no = policy_context["retry_no"]
            del policy_context["retry_no"]

        retries_remain = retry_no + 1 < self.count

        stop_continue_flag = task_ex.state == states.SUCCESS and not self._continue_on_clause
        stop_continue_flag = stop_continue_flag or (self._continue_on_clause and not continue_on_evaluation)
        break_triggered = task_ex.state == states.ERROR and self.break_on

        if not retries_remain or break_triggered or stop_continue_flag:
            return

        _log_task_delay(task_ex, self.delay)

        data_flow.invalidate_task_execution_result(task_ex)
        task_ex.state = states.DELAYED

        policy_context["retry_no"] = retry_no + 1
        runtime_context[context_key] = policy_context

        scheduler.schedule_call(None, _RUN_EXISTING_TASK_PATH, self.delay, task_ex_id=task_ex.id)

    def after_task_complete(self, task_ex, task_spec):
        """Possible Cases:

        1. state = SUCCESS
           No need to move to next iteration.
        2. retry:count = 5, current:count = 2, state = ERROR,
           state = IDLE/DELAYED, current:count = 3
        3. retry:count = 5, current:count = 4, state = ERROR
        Iterations complete therefore state = #{state}, current:count = 4.
        """
        super(RetryPolicy, self).after_task_complete(task_ex, task_spec)

        context_key = 'retry_task_policy'

        runtime_context = _ensure_context_has_key(
            task_ex.runtime_context,
            context_key
        )

        task_ex.runtime_context = runtime_context

        state = task_ex.state

        if state != states.ERROR:
            return

        wf_trace.info(
            task_ex,
            "Task '%s' [%s -> ERROR]"
            % (task_ex.name, task_ex.state)
        )

        policy_context = runtime_context[context_key]

        retry_no = 0

        if 'retry_no' in policy_context:
            retry_no = policy_context['retry_no']
            del policy_context['retry_no']

        retries_remain = retry_no + 1 < self.count

        if not retries_remain or self.break_on:
            return

        _log_task_delay(task_ex, self.delay)

        task_ex.state = states.DELAYED

        policy_context['retry_no'] = retry_no + 1
        runtime_context[context_key] = policy_context

        scheduler.schedule_call(
            None,
            _RUN_EXISTING_TASK_PATH,
            self.delay,
            task_ex_id=task_ex.id,
        )