def test_on_ended(self, run, move_completed):
        """Once a step ends the FSM checks if it completed or else"""
        f = FSM(state_id)

        consume_completed = [
            mock.Mock(name="method_mocked"),
            mock.Mock(name="properties_mocked"),
            json.dumps(msg_completed)
        ]

        consume_errored = [
            mock.Mock(name="method_mocked"),
            mock.Mock(name="properties_mocked"),
            json.dumps(msg_errored)
        ]

        # First check in the case where the job completed
        result = f.on_ended(f.ch, *consume_completed)
        f.move_active_to_completed.assert_called_once_with()
        f._run.assert_called_once_with()

        # Check the case where the job ended not "completed"
        f.move_active_to_completed.reset_mock()
        f._run.reset_mock()
        result = f.on_ended(f.ch, *consume_errored)

        self.assertFalse(f.move_active_to_completed.called)
        self.assertFalse(f._run.called)
        self.assertFalse(result)

    def test__run(self, setup, dequeue, on_started):
        """The _run() method can send a proper message to a worker"""
        f = FSM(state_id)
        f.reply_queue = temp_queue

        f.group = "mock tests"
        f.dynamic = {}
        f.active_step = _active_step_string
        f.active_sequence = new_active_sequence()
        f.execution = new_playbook()['execution']

        consume_iter = [(
            mock.Mock(name="method_mocked"),
            mock.Mock(name="properties_mocked"),
            json.dumps(msg_completed),
        )]

        publish = mock.Mock()
        channel = mock.Mock()
        channel.consume.return_value = iter(consume_iter)
        channel.basic_publish = publish
        f.ch = channel

        with mock.patch('recore.amqp.MQ_CONF') as mq_conf:
            mq_conf = MQ_CONF
            f._run()

        setup.assert_called_once_with()
        dequeue.assert_called_once_with()
        f.ch.basic_ack.assert_called_once_with(consume_iter[0][0].delivery_tag)
        f.ch.cancel.assert_called_once_with()
        on_started.assert_called_once_with(f.ch, *consume_iter[0])

 def test__run_finished(self, setup, cleanup):
     """When the FSM is out of steps it raises IndexError and calls _cleanup"""
     f = FSM(state_id)
     result = f._run()
     f._setup.assert_called_once_with()
     f.dequeue_next_active_step.assert_called_once_with()
     cleanup.assert_called_once_with()
     self.assertTrue(result)

    def test_step_notification_started_no_notifications(self, send_notification, on_started, dequeue_step, setup):
        """Per-step notifications don't happen if no notifications are defined"""
        f = FSM(state_id)

        msg_started = {'status': 'started'}

        consume_iter = [
            (mock.Mock(name="method_mocked"),
             mock.Mock(name="properties_mocked"),
             json.dumps(msg_started))
        ]

        # Pre-test scaffolding. Hard-code some mocked out
        # attributes/variables because we're skipping the usual
        # initialization steps.
        f.conn = mock.Mock(pika.connection.Connection)
        publish = mock.Mock()
        channel = mock.Mock()
        channel.consume.return_value = iter(consume_iter)
        channel.basic_publish = publish
        f.ch = channel
        f.active_sequence = {'hosts': ['localhost']}
        f.group = 'testgroup'
        f.dynamic = {}
        f.active_step = new_notify_step()

        # Run the method now. It should terminate when it reaches the
        # end of _run() with a call to a mocked out on_started()
        with mock.patch('recore.amqp.MQ_CONF') as mq_conf:
            mq_conf = MQ_CONF
            f._run()

        self.assertEqual(send_notification.call_count, 0)

    def test__cleanup_failed(self):
        """Cleanup fails if update_state raises"""
        f = FSM(state_id)
        f.ch = mock.Mock(pika.channel.Channel)
        f.conn = mock.Mock(pika.connection.Connection)

        with mock.patch.object(f, 'update_state',
                               mock.Mock(side_effect=Exception("derp"))) as (
                us_exception):
            with self.assertRaises(Exception):
                f._cleanup()

    def test_step_notification_started_two_transports(self, send_notification, on_started, dequeue_step, setup):
        """Per-step notifications happen for all defined transports

Tests for the case where multiple notification transports (irc, email, etc) are defined"""

        f = FSM(TEST_PBID, state_id)

        msg_started = {'status': 'started'}

        consume_iter = [
            (mock.Mock(name="method_mocked"),
             mock.Mock(name="properties_mocked"),
             json.dumps(msg_started))
        ]

        # Pre-test scaffolding. Hard-code some mocked out
        # attributes/variables because we're skipping the usual
        # initialization steps.
        f.conn = mock.Mock(pika.connection.Connection)
        publish = mock.Mock()
        channel = mock.Mock()
        channel.consume.return_value = iter(consume_iter)
        channel.basic_publish = publish
        f.ch = channel
        f.active_sequence = {'hosts': ['localhost']}
        f.group = 'testgroup'
        f.dynamic = {}

        _step = {
            "service:Restart": {
                "service": "megafrobber",
                "notify": {
                    "started": {
                        "irc": ['#achannel'],
                        "email": ['[email protected]']
                    }
                }
            }
        }

        f.active_step = _step

        # Run the method now. It should terminate when it reaches the
        # end of _run() with a call to a mocked out on_started()
        with mock.patch('recore.amqp.MQ_CONF') as mq_conf:
            mq_conf = MQ_CONF
            set_field = mock.MagicMock()
            filter = mock.MagicMock(return_value=set_field)
            f.filter = filter
            f._run()

        self.assertEqual(send_notification.call_count, 2)

    def test__run(self, setup, dequeue, on_started):
        """The _run() method can send a proper message to a worker"""
        f = FSM(state_id)
        f.reply_queue = temp_queue

        f.project = "mock tests"
        f.dynamic = {}
        f.active = {
            'plugin': 'fake',
            'parameters': {'no': 'parameters'}
        }
        consume_iter = [
            (mock.Mock(name="method_mocked"),
             mock.Mock(name="properties_mocked"),
             json.dumps(msg_completed))
         ]

        publish = mock.Mock()
        channel = mock.Mock()
        channel.consume.return_value = iter(consume_iter)
        channel.basic_publish = publish
        f.ch = channel

        f._run()

        setup.assert_called_once_with()
        dequeue.assert_called_once_with()
        f.ch.basic_ack.assert_called_once_with(consume_iter[0][0].delivery_tag)
        f.ch.cancel.assert_called_once_with()
        on_started.assert_called_once_with(f.ch, *consume_iter[0])

    def test_dequeue_next_active_step(self):
        """The FSM can remove the next step and update Mongo with it"""
        f = FSM(TEST_PBID, state_id)
        f.active_step = _active_step_string
        f.active_sequence = new_active_sequence()
        f.executed = []
        f.execution = new_playbook()['execution']

        _update_state = {
            '$set': {
                'active_step': _active_step_string,
                'active_sequence': f.active_sequence,
                'executed': [],
                'execution': new_playbook()['execution'],
            }
        }

        with mock.patch.object(f, 'update_state') as (
                us):
            set_field = mock.MagicMock()
            filter = mock.MagicMock(return_value=set_field)
            f.filter = filter
            f.dequeue_next_active_step()
            us.assert_called_once_with(_update_state)
            self.assertEqual(f.active_step, _active_step_string)

    def test__connect_mq(self, connection, channel, creds):
        """FSM connecting to AMQP sets its reply_queue attribute"""
        with mock.patch.dict('recore.fsm.recore.amqp.MQ_CONF', MQ_CONF):
            creds.return_value = mock.MagicMock(spec=pika.credentials.PlainCredentials, name="mocked creds")
            mocked_conn = mock.MagicMock(spec=pika.connection.Connection, name="mocked connection")
            mocked_channel = mock.MagicMock(spec=pika.channel.Channel, name="mocked channel")
            channel.return_value = mocked_channel
            channel.queue_declare.return_value.method.queue = temp_queue
            mocked_conn.channel.return_value = channel
            connection.return_value = mocked_conn

            f = FSM(TEST_PBID, state_id)
            (ch, conn) = f._connect_mq()
            self.assertEqual(f.reply_queue, temp_queue, msg="Expected %s for reply_queue, instead got %s" %
                             (temp_queue, f.reply_queue))

    def test__setup_lookup_state_none(self):
        """if lookup_state returns None then a LookupError is raised"""
        f = FSM(state_id)

        with mock.patch('recore.mongo.database') as (
                mongo.database):
            mongo.database = mock.MagicMock(pymongo.database.Database)
            mongo.database.__getitem__.return_value = mock.MagicMock(pymongo.collection.Collection)

            with mock.patch('recore.mongo.lookup_state') as (
                    mongo.lookup_state):
                # Didn't find the state document in MongoDB
                mongo.lookup_state.return_value = None

                with self.assertRaises(LookupError):
                    f._setup()

    def test_update_missing_state(self):
        """We notice if no document was found to update"""
        f = FSM(state_id)
        f.state_coll = mock.MagicMock(spec=pymongo.collection.Collection,
                                      return_value=True)

        f.state_coll.update.return_value = None

        _update_state = {
            '$set': {
                'ended': UTCNOW
            }
        }

        with self.assertRaises(Exception):
            f.update_state(_update_state)

    def test_dequeue_next_active_step(self):
        """The FSM can remove the next step and update Mongo with it"""
        f = FSM(state_id)
        f.remaining = ["Step 1", "Step 2"]

        _update_state = {
            '$set': {
                'active_step': "Step 1",
                'remaining_steps': ["Step 2"]
            }
        }

        with mock.patch.object(f, 'update_state') as (
                us):
            f.dequeue_next_active_step()
            us.assert_called_once_with(_update_state)
            self.assertEqual(f.active, "Step 1")

    def test_update_state_mongo_failed(self):
        """We notice if mongo failed while updating state"""
        f = FSM(state_id)
        f.state_coll = mock.MagicMock(spec=pymongo.collection.Collection,
                                      return_value=True)

        mocked_update = mock.MagicMock(side_effect=pymongo.errors.PyMongoError)
        f.state_coll.update = mocked_update

        _update_state = {
            '$set': {
                'ended': UTCNOW
            }
        }

        with self.assertRaises(pymongo.errors.PyMongoError):
            f.update_state(_update_state)

    def test__setup_amqp_connect_fails(self):
        """_setup raises exception if amqp connection can't be made"""
        f = FSM(state_id)
        f._connect_mq = mock.MagicMock(side_effect=pika.exceptions.AMQPError("Couldn't connect to AMQP"))

        with mock.patch('recore.mongo.database') as (
                mongo.database):
            mongo.database = mock.MagicMock(pymongo.database.Database)
            mongo.database.__getitem__.return_value = mock.MagicMock(pymongo.collection.Collection)

            with mock.patch('recore.mongo.lookup_state') as (
                    mongo.lookup_state):
                # Found the state document in MongoDB
                mongo.lookup_state.return_value = _state

                with self.assertRaises(pika.exceptions.AMQPError):
                    f._setup()

    def test__setup(self):
        """Setup works with an existing state document"""
        f = FSM(state_id)
        # An AMQP connection hasn't been made yet
        f._connect_mq = mock.MagicMock(return_value=(mock.Mock(pika.channel.Channel),
                                      mock.Mock(pika.connection.Connection)))

        with mock.patch('recore.mongo.database') as (
                mongo.database):
            mongo.database = mock.MagicMock(pymongo.database.Database)
            mongo.database.__getitem__.return_value = mock.MagicMock(pymongo.collection.Collection)

            with mock.patch('recore.mongo.lookup_state') as (
                    mongo.lookup_state):
                mongo.lookup_state.return_value = _state

                f._setup()
                assert f.project == _state['project']

    def test__setup_failed_pre_deploy_check(self, send_notification, move_remaining, amqp_conf):
        """Setup fails with an existing state document and a failed pre-deploy check"""
        f = FSM(TEST_PBID, state_id)
        # An AMQP connection hasn't been made yet
        amqp_conf.get.return_value = PRE_DEPLOY_CONF['PRE_DEPLOY_CHECK']
        msg_started = {'status': 'completed', 'data': {'exists': False}}

        consume_iter = [
            (mock.Mock(name="method_mocked"),
             mock.Mock(name="properties_mocked"),
             json.dumps(msg_started))
        ]

        f.conn = mock.Mock(pika.connection.Connection)
        publish = mock.Mock()
        channel = mock.Mock()
        channel.consume.return_value = iter(consume_iter)
        channel.basic_publish = publish
        f.ch = channel

        with mock.patch('recore.mongo.database') as (
                mongo.database):
            mongo.database = mock.MagicMock(pymongo.database.Database)
            mongo.database.__getitem__.return_value = mock.MagicMock(pymongo.collection.Collection)

            with mock.patch('recore.mongo.lookup_state') as (
                    mongo.lookup_state):
                mongo.lookup_state.return_value = _state

                with mock.patch('recore.amqp.MQ_CONF') as mq_conf:
                    mq_conf = MQ_CONF
                    set_field = mock.MagicMock()
                    filter = mock.MagicMock(return_value=set_field)
                    f.filter = filter
                    f._setup()
                    assert f.group == _state['group']

        # No matter where a release fails, 'move_remaining_to_skipped' will be called
        move_remaining.assert_called_once_with()
        # the first run/pre-deploy steps will record the failed state
        assert f.initialized == False

        # The starting phase notification will be sent
        assert send_notification.call_count == 1
        assert send_notification.call_args[0][4] == 'started'

        # After first_run finishes self.failed should be True
        with mock.patch('recore.amqp.MQ_CONF') as mq_conf:
            mq_conf = MQ_CONF
            f._cleanup()
        assert send_notification.call_count == 2
        assert send_notification.call_args[0][4] == 'failed'
        assert f.failed == True

    def test_on_started(self, ended, pdc):
        """Once started, the FSM waits for a response, and then calls on_ended"""
        pdc.return_value = True
        with nested(
                mock.patch('recore.mongo.lookup_state'),
                mock.patch('recore.mongo.database')
            ) as (lookup_state, database):

            lookup_state.return_value = _state.copy()
            # {
            #     'group': 'PROJECT',
            #     'dynamic': {},
            #     'completed_steps': [],
            #     'active_step': 'active_step',
            #     'remaining_steps': [],
            # }
            f = FSM(TEST_PBID, state_id)
            f.reply_queue = temp_queue
            f.group = 'GROUP'

            consume_iter = [
                (mock.Mock(name="method_mocked"),
                 mock.Mock(name="properties_mocked"),
                 json.dumps(msg_started))
            ]

            publish = mock.Mock()
            channel = mock.Mock()
            channel.consume.return_value = iter(consume_iter)
            channel.basic_publish = publish
            f.ch = channel
            f.conn = mock.Mock(pika.connection.Connection)

            set_field = mock.MagicMock()
            filter = mock.MagicMock(return_value=set_field)
            f.filter = filter
            f._setup()
            f.on_started(f.ch, *consume_iter[0])

            f.ch.basic_ack.assert_called_once_with(consume_iter[0][0].delivery_tag)
            f.ch.cancel.assert_called_once_with()
            f.on_ended.assert_called_once_with(f.ch, *consume_iter[0])

    def test_step_notification_started(self, send_notification, on_started, dequeue_step, setup):
        """Per-step notifications work when starting a step

Tests for the case where only one notification transport (irc, email, etc) is defined"""
        f = FSM(TEST_PBID, state_id)

        msg_started = {'status': 'started'}

        consume_iter = [
            (mock.Mock(name="method_mocked"),
             mock.Mock(name="properties_mocked"),
             json.dumps(msg_started))
        ]

        # Pre-test scaffolding. Hard-code some mocked out
        # attributes/variables because we're skipping the usual
        # initialization steps.
        f.conn = mock.Mock(pika.connection.Connection)
        publish = mock.Mock()
        channel = mock.Mock()
        channel.consume.return_value = iter(consume_iter)
        channel.basic_publish = publish
        f.ch = channel
        f.active_sequence = {'hosts': ['localhost']}
        f.group = 'testgroup'
        f.dynamic = {}
        f.active_step = new_notify_step('started')

        # Run the method now. It should terminate when it reaches the
        # end of _run() with a call to a mocked out on_started()
        with mock.patch('recore.amqp.MQ_CONF') as mq_conf:
            mq_conf = MQ_CONF
            set_field = mock.MagicMock()
            filter = mock.MagicMock(return_value=set_field)
            f.filter = filter
            f._run()

        self.assertEqual(send_notification.call_count, 1)
        self.assertEqual(send_notification.call_args[0][1], 'notify.irc')
        self.assertEqual(send_notification.call_args[0][2], state_id)
        self.assertEqual(send_notification.call_args[0][3], ['#achannel'])
        self.assertEqual(send_notification.call_args[0][4], 'started')

    def test_post_deploy_passed(self, send_notification, move_active, run, skipped):
        """Post-deploy action passes"""
        f = FSM(TEST_PBID, state_id)

        msg_completed = {'status': 'started'}

        consume_iter = [
            (mock.Mock(name="method_mocked"),
             mock.Mock(name="properties_mocked"),
             json.dumps(msg_completed))
        ]

        # Pre-test scaffolding. Hard-code some mocked out
        # attributes/variables because we're skipping the usual
        # initialization steps.
        f.conn = mock.Mock(pika.connection.Connection)
        publish = mock.Mock()
        channel = mock.Mock()
        channel.consume.return_value = iter(consume_iter)
        channel.basic_publish = publish
        f.ch = channel
        f.active_sequence = {'hosts': ['localhost']}
        f.group = 'testgroup'
        f.dynamic = {}
        f.active_step = new_notify_step('failed')
        f.post_deploy_action = [
            {
                "NAME": "Update dates",
                "COMMAND": "servicenow",
                "SUBCOMMAND": "updatedates",
                "PARAMETERS": {
                    "foo": "bar"
                }
            }
        ]

        with mock.patch('recore.amqp.MQ_CONF') as mq_conf:
            mq_conf = MQ_CONF
            set_field = mock.MagicMock()
            filter = mock.MagicMock(return_value=set_field)
            f.filter = filter
            self.assertEqual(f._post_deploy_action(), True)

    def test__connect_mq(self, mq_conf, connection, channel, creds):
        """FSM connecting to AMQP sets its reply_queue attribute"""
        mq_conf = {
            'NAME': 'user',
            'PASSWORD': 'pass',
            'SERVER': '127.0.0.1',
            'EXCHANGE': 'foochange'
        }
        creds.return_value = mock.MagicMock(spec=pika.credentials.PlainCredentials, name="mocked creds")
        mocked_conn = mock.MagicMock(spec=pika.connection.Connection, name="mocked connection")
        mocked_channel = mock.MagicMock(spec=pika.channel.Channel, name="mocked channel")
        channel.return_value = mocked_channel
        channel.queue_declare.return_value.method.queue = temp_queue
        mocked_conn.channel.return_value = channel
        connection.return_value = mocked_conn

        f = FSM(state_id)
        (ch, conn) = f._connect_mq()
        self.assertEqual(f.reply_queue, temp_queue, msg="Expected %s for reply_queue, instead got %s" %
                         (temp_queue, f.reply_queue))