def test_receive_bad_op(self):

        class FakeMsg(object):
            def __init__(self):
                self.named = ["ein", "zwei"]
        cvalue = FakeMsg()

        e = RPCResponseEndpointUnit(routing_obj=self, interceptors={})
        ch = self._setup_mock_channel(value=cvalue, op="no_exist")
        e.attach_channel(ch)

        self._do_listen(e)

        assert_called_once_with_header(self, ch.send, {'status_code':400,
                                                       'error_message':'Unknown op name: no_exist',
                                                       'conv-id': sentinel.conv_id,
                                                       'conv-seq': 2,
                                                       'protocol':'',
                                                       'performative': 'failure',
                                                       'language':'ion-r2',
                                                       'encoding':'msgpack',
                                                       'format':'list',
                                                       'receiver': ',',
                                                       'msg-rcvd':ANY,
                                                       'ts': sentinel.ts})

    def test__get_sflow_manager_with_container(self):
        Container.instance = None
        c = Container() # ensure an instance
        e = RPCResponseEndpointUnit(interceptors={})
        self.assertEquals(e._get_sflow_manager(), c.sflow_manager)

        Container.instance = None

    def test_recv_bad_kwarg(self):
        # we try to call simple with the kwarg "not_named" instead of the correct one
        class FakeMsg(object):
            def __init__(self):
                self.not_named = ["ein", "zwei"]
        cvalue = FakeMsg()

        e = RPCResponseEndpointUnit(routing_obj=self, interceptors={})
        ch = self._setup_mock_channel(value=cvalue, op="simple")
        e.attach_channel(ch)

        self._do_listen(e)

        # test to make sure send got called with our error
        assert_called_once_with_header(self, ch.send, {'status_code':400,
                                                       'error_message':'Argument not_named not present in op signature',
                                                       'conv-id': sentinel.conv_id,
                                                       'conv-seq': 2,
                                                       'protocol':'',
                                                       'performative': 'failure',
                                                       'language':'ion-r2',
                                                       'encoding':'msgpack',
                                                       'format':'NoneType',
                                                       'receiver': ',',
                                                       'msg-rcvd':ANY,
                                                       'ts': sentinel.ts})

    def test_receive_bad_op(self):
        class FakeMsg(object):
            def __init__(self):
                self.named = ["ein", "zwei"]

        cvalue = FakeMsg()

        e = RPCResponseEndpointUnit(routing_obj=self)
        ch = self._setup_mock_channel(value=cvalue, op="no_exist")
        e.attach_channel(ch)

        e.spawn_listener()
        e._recv_greenlet.join()

        # test to make sure send got called with our error
        ch.send.assert_called_once_with(
            None,
            {
                "status_code": 400,
                "error_message": "Unknown op name: no_exist",
                "conv-id": "",
                "conv-seq": 2,
                "protocol": "",
                "performative": "failure",
                "language": "ion-r2",
                "encoding": "msgpack",
                "format": "NoneType",
                "receiver": ",",
                "ts": sentinel.ts,
                "reply-by": "todo",
            },
        )

    def test_recv_bad_kwarg(self):
        # we try to call simple with the kwarg "not_named" instead of the correct one
        class FakeMsg(object):
            def __init__(self):
                self.not_named = ["ein", "zwei"]

        cvalue = FakeMsg()

        e = RPCResponseEndpointUnit(routing_obj=self)
        ch = self._setup_mock_channel(value=cvalue, op="simple")
        e.attach_channel(ch)

        e.spawn_listener()
        e._recv_greenlet.join()

        # test to make sure send got called with our error
        ch.send.assert_called_once_with(
            None,
            {
                "status_code": 500,
                "error_message": "simple() got an unexpected keyword argument 'not_named'",
                "conv-id": "",
                "conv-seq": 2,
                "protocol": "",
                "performative": "failure",
                "language": "ion-r2",
                "encoding": "msgpack",
                "format": "NoneType",
                "receiver": ",",
                "ts": sentinel.ts,
                "reply-by": "todo",
            },
        )

    def test_recv_bad_kwarg(self):
        # we try to call simple with the kwarg "not_named" instead of the correct one
        class FakeMsg(object):
            def __init__(self):
                self.not_named = ["ein", "zwei"]
        cvalue = FakeMsg()

        e = RPCResponseEndpointUnit(routing_obj=self)
        ch = self._setup_mock_channel(value=cvalue, op="simple")
        e.attach_channel(ch)

        e.spawn_listener()
        e._recv_greenlet.join()

        # test to make sure send got called with our error
        ch.send.assert_called_once_with(None, {'status_code':500,
                                               'error_message':'simple() got an unexpected keyword argument \'not_named\'',
                                               'conv-id': '',
                                               'conv-seq': 2,
                                               'protocol':'',
                                               'performative': 'failure',
                                               'language':'ion-r2',
                                               'encoding':'msgpack',
                                               'format':'NoneType',
                                               'receiver': ',',
                                               'reply-by': 'todo'})

    def test_receive_bad_op(self):

        class FakeMsg(object):
            def __init__(self):
                self.named = ["ein", "zwei"]
        cvalue = FakeMsg()

        e = RPCResponseEndpointUnit(routing_obj=self)
        ch = self._setup_mock_channel(value=cvalue, op="no_exist")
        e.attach_channel(ch)

        e.spawn_listener()
        e._recv_greenlet.join()

        # test to make sure send got called with our error
        ch.send.assert_called_once_with(None, {'status_code':400,
                                               'error_message':'Unknown op name: no_exist',
                                               'conv-id': '',
                                               'conv-seq': 2,
                                               'protocol':'',
                                               'performative': 'failure',
                                               'language':'ion-r2',
                                               'encoding':'msgpack',
                                               'format':'NoneType',
                                               'receiver': ',',
                                               'reply-by': 'todo'})

    def test_message_received_dict(self):
        rout_obj = Mock()
        e = RPCResponseEndpointUnit(routing_obj=rout_obj)

        msg_dict = {"iam": "adict"}
        e.message_received(msg_dict, {"op": "anyop"})

        rout_obj.anyop.assert_called_once_with(iam="adict")

    def test_message_received_dict(self):
        rout_obj = Mock()
        e = RPCResponseEndpointUnit(routing_obj=rout_obj)

        msg_dict = {'iam':'adict'}
        e.message_received(msg_dict, {'op':'anyop'})

        rout_obj.anyop.assert_called_once_with(iam='adict')

    def test__sample_request_no_sample(self, mocklog):
        e = RPCResponseEndpointUnit(interceptors={})

        e._get_sflow_manager = Mock(return_value=Mock(spec=SFlowManager))
        e._get_sflow_manager.return_value.should_sample = False
        e._get_sample_name = Mock()

        e._sample_request(sentinel.status, sentinel.status_descr, sentinel.msg, sentinel.headers, sentinel.response, sentinel.response_headers)

        self.assertEquals(mocklog.debug.call_count, 1)
        self.assertIn("not to sample", mocklog.debug.call_args[0][0])

    def test__sample_request_exception(self, mocklog):

        e = RPCResponseEndpointUnit(interceptors={})

        e._get_sflow_manager = Mock(return_value=Mock(spec=SFlowManager))
        e._get_sflow_manager.return_value.should_sample = True
        e._build_sample = Mock(side_effect=TestError)

        e._sample_request(sentinel.status, sentinel.status_descr, sentinel.msg, sentinel.headers, sentinel.response, sentinel.response_headers)

        mocklog.exception.assert_called_once_with("Could not sample, ignoring")

    def test__message_received_interceptor_exception(self):
        e = RPCResponseEndpointUnit(routing_obj=self)
        e.send = Mock()
        e.send.return_value = sentinel.sent
        with patch('pyon.net.endpoint.ResponseEndpointUnit._message_received', new=Mock(side_effect=exception.IonException)):
            retval = e._message_received(sentinel.msg, {})

            self.assertEquals(retval, sentinel.sent)
            e.send.assert_called_once_with(None, {'status_code': -1,
                                                  'error_message':'',
                                                  'conv-id': '',
                                                  'conv-seq': 2,
                                                  'protocol':''})

    def test_endpoint_receive(self):
        self._ar = event.AsyncResult()

        # build a command object to be returned by the mocked channel
        class FakeMsg(object):
            def __init__(self):
                self.named = ["ein", "zwei"]
        cvalue = FakeMsg()

        e = RPCResponseEndpointUnit(routing_obj=self, interceptors={})
        ch = self._setup_mock_channel(value=cvalue, op="simple")
        e.attach_channel(ch)

        self._do_listen(e)
        args = self._ar.get(timeout=10)

        self.assertEquals(args, ["ein", "zwei"])

    def _build_header(self, raw_msg, raw_headers):
        """
        Override to direct the calls in _build_header - first the RPCResponse side, then the Process mixin.
        """
        header1 = RPCResponseEndpointUnit._build_header(self, raw_msg, raw_headers)
        header2 = ProcessEndpointUnitMixin._build_header(self, raw_msg, raw_headers)

        header1.update(header2)

        return header1

    def test__message_received_interceptor_exception(self):
        e = RPCResponseEndpointUnit(routing_obj=self)
        e.send = Mock()
        e.send.return_value = sentinel.sent
        with patch(
            "pyon.net.endpoint.ResponseEndpointUnit._message_received", new=Mock(side_effect=exception.IonException)
        ):
            retval = e._message_received(sentinel.msg, {})

            self.assertEquals(retval, sentinel.sent)
            e.send.assert_called_once_with(
                None,
                {
                    "status_code": -1,
                    "error_message": "",
                    "conv-id": "",
                    "conv-seq": 2,
                    "protocol": "",
                    "performative": "failure",
                },
            )

    def test__message_received_error_in_op(self):
        # we want to make sure IonExceptions raised in business logic get a response, now that
        # _message_received sends the responses

        class FakeMsg(object):
            pass

        cvalue = FakeMsg()

        e = RPCResponseEndpointUnit(routing_obj=self)
        ch = self._setup_mock_channel(value=cvalue, op="error_op")
        e.attach_channel(ch)

        e.send = Mock()

        e.spawn_listener()
        e._recv_greenlet.join()

        e.send.assert_called_once_with(
            None,
            {
                "status_code": 401,
                "error_message": str(sentinel.unauth),
                "conv-id": "",
                "conv-seq": 2,
                "protocol": "",
                "performative": "failure",
            },
        )

    def test__message_received_error_in_op(self):
        # we want to make sure IonExceptions raised in business logic get a response, now that
        # _message_received sends the responses

        class FakeMsg(object):
            pass
        cvalue = FakeMsg()

        e = RPCResponseEndpointUnit(routing_obj=self, interceptors={})
        ch = self._setup_mock_channel(value=cvalue, op="error_op")
        e.attach_channel(ch)

        e.send = Mock()

        self._do_listen(e)

        assert_called_once_with_header(self, e.send, {'status_code': 401,
                                                      'error_message': str(sentinel.unauth),
                                                      'conv-id': sentinel.conv_id,
                                                      'conv-seq': 2,
                                                      'protocol':'',
                                                      'performative':'failure'})

    def test__build_sample(self):
        e = RPCResponseEndpointUnit(interceptors={})

        heads = {'conv-id': sentinel.conv_id,
                 'ts': '1',
                 'op': 'remove_femur',
                 'sender': 'sender',
                 'receiver': 'getter'}
        resp_heads = {'sender-service': 'theservice'}

        samp = e._build_sample("app_name", 200, "Ok", "msg", heads, "response", resp_heads, sentinel.qlen)

        self.assertEquals(samp, {
            'app_name' : "app_name",
            'op' : 'remove_femur',
            'attrs' : {'ql':sentinel.qlen},
            'status_descr' : "Ok",
            'status' : '0',
            'req_bytes' : len('msg'),
            'resp_bytes': len('response'),
            'uS' : 999000, # it's in microseconds!
            'initiator' : 'sender',
            'target' : 'theservice'
        })

    def _message_received(self, msg, headers):
        """
        Message received override.

        Sets the process' context here to be picked up by subsequent calls out by this service to other services, or replies.
        """
        ######
        ###### THIS IS WHERE THE THREAD LOCAL HEADERS CONTEXT IS SET ######
        ######

        # With the property _routing_call set, as is the case 95% of the time in the Process-level endpoints,
        # we have to set the call context from the ION process' calling greenlet, as context is greenlet-specific.
        # This is done in the _make_routing_call override here, passing it the context to be set.
        # See also IonProcessThread._control_flow.

        with self._process.push_context(headers):
            return RPCResponseEndpointUnit._message_received(self, msg, headers)

    def message_received(self, msg, headers):
        #This is the hook for checking governance pre-conditions before calling a service operation
        #TODO - replace with a process specific interceptor stack of some sort.
        gc = self._routing_obj.container.governance_controller
        if gc:
            gc.check_process_operation_preconditions(self._routing_obj, msg, headers)

        result, response_headers = RPCResponseEndpointUnit.message_received(self, msg, headers)

        # decorate our response_headers with process-saturation, as we need them to be set in the headers
        # earlier than send/build_header so the sampling takes notice
        try:
            response_headers['process-saturation'] = self._get_process_saturation()
        except Exception as ex:
            log.warn("Could not set process-saturation header, ignoring: %s", ex)

        return result, response_headers