def _create_service_mock(self, service_name):
     # set self.clients if not already set
     clients = Mock(name='clients')
     base_service = get_service_registry().get_service_base(service_name)
     # Save it to use in test_verify_service
     self.base_service = base_service
     self.addCleanup(delattr, self, 'base_service')
     dependencies = base_service.dependencies
     for dep_name in dependencies:
         dep_service = get_service_registry().get_service_base(dep_name)
         # Force mock service to use interface
         mock_service = Mock(name='clients.%s' % dep_name,
                 spec=dep_service)
         setattr(clients, dep_name, mock_service)
         # set self.dep_name for convenience
         setattr(self, dep_name, mock_service)
         self.addCleanup(delattr, self, dep_name)
         iface = list(implementedBy(dep_service))[0]
         names_and_methods = iface.namesAndDescriptions()
         for func_name, _ in names_and_methods:
             mock_func = mocksignature(getattr(dep_service, func_name),
                     mock=Mock(name='clients.%s.%s' % (dep_name,
                         func_name)), skipfirst=True)
             setattr(mock_service, func_name, mock_func)
     return clients

def svc_defs(svcs=None, op=None):
    """Returns service definitions for service name(s)
    @param svcs name or list of names of service
    """
    from pyon.core.bootstrap import get_service_registry

    if not getattr(svcs, '__iter__', False) and op is not None:
        svcdef = get_service_registry().services[svcs]
        print "Service definition for: %s (version %s) operation %s" % (svcs, svcdef.version or 'ND', op)
        print "".join([str(o) for o in svcdef.operations if o.name == op])
        return svcdef

    elif svcs is not None:
        if not getattr(svcs, '__iter__', False):
            svcs = (svcs,)
        for svcname in svcs:
            svcdef = get_service_registry().services[svcname]
            svcops = "\n     ".join(sorted([o.name for o in svcdef.operations]))
            print "Service definition for: %s (version %s)" % (svcname, svcdef.version or 'ND')
            print "ops: %s" % (svcops)
            return svcdef

    else:
        print "List of defined services"
        print "------------------------"

        for svcname in sorted(get_service_registry().services.keys()):
            svcdef = get_service_registry().services[svcname]
            print "%s %s" % (svcname, svcdef.version)

        print "\nType svc_defs('name') or svc_defs(['name1','name2']) for definition"
        return None

    def on_start(self):
        logger = self.CFG.get("logger")
        level = self.CFG.get("level")
        recursive = self.CFG.get("recursive", False)

        svc = get_service_registry().services['system_management'].client(process=self)
        svc.set_log_level(logger=logger, level=level, recursive=recursive)

    def on_start(self):
        rr = self.container.resource_registry

        name = self.CFG.get("dataset")
        if name: # We are looking for an external dataset!
            log.trace("Looking for an external dataset with name %s", name)
            objects,_ = rr.find_resources(RT.ExternalDataset)
            filtered_objs = [obj for obj in objects if obj.name == name]
            if (filtered_objs) == []:
                raise ConfigNotFound("No appropriate ExternalDataset objects loaded")
            external_dataset = filtered_objs[0]
    
            instance_id = rr.read_object(subject=external_dataset._id,
                                         predicate=PRED.hasAgentInstance,
                                         object_type=RT.ExternalDatasetAgentInstance,
                                         id_only=True)
            dams = get_service_registry().services['data_acquisition_management'].client(process=self)
            dams.start_external_dataset_agent_instance(instance_id)
            client = ResourceAgentClient(external_dataset._id, process=FakeProcess())
        else:
            name = self.CFG.get("instrument")
            
        if name: # We are looking for an instrument device!
            log.trace("Looking for an instrument device with name %s", name)
            objects,_ = rr.find_resources(RT.InstrumentDevice)
            filtered_objs = [obj for obj in objects if obj.name == name]
            if (filtered_objs) == []:
                raise ConfigNotFound("No appropriate InstrumentDevice objects loaded")
            instrument_device = filtered_objs[0]
            log.trace("Found instrument device: %s", instrument_device)
    
            instance_id = rr.read_object(subject=instrument_device._id,
                                         predicate=PRED.hasAgentInstance,
                                         object_type=RT.ExternalDatasetAgentInstance,
                                         id_only=True)
            dams = get_service_registry().services['data_acquisition_management'].client(process=self)
            proc_id = dams.start_external_dataset_agent_instance(instance_id)
            client = ResourceAgentClient(instrument_device._id, process=FakeProcess())
        else:
            raise ConfigNotFound("Could not find dataset or instrument_device value name")

        log.info('Activating agent...')
        client.execute_agent(AgentCommand(command=ResourceAgentEvent.INITIALIZE))
        client.execute_agent(AgentCommand(command=ResourceAgentEvent.GO_ACTIVE))
        client.execute_agent(AgentCommand(command=ResourceAgentEvent.RUN))
        client.execute_resource(command=AgentCommand(command=DriverEvent.START_AUTOSAMPLE))
        log.info('Now reading: %s', name)

    def _start_service(self, servicename, servicecls=None, config=None):
        if servicename and not servicecls:
            global scanned_services
            if not scanned_services:
                get_service_registry().discover_service_classes()
                scanned_services = True
            assert servicename in get_service_registry().services, "Service %s unknown" % servicename
            servicecls = get_service_registry().services[servicename].impl[0]

        assert servicecls, "Cannot start service %s" % servicename

        if type(servicecls) is str:
            mod, cls = servicecls.rsplit('.', 1)
        else:
            mod = servicecls.__module__
            cls = servicecls.__name__
        self.container.spawn_process(servicename, mod, cls, config)

 def _register_service_definitions(self):
     from pyon.core.bootstrap import get_service_registry
     svc_list = []
     for svcname, svc in get_service_registry().services.iteritems():
         svc_def = ServiceDefinition(name=svcname, definition="")
         svc_list.append(svc_def)
     self._create_mult(svc_list)
     log.info("Created %d ServiceDefinition resources" % len(svc_list))

 def _is_policy_management_service_available(self):
     """
     Method to verify if the Policy Management Service is running in the system. If the container cannot connect to
     the RR then assume it is remote container so do not try to access Policy Management Service
     """
     policy_service = get_service_registry().is_service_available('policy_management', True)
     if policy_service:
         return True
     return False

    def _ems_available(self):
        """
        Returns True if the EMS is (likely) available and the auto_register CFG entry is True.

        @TODO: make a property
        """
        if CFG.get_safe('container.exchange.auto_register', False) and self.use_ems:
            # ok now make sure it's in the directory
            exchange_service = get_service_registry().is_service_available('exchange_management')
            if exchange_service:
                return True

        return False

    def _call_target(self, target, value=None, resource_id=None, res_type=None, cmd_args=None, cmd_kwargs=None):
        """
        Makes a call to a specified function. Function specification can be of varying type.
        """
        try:
            if not target:
                return None
            match = re.match("(func|serviceop):([\w.]+)\s*\(\s*([\w,$\s]*)\s*\)\s*(?:->\s*([\w\.]+))?\s*$", target)
            if match:
                func_type, func_name, func_args, res_path = match.groups()
                func = None
                if func_type == "func":
                    if func_name.startswith("self."):
                        func = getattr(self, func_name[5:])
                    else:
                        func = named_any(func_name)
                elif func_type == "serviceop":
                    svc_name, svc_op = func_name.split('.', 1)
                    try:
                        svc_client_cls = get_service_registry().get_service_by_name(svc_name).client
                    except Exception as ex:
                        log.error("No service client found for service: %s", svc_name)
                    else:
                        svc_client = svc_client_cls(process=self)
                        func = getattr(svc_client, svc_op)

                if not func:
                    return None

                args = self._get_call_args(func_args, resource_id, res_type, value, cmd_args)
                kwargs = {} if not cmd_kwargs else cmd_kwargs

                func_res = func(*args, **kwargs)
                log.info("Function %s result: %s", func, func_res)

                if res_path and isinstance(func_res, dict):
                    func_res = get_safe(func_res, res_path, None)

                return func_res

            else:
                log.error("Unknown call target expression: %s", target)

        except Unauthorized as ex:
            # No need to log as this is not an application error, however, need to pass on the exception because
            # when called by the Service Gateway, the error message in the exception is required
            raise ex

        except Exception as ex:
            log.exception("_call_target exception")
            raise ex  #Should to pass it back because when called by the Service Gateway, the error message in the exception is required

    def _call_target(self, target, value=None, resource_id=None, res_type=None, cmd_args=None, cmd_kwargs=None):
        """
        Makes a call to a specified function. Function specification can be of varying type.
        """
        try:
            if not target:
                return None
            match = re.match("(func|serviceop):([\w.]+)\s*\(\s*([\w,$\s]*)\s*\)\s*(?:->\s*([\w\.]+))?\s*$", target)
            if match:
                func_type, func_name, func_args, res_path = match.groups()
                func = None
                if func_type == "func":
                    if func_name.startswith("self."):
                        func = getattr(self, func_name[5:])
                    else:
                        func = named_any(func_name)
                elif func_type == "serviceop":
                    svc_name, svc_op = func_name.split(".", 1)
                    try:
                        svc_client_cls = get_service_registry().get_service_by_name(svc_name).client
                    except Exception as ex:
                        log.error("No service client found for service: %s", svc_name)
                    else:
                        svc_client = svc_client_cls(process=self)
                        func = getattr(svc_client, svc_op)

                if not func:
                    return None

                args = self._get_call_args(func_args, resource_id, res_type, value, cmd_args)
                kwargs = {} if not cmd_kwargs else cmd_kwargs

                func_res = func(*args, **kwargs)
                log.info("Function %s result: %s", func, func_res)

                if res_path and isinstance(func_res, dict):
                    func_res = get_safe(func_res, res_path, None)

                return func_res

            else:
                log.error("Unknown call target expression: %s", target)

        except Exception as ex:
            log.exception("_call_target exception")
            return None

def get_remote_info(execution_object, method_name):
    """
    Returns the service client and operation name
    """
    # This is a remote method.
    rmi_call = method_name.split('.')
    # Retrieve service definition
    service_name = rmi_call[0]
    operation = rmi_call[1]
    if service_name == 'resource_registry':
        service_client = execution_object._rr
    else:
        target_service = get_service_registry().get_service_by_name(service_name)
        service_client = target_service.client(node=execution_object.service_provider.container.instance.node,
                                               process=execution_object.service_provider)

    return service_client, operation

    def on_start(self):
        name = self.CFG.get("dataset")

        rr = self.container.resource_registry

        objects,_ = rr.find_resources(RT.ExternalDataset)
        filtered_objs = [obj for obj in objects if obj.name == name]
        external_dataset = filtered_objs[0]

        instance_id = rr.read_object(subject=external_dataset._id, predicate=PRED.hasAgentInstance, object_type=RT.ExternalDatasetAgentInstance, id_only=True)
        dams = get_service_registry().services['data_acquisition_management'].client(process=self)
        dams.start_external_dataset_agent_instance(instance_id)

        # OLD METHOD of starting agent, now this code has been implemented and tested in DAMS
        #obj,_ = rr.find_objects(subject=external_dataset._id, predicate=PRED.hasAgentInstance, object_type=RT.ExternalDatasetAgentInstance)
        #agent_instance = obj[0]
        #obj,_ = rr.find_objects(object_type=RT.ExternalDatasetAgent, predicate=PRED.hasAgentDefinition, subject=agent_instance._id)
        #agent = obj[0]
        #stream_definition_id = agent_instance.dataset_driver_config['dh_cfg']['stream_def']
        #stream_definition = rr.read(stream_definition_id)
        #data_producer_id = agent_instance.dataset_driver_config['dh_cfg']['data_producer_id']
        #data_producer = rr.read(data_producer_id) #subject="", predicate="", object_type="", assoc="", id_only=False)
        #data_product = rr.read_object(object_type=RT.DataProduct, predicate=PRED.hasOutputProduct, subject=external_dataset._id)
        #ids,_ = rr.find_objects(data_product._id, PRED.hasStream, RT.Stream, id_only=True)
        #stream_id = ids[0]
        #route = pubsub.read_stream_route(stream_id)
        #agent_instance.dataset_agent_config['driver_config']['dh_cfg']['stream_id'] = stream_id
        #agent_instance.dataset_agent_config['driver_config']['stream_id'] = stream_id
        #agent_instance.dataset_agent_config['driver_config']['dh_cfg']['stream_route'] = route
        #
        #log.info('launching agent process: %s.%s', agent.handler_module, agent.handler_class)
        #self.container.spawn_process(name=agent_instance.name,
        #    module=agent.handler_module, cls=agent.handler_class,
        #    config=agent_instance.dataset_agent_config)
        #
        # should i wait for process (above) to start
        # before launching client (below)?
        #
        log.info('activating agent...')
        client = ResourceAgentClient(external_dataset._id, process=FakeProcess())
        client.execute_agent(AgentCommand(command=ResourceAgentEvent.INITIALIZE))
        client.execute_agent(AgentCommand(command=ResourceAgentEvent.GO_ACTIVE))
        client.execute_agent(AgentCommand(command=ResourceAgentEvent.RUN))
        client.execute_resource(command=AgentCommand(command=DriverEvent.START_AUTOSAMPLE))
        log.info('now reading external dataset: %s', name)

    def _gen_paths(self, service_name):
        paths = {}
        sr = get_service_registry()
        if service_name == ALL_SERVICES:
            for svc_name in sorted(sr.services_by_name):
                if svc_name in self.config.get("exclude_services", []):
                    continue
                svc_schema = sr.services[svc_name].schema
                # For operations
                for op_name in svc_schema["op_list"]:
                    self._add_service_op_entries(paths, svc_name, op_name, svc_schema)
        else:
            svc_def = sr.services.get(service_name, None)
            if svc_def:
                svc_schema = svc_def.schema
                for op_name in svc_schema["op_list"]:
                    self._add_service_op_entries(paths, service_name, op_name, svc_schema)

        return paths

    def _make_service_request(self, service_name=None, operation=None, id_param=None):
        """
        Executes a secure call to a SciON service operation via messaging.
        """
        if not service_name:
            if self.develop_mode:
                # Return a list of available services
                result = dict(available_services=get_service_registry().services.keys())
                return result
            else:
                raise BadRequest("Service name missing")
        service_name = str(service_name)

        if not operation:
            if self.develop_mode:
                # Return a list of available operations
                result = dict(available_operations=[])
                return result
            else:
                raise BadRequest("Service operation missing")
        operation = str(operation)

        # Apply service white list and black list for initial protection and get service client
        service_def = self.get_secure_service_def(service_name)
        target_client = service_def.client

        # Get service request arguments and operation parameter values request
        req_args = self._get_request_args()

        param_list = self.create_parameter_list(service_def, operation, req_args, id_param)

        # Validate requesting user and expiry and add governance headers
        ion_actor_id, expiry = self.get_governance_info_from_request(req_args)
        in_login_whitelist = self.in_login_whitelist("request", service_name, operation)
        ion_actor_id, expiry = self.validate_request(ion_actor_id, expiry, in_whitelist=in_login_whitelist)
        param_list["headers"] = self.build_message_headers(ion_actor_id, expiry)

        # Make service operation call
        client = target_client(process=self.process)
        method_call = getattr(client, operation)
        result = method_call(**param_list)

        return result

    def execute_method(self, resource_id, method_name, **kwargs):

        try:

            #First look to see if this is a remote method
            if method_name.find('.') > 0:

                #This is a remote method.
                rmi_call = method_name.split('.')
                #Retrieve service definition
                service_name = rmi_call[0]
                operation = rmi_call[1]
                if service_name == 'resource_registry':
                    service_client = self._rr
                else:
                    target_service = get_service_registry().get_service_by_name(service_name)
                    service_client = target_service.client(node=self.service_provider.container.instance.node, process=self.service_provider)

                methodToCall = getattr(service_client, operation)
                param_list = [resource_id]
                param_dict = self._get_method_arguments(service_client, operation, **kwargs)
                ret = methodToCall(*param_list, **param_dict )
                return ret

            else:
                #For local methods, first look for the method in the current class
                func = getattr(self, method_name, None)
                if func:
                    param_dict = self._get_method_arguments(self,method_name, **kwargs)
                    return func(resource_id, **param_dict)
                else:
                    #Next look to see if the method exists in the service provider process
                    func = getattr(self.service_provider, method_name, None)
                    if func:
                        param_dict = self._get_method_arguments(self.service_provider,method_name, **kwargs)
                        return func(resource_id, **param_dict)

                return None

        except Exception, e:
            log.error('Error executing method %s for resource id %s: %s' % (method_name, resource_id, str(e)))
            return None

    def get_secure_service_def(self, service_name):
        """Checks whether the service indicated by given service_name exists and/or
        is exposed after white and black listing. Returns service registry entry.
        """
        if self.service_whitelist:
            if service_name not in self.service_whitelist:
                raise Unauthorized("Service access not permitted")
        if self.service_blacklist:
            if service_name in self.service_blacklist:
                raise Unauthorized("Service access not permitted")

        # Retrieve service definition
        target_service = get_service_registry().get_service_by_name(service_name)
        if not target_service:
            raise BadRequest("The requested service (%s) is not available" % service_name)
        if not target_service.client:
            raise Inconsistent("Cannot find a client class for the specified service: %s" % service_name)
        if not target_service.schema:
            raise Inconsistent("Cannot find a schema for the specified service: %s" % service_name)

        return target_service

    def _gen_tags(self, service_name):
        tags = []
        sr = get_service_registry()
        if service_name == ALL_SERVICES:
            for svc_name in sorted(sr.services_by_name):
                if svc_name in self.config.get("exclude_services", []):
                    continue
                svc_schema = sr.services[svc_name].schema
                tag_entry = dict(name=svc_name, description=svc_schema["description"])
                if self.config.get("externalDocs", {}):
                    tag_entry["externalDocs"] = self.config["externalDocs"]
                tags.append(tag_entry)
        else:
            svc_def = sr.services.get(service_name, None)
            if svc_def:
                svc_schema = svc_def.schema
                tag_entry = dict(name=service_name, description=svc_schema["description"])
                if self.config.get("externalDocs", {}):
                    tag_entry["externalDocs"] = self.config["externalDocs"]
                tags.append(tag_entry)

        return tags

 def _get_service_queue(self, request):
     """
     """
     id = request.resource_id
     svc_name = request.svc_name
     svc_queue = None
     if id:
         svc_queue = self._service_command_queues.get(id, None)
         if not svc_queue:
             if id == 'fake_id':
                 svc_queue = ServiceCommandQueue(id, None, self._result_complete)
                 svc_queue.start()
                 self._service_command_queues[id] = svc_queue
                 
             else:
                 try:
                     res_client = ResourceAgentClient(id, process=self)
                 except NotFound:
                     res_client = None
                     
                 else:
                     # Create and return a new queue with this client.
                     svc_queue = ServiceCommandQueue(id, res_client, self._result_complete)
                     svc_queue.start()
                     self._service_command_queues[id] = svc_queue
         
     elif svc_name:
         svc_queue = self._service_command_queues.get(svc_name, None)
         if not svc_queue:
             svc_cls = get_service_registry().get_service_by_name(svc_name)
             if svc_cls:
                 svc_client_cls = svc_cls.client
                 svc_client = svc_client_cls(process=self)
                 svc_queue = ServiceCommandQueue(svc_name, svc_client, self._result_complete)
                 svc_queue.start()
                 self._service_command_queues[svc_name] = svc_queue
                 
     return svc_queue

    def _start_dependencies(self):
        """
        Starts the services declared in the class or instance variable "service_dependencies"
        """
        self.clients = DotDict()
        svc_deps = getattr(self, "service_dependencies", {})
        log.debug("Starting service dependencies. Number=%s" % len(svc_deps))
        if not svc_deps:
            return
        for svc in svc_deps:
            config = None
            if type(svc) in (tuple, list):
                config = svc[1]
                svc = svc[0]

            # Start the service
            self._start_service(svc, config=config)

            # Create a client
            clcls = get_service_registry().services[svc].simple_client
            self.clients[svc] = clcls(name=svc, node=self.container.node)

        log.debug("Service dependencies started")

def process_gateway_request(service_name, operation):


    try:

        if not service_name:
            raise BadRequest("Target service name not found in the URL")

        #Retrieve service definition
        from pyon.core.bootstrap import get_service_registry
        # MM: Note: service_registry can do more now
        target_service = get_service_registry().get_service_by_name(service_name)

        if not target_service:
            raise BadRequest("The requested service (%s) is not available" % service_name)

        if operation == '':
            raise BadRequest("Service operation not specified in the URL")


        #Find the concrete client class for making the RPC calls.
        if not target_service.client:
            raise Inconsistent("Cannot find a client class for the specified service: %s" % service_name )

        target_client = target_service.client

        #Retrieve json data from HTTP Post payload
        json_params = None
        if request.method == "POST":
            payload = request.form['payload']
            #debug only
            #payload = '{"serviceRequest": { "serviceName": "resource_registry", "serviceOp": "find_resources", "params": { "restype": "BankAccount", "lcstate": "", "name": "", "id_only": false } } }'
            json_params = json.loads(payload)

            if not json_params.has_key('serviceRequest'):
                raise Inconsistent("The JSON request is missing the 'serviceRequest' key in the request")

            if not json_params['serviceRequest'].has_key('serviceName'):
                raise Inconsistent("The JSON request is missing the 'serviceName' key in the request")

            if not json_params['serviceRequest'].has_key('serviceOp'):
                raise Inconsistent("The JSON request is missing the 'serviceOp' key in the request")

            if json_params['serviceRequest']['serviceName'] != target_service.name:
                raise Inconsistent("Target service name in the JSON request (%s) does not match service name in URL (%s)" % (str(json_params['serviceRequest']['serviceName']), target_service.name ) )

            if json_params['serviceRequest']['serviceOp'] != operation:
                raise Inconsistent("Target service operation in the JSON request (%s) does not match service name in URL (%s)" % ( str(json_params['serviceRequest']['serviceOp']), operation ) )

        param_list = create_parameter_list('serviceRequest', service_name, target_client,operation, json_params)


        #Validate requesting user and expiry and add governance headers
        ion_actor_id, expiry = get_governance_info_from_request('serviceRequest', json_params)
        ion_actor_id, expiry = validate_request(ion_actor_id, expiry)
        param_list['headers'] = build_message_headers(ion_actor_id, expiry)

        client = target_client(node=Container.instance.node, process=service_gateway_instance)
        methodToCall = getattr(client, operation)
        result = methodToCall(**param_list)

        return gateway_json_response(result)


    except Exception, e:
        return build_error_response(e)