def build_data_dict(self, rdt):
        np_dict = {}
        
        time_array = rdt[rdt.temporal_parameter]
        if time_array is None:
            raise ValueError("A granule needs a time array")
        for k,v in rdt.iteritems():
            # Sparse values are different and aren't constructed using NumpyParameterData
            if isinstance(rdt.param_type(k), SparseConstantType):
                value = v[0]
                if hasattr(value, 'dtype'):
                    value = np.asscalar(value)
                time_start = np.asscalar(time_array[0])
                np_dict[k] = ConstantOverTime(k, value, time_start=time_start, time_end=None) # From now on
                continue
            elif isinstance(rdt.param_type(k), CategoryType):
                log.warning("Category types temporarily unsupported")
                continue
            elif isinstance(rdt.param_type(k), RecordType):
                value = v
            else:
                value = v

            try:
                if k == 'temp_sample':
                    print repr(value)
                np_dict[k] = NumpyParameterData(k, value, time_array)
            except:
                raise

        return np_dict

 def _req_callback(self, result):
     """
     Terrestrial server callback for result receipts.
     Pop pending command, append result and publish.
     """
     log.debug('Terrestrial server got result: %s', str(result))
     
     try:
         id = result['command_id']
         _result = result['result']
         cmd = self._tx_dict.pop(id)
         cmd.time_completed = time.time()
         cmd.result = _result
         if cmd.resource_id:
             origin = cmd.resource_id
         elif cmd.svc_name:
             origin = cmd.svc_name + self._xs_name
         else:
             raise KeyError
         
         self._publisher.publish_event(
                                 event_type='RemoteCommandResult',
                                 command=cmd,
                                 origin=origin)
         log.debug('Published remote result: %s to origin %s.', str(result),
                   str(origin))
     except KeyError:
         log.warning('Error publishing remote result: %s.', str(result))

    def _register_service(self):

        definition = self.process_definition
        existing_services, _ = self.container.resource_registry.find_resources(
            restype="Service", name=definition.name)

        if len(existing_services) > 0:
            if len(existing_services) > 1:
                log.warning("There is more than one service object for %s. Using the first one" % definition.name)
            service_id = existing_services[0]._id
        else:
            svc_obj = Service(name=definition.name, exchange_name=definition.name)
            service_id, _ = self.container.resource_registry.create(svc_obj)

        svcdefs, _ = self.container.resource_registry.find_resources(
            restype="ServiceDefinition", name=definition.name)

        if svcdefs:
            try:
                self.container.resource_registry.create_association(
                    service_id, "hasServiceDefinition", svcdefs[0]._id)
            except BadRequest:
                log.warn("Failed to associate %s Service and ServiceDefinition. It probably exists.",
                    definition.name)
        else:
            log.error("Cannot find ServiceDefinition resource for %s",
                    definition.name)

        return service_id, definition.name

 def acquire_samples(self, max_samples=0):
     log.debug('Orb_DataAgentPlugin.acquire_samples')
     if os.path.exists(self.data_dir):
         files = os.listdir(self.data_dir)
         cols = []
         rows = []
         for f in files:
             fpath = self.data_dir + f
             with open(fpath) as fh:
                 try:
                     pkt = json.load(fh)
                     if not cols:
                         cols = [str(c['chan']) for c in pkt['channels']]              
                     row = self._extract_row(pkt, cols)
                     dims = [len(c) for c in row[:3]]
                     if all(d==400 for d in dims):
                         rows.append(row)
                     else:
                         log.warning('Inconsistent dimensions %s, %s' % (str(dims), fpath))
                     fh.close()
                     os.remove(fpath)
                     log.info('sample: ' + fpath)
                 except Exception as ex:
                     log.warn(ex)
                     log.warn('Incomplete packet %s' % fpath)
                     
         if cols and rows:
             coltypes = {}
             for c in cols:
                 coltypes[c] = '400i4'
             cols.append('time')
             samples = dict(cols=cols, data=rows, coltypes=coltypes)
             return samples

 def tearDown(self):
     self.waiter.stop()
     try:
         self.container.terminate_process(self._haa_pid)
     except BadRequest:
         log.warning("Couldn't terminate HA Agent in teardown (May have been terminated by a test)")
     self._stop_container()

    def retrieve_function_and_define_args(self, stream_id, dataprocess_id):
        import importlib
        argument_list = {}
        function = ''
        context = {}

        #load the details of this data process
        dataprocess_info = self._dataprocesses[dataprocess_id]


        try:
            #todo: load once into a 'set' of modules?
            #load the associated transform function
            egg_uri = dataprocess_info.get_safe('uri','')
            if egg_uri:
                egg = self.download_egg(egg_uri)
                import pkg_resources
                pkg_resources.working_set.add_entry(egg)
            else:
                log.warning('No uri provided for module in data process definition.')

            module = importlib.import_module(dataprocess_info.get_safe('module', '') )

            function = getattr(module, dataprocess_info.get_safe('function','') )
            arguments = dataprocess_info.get_safe('arguments', '')
            argument_list = dataprocess_info.get_safe('argument_map', {})

            if self.has_context_arg(function,argument_list ):
                context = self.create_context_arg(stream_id, dataprocess_id)

        except ImportError:
            log.error('Error running transform')
        log.debug('retrieve_function_and_define_args  argument_list: %s',argument_list)
        return function, argument_list, context

    def _register_service(self):
        if not self.process_definition_id:
            log.error("No process definition id. Not registering service")
            return

        if len(self.pds) < 1:
            log.error("Must have at least one PD available to register a service")
            return

        pd_name = self.pds[0]
        pd = ProcessDispatcherServiceClient(to_name=pd_name)
        definition = pd.read_process_definition(self.process_definition_id)

        existing_services, _ = self.container.resource_registry.find_resources(
                restype="Service", name=definition.name)

        if len(existing_services) > 0:
            if len(existing_services) > 1:
                log.warning("There is more than one service object for %s. Using the first one" % definition.name)
            service_id = existing_services[0]._id
        else:
            svc_obj = Service(name=definition.name, exchange_name=definition.name)
            service_id, _ = self.container.resource_registry.create(svc_obj)

        svcdefs, _ = self.container.resource_registry.find_resources(
                restype="ServiceDefinition", name=definition.name)

        if svcdefs:
            self.container.resource_registry.create_association(
                    service_id, "hasServiceDefinition", svcdefs[0]._id)
        else:
            log.error("Cannot find ServiceDefinition resource for %s",
                    definition.name)

        return service_id

 def _result_complete(self, result):
     """
     """
     if self._client:
         log.debug('Remote endpoint enqueuing result %s.', str(result))
         self._client.enqueue(result)
     else:
         log.warning('Received a result but no client available to transmit.')

    def on_start(self):
        super(NotificationWorker, self).on_start()

        self.smtp_client = setting_up_smtp_client()

        # ------------------------------------------------------------------------------------
        # Start by loading the user info and reverse user info dictionaries
        # ------------------------------------------------------------------------------------

        try:
            self.user_info = self.load_user_info()
            self.reverse_user_info = calculate_reverse_user_info(self.user_info)

            log.info("On start up, notification workers loaded the following user_info dictionary: %s" % self.user_info)
            log.info("The calculated reverse user info: %s" % self.reverse_user_info)

        except NotFound as exc:
            if exc.message.find("users_index") > -1:
                log.warning("Notification workers found on start up that users_index have not been loaded yet.")
            else:
                raise NotFound(exc.message)

        # ------------------------------------------------------------------------------------
        # Create an event subscriber for Reload User Info events
        # ------------------------------------------------------------------------------------

        def reload_user_info(event_msg, headers):
            """
            Callback method for the subscriber to ReloadUserInfoEvent
            """

            notification_id = event_msg.notification_id
            log.info(
                "(Notification worker received a ReloadNotificationEvent. The relevant notification_id is %s"
                % notification_id
            )

            try:
                self.user_info = self.load_user_info()
            except NotFound:
                log.warning("ElasticSearch has not yet loaded the user_index.")

            self.reverse_user_info = calculate_reverse_user_info(self.user_info)
            self.test_hook(self.user_info, self.reverse_user_info)

            log.debug("After a reload, the user_info: %s" % self.user_info)
            log.debug("The recalculated reverse_user_info: %s" % self.reverse_user_info)

        # the subscriber for the ReloadUSerInfoEvent
        self.reload_user_info_subscriber = EventSubscriber(event_type="ReloadUserInfoEvent", callback=reload_user_info)
        self.reload_user_info_subscriber.start()

        # ------------------------------------------------------------------------------------
        # Create an event subscriber for all events that are of interest for notifications
        # ------------------------------------------------------------------------------------

        self.event_subscriber = EventSubscriber(queue_name="uns_queue", callback=self.process_event)
        self.event_subscriber.start()

    def on_start(self):
        super(NotificationWorker,self).on_start()

        self.reverse_user_info = None
        self.user_info = None

        #------------------------------------------------------------------------------------
        # Start by loading the user info and reverse user info dictionaries
        #------------------------------------------------------------------------------------

        try:
            self.user_info = self.load_user_info()
            self.reverse_user_info =  calculate_reverse_user_info(self.user_info)

            log.debug("On start up, notification workers loaded the following user_info dictionary: %s" % self.user_info)
            log.debug("The calculated reverse user info: %s" % self.reverse_user_info )

        except NotFound as exc:
            if exc.message.find('users_index') > -1:
                log.warning("Notification workers found on start up that users_index have not been loaded yet.")
            else:
                raise NotFound(exc.message)

        #------------------------------------------------------------------------------------
        # Create an event subscriber for Reload User Info events
        #------------------------------------------------------------------------------------

        def reload_user_info(event_msg, headers):
            '''
            Callback method for the subscriber to ReloadUserInfoEvent
            '''

            notification_id =  event_msg.notification_id
            log.debug("(Notification worker received a ReloadNotificationEvent. The relevant notification_id is %s" % notification_id)

            try:
                self.user_info = self.load_user_info()
            except NotFound:
                log.warning("ElasticSearch has not yet loaded the user_index.")

            self.reverse_user_info =  calculate_reverse_user_info(self.user_info)
            self.test_hook(self.user_info, self.reverse_user_info)

            log.debug("After a reload, the user_info: %s" % self.user_info)
            log.debug("The recalculated reverse_user_info: %s" % self.reverse_user_info)

        # the subscriber for the ReloadUSerInfoEvent
        self.reload_user_info_subscriber = EventSubscriber(
            event_type="ReloadUserInfoEvent",
            origin='UserNotificationService',
            callback=reload_user_info
        )

        self.add_endpoint(self.reload_user_info_subscriber)

    def on_sample(self, sample):
        
        try:
            stream_name = sample['stream_name']
            self._stream_buffers[stream_name].insert(0, sample)
            if not self._stream_greenlets[stream_name]:
                self._publish_stream_buffer(stream_name)

        except KeyError:
            log.warning('Instrument agent %s received sample with bad stream name %s.',
                      self._agent._proc_name, stream_name)

 def _stop_pagent(self):
     """
     Stop the port agent.
     """
     if self._pagent:
         pid = self._pagent.get_pid()
         if pid:
             log.info('Stopping pagent pid %i.', pid)
             self._pagent.stop()
         else:
             log.warning('No port agent running.')

        def reload_user_info(event_msg, headers):
            '''
            Callback method for the subscriber to ReloadUserInfoEvent
            '''

            try:
                self.user_info = self.load_user_info()
            except NotFound:
                log.warning("ElasticSearch has not yet loaded the user_index.")

            self.reverse_user_info =  calculate_reverse_user_info(self.user_info)
            self.test_hook(self.user_info, self.reverse_user_info)

 def get_stored_values(self, lookup_value):
     if not self.new_lookups.empty():
         new_values = self.new_lookups.get()
         self.lookup_docs = new_values + self.lookup_docs
     lookup_value_document_keys = self.lookup_docs
     for key in lookup_value_document_keys:
         try:
             document = self.stored_value_manager.read_value(key)
             if lookup_value in document:
                 return document[lookup_value] 
         except NotFound:
             log.warning('Specified lookup document does not exist')
     return None

    def cancel(self, process_id):
        self.container.proc_manager.terminate_process(process_id)
        log.debug('PD: Terminated Process (%s)', process_id)
        try:
            self._remove_process(process_id)
        except ValueError:
            log.warning("PD: No record of %s to remove?" % process_id)

        self.event_pub.publish_event(event_type="ProcessLifecycleEvent",
            origin=process_id, origin_type="DispatchedProcess",
            state=ProcessStateEnum.TERMINATE)

        return True

def process_gateway_agent_request(resource_id, operation):
    try:
        if not resource_id:
            raise BadRequest("Am agent resource_id was not found in the URL")
        if operation == '':
            raise BadRequest("An agent operation was not specified in the URL")

        # Ensure there is no unicode
        resource_id = str(resource_id)
        operation = str(operation)

        # Retrieve json data from HTTP Post payload
        json_params = None
        if request.method == "POST":
            payload = request.form['payload']

            json_params = json_loads(str(payload))

            if 'agentRequest' not in json_params:
                raise Inconsistent("The JSON request is missing the 'agentRequest' key in the request")
            if 'agentId' not in json_params['agentRequest']:
                raise Inconsistent("The JSON request is missing the 'agentRequest' key in the request")
            if 'agentOp' not in json_params['agentRequest']:
                raise Inconsistent("The JSON request is missing the 'agentOp' key in the request")
            if json_params['agentRequest']['agentId'] != resource_id:
                raise Inconsistent("Target agent id in the JSON request (%s) does not match agent id in URL (%s)" % (
                    str(json_params['agentRequest']['agentId']), resource_id))
            if json_params['agentRequest']['agentOp'] != operation:
                raise Inconsistent("Target agent operation in the JSON request (%s) does not match agent operation in URL (%s)" % (
                    str(json_params['agentRequest']['agentOp']), operation))

        resource_agent = ResourceAgentClient(resource_id, process=service_gateway_instance)

        param_list = create_parameter_list('agentRequest', 'resource_agent', ResourceAgentProcessClient, operation, json_params)

        # Validate requesting user and expiry and add governance headers
        ion_actor_id, expiry = get_governance_info_from_request('agentRequest', json_params)
        ion_actor_id, expiry = validate_request(ion_actor_id, expiry)
        param_list['headers'] = build_message_headers(ion_actor_id, expiry)

        methodToCall = getattr(resource_agent, operation)
        result = methodToCall(**param_list)

        return gateway_json_response(result)

    except Exception, e:
        if e is NotFound:
            log.warning('The agent instance for id %s is not found.' % resource_id)
        return build_error_response(e)

    def tearDown(self):

        new_policy = {'preserve_n': 0}
        self.haa_client.reconfigure_policy(new_policy)

        self.await_ha_state('STEADY')
        self.assertEqual(len(self.get_running_procs()), 0)

        self.waiter.stop()
        try:
            self._kill_haagent()
        except BadRequest:
            log.warning("Couldn't terminate HA Agent in teardown (May have been terminated by a test)")
        self.container.resource_registry.delete(self.service_def_id, del_associations=True)
        self._stop_container()

    def _restart_transform(self, transform_id):
        transform = self.clients.resource_registry.read(transform_id)
        configuration = transform.configuration
        proc_def_ids,other = self.clients.resource_registry.find_objects(subject=transform_id,predicate=PRED.hasProcessDefinition,id_only=True)

        if len(proc_def_ids) < 1:
            log.warning('Transform did not have a correct process definition.')
            return

        pid = self.clients.process_dispatcher.schedule_process(
            process_definition_id=proc_def_ids[0],
            configuration=configuration
        )

        transform.process_id = pid
        self.clients.resource_registry.update(transform)

    def on_sample_mult(self, sample_list):
        """
        Enqueues a list of granules and publishes them
        """
        streams = set()
        for sample in sample_list:
            try:
                stream_name = sample['stream_name']
                self._stream_buffers[stream_name].insert(0, sample)
                streams.add(stream_name)
            except KeyError:
                log.warning('Instrument agent %s received sample with bad stream name %s.',
                          self._agent._proc_name, stream_name)

        for stream_name in streams:
            if not self._stream_greenlets[stream_name]:
                self._publish_stream_buffer(stream_name)

        def reload_user_info(event_msg, headers):
            """
            Callback method for the subscriber to ReloadUserInfoEvent
            """

            notification_id =  event_msg.notification_id
            log.debug("(UNS instance) received a ReloadNotificationEvent. The relevant notification_id is %s" % notification_id)

            try:
                self.user_info = self.load_user_info()
            except NotFound:
                log.warning("ElasticSearch has not yet loaded the user_index.")

            self.reverse_user_info =  calculate_reverse_user_info(self.user_info)

            log.debug("(UNS instance) After a reload, the user_info: %s" % self.user_info)
            log.debug("(UNS instance) The recalculated reverse_user_info: %s" % self.reverse_user_info)