def create_parameter_context(self, name='', parameter_context=None, description='', reference_urls=None, parameter_type='', internal_name='', value_encoding='', code_report='', units='', fill_value='', display_name='', parameter_function_id='', parameter_function_map='', standard_name='', ooi_short_name='', precision='', visible=True):
        
        validate_true(name, 'Name field may not be empty')
        validate_is_instance(parameter_context, dict, 'parameter_context field is not dictable.')
        parameter_context = self.numpy_walk(parameter_context)
        pc_res = ParameterContextResource(name=name, parameter_context=parameter_context, description=description)
        pc_res.reference_urls = reference_urls or []
        pc_res.parameter_type = parameter_type
        pc_res.internal_name = internal_name or name
        pc_res.value_encoding = value_encoding
        pc_res.code_report = code_report or ''
        pc_res.units = units
        pc_res.fill_value = fill_value
        pc_res.display_name = display_name
        pc_res.parameter_function_id = parameter_function_id
        pc_res.parameter_function_map = parameter_function_map
        pc_res.standard_name = standard_name
        pc_res.ooi_short_name = ooi_short_name
        pc_res.precision = precision or '5'
        pc_res.visible = visible

        pc_id, ver = self.clients.resource_registry.create(pc_res)
        if parameter_function_id:
            self.read_parameter_function(parameter_function_id)
            self.clients.resource_registry.create_association(subject=pc_id, predicate=PRED.hasParameterFunction, object=parameter_function_id)
        
        return pc_id

    def create_parameter_context(self, name='', parameter_context=None, description='', reference_urls=None, parameter_type='', internal_name='', value_encoding='', code_report=None, units='', fill_value='', display_name='', parameter_function_id='', parameter_function_map=None, standard_name='', ooi_short_name='', precision=''):
        res, _ = self.clients.resource_registry.find_resources(restype=RT.ParameterContext, name=name, id_only=False)
        if len(res):
            for r in res:
                if r.name == name and self._compare_pc(r.parameter_context, parameter_context):
                    return r._id
        
        validate_true(name, 'Name field may not be empty')
        validate_is_instance(parameter_context, dict, 'parameter_context field is not dictable.')
        pc_res = ParameterContextResource(name=name, parameter_context=parameter_context, description=description)
        pc_res.reference_urls = reference_urls or []
        pc_res.parameter_type = parameter_type
        pc_res.internal_name = internal_name or name
        pc_res.value_encoding = value_encoding
        pc_res.code_report = code_report or {}
        pc_res.units = units
        pc_res.fill_value = fill_value
        pc_res.display_name = display_name
        pc_res.parameter_function_id = parameter_function_id
        pc_res.parameter_function_map = parameter_function_map
        pc_res.standard_name = standard_name
        pc_res.ooi_short_name = ooi_short_name
        pc_res.precision = precision or '5'

        pc_id, ver = self.clients.resource_registry.create(pc_res)
        if parameter_function_id:
            self.read_parameter_function(parameter_function_id)
            self.clients.resource_registry.create_association(subject=pc_id, predicate=PRED.hasParameterFunction, object=parameter_function_id)
        
        return pc_id

    def suspend_data_product_persistence(self, data_product_id=''):
        """Suspend data product data persistence into a data set, multiple options

        @param data_product_id    str
        @param type    str
        @throws NotFound    object with specified id does not exist
        """

        #--------------------------------------------------------------------------------
        # retrieve the data_process object
        #--------------------------------------------------------------------------------
        data_product_obj = self.clients.resource_registry.read(data_product_id)

        validate_is_not_none(data_product_obj, 'Should not have been empty')
        validate_is_instance(data_product_obj, DataProduct)

        if data_product_obj.dataset_configuration_id is None:
            raise NotFound("Data Product %s dataset configuration does not exist" % data_product_id)

        #--------------------------------------------------------------------------------
        # get the Stream associated with this data product; if no stream then create one, if multiple streams then Throw
        #streams = self.data_product.find_stemming_stream(data_product_id)
        #--------------------------------------------------------------------------------
        stream_id = self._get_stream_id(data_product_id)
        validate_is_not_none(stream_id, 'Data Product %s must have one stream associated' % str(data_product_id))

        ret = self.clients.ingestion_management.unpersist_data_stream(stream_id=stream_id, ingestion_configuration_id=data_product_obj.dataset_configuration_id)

 def read_dataset(self, dataset_id=''):
     """
     @throws NotFound if resource does not exist.
     """
     retval = self.clients.resource_registry.read(dataset_id)
     validate_is_instance(retval,DataSet)
     return retval

    def persist_data_stream(self, stream_id='', ingestion_configuration_id='', dataset_id=''):
        # Figure out which MIME or xpath in the stream definition belongs where

        # Just going to use the first queue for now

        validate_is_instance(stream_id,basestring, 'stream_id %s is not a valid string' % stream_id)
        validate_true(dataset_id,'Clients must specify the dataset to persist')

        ingestion_config = self.read_ingestion_configuration(ingestion_configuration_id)
        if self.is_persisted(stream_id):
            raise BadRequest('This stream is already being persisted')
        stream = self.clients.pubsub_management.read_stream(stream_id)
        stream.persisted = True
        self.clients.pubsub_management.update_stream(stream)

        ingestion_queue = self._determine_queue(stream_id, ingestion_config.queues)

        subscription_id = self.clients.pubsub_management.create_subscription(
            query=StreamQuery(stream_ids=[stream_id]),
            exchange_name=ingestion_queue.name,
            exchange_point=ingestion_config.exchange_point
        )

        self.clients.pubsub_management.activate_subscription(subscription_id=subscription_id)

        self.clients.resource_registry.create_association(
            subject=ingestion_configuration_id,
            predicate=PRED.hasSubscription,
            object=subscription_id
        )
        self._existing_dataset(stream_id,dataset_id)
        return dataset_id

    def retrieve(self, dataset_id='', query=None, delivery_format=None, module='', cls='', kwargs=None):

        if query is None:
            query = {}
        if delivery_format is None:
            delivery_format = {}

        validate_is_instance(query,dict,'Query was improperly formatted.')
        validate_true(dataset_id, 'No dataset provided')
        

        replay_instance = ReplayProcess()

        replay_instance.dataset = self.clients.dataset_management.read_dataset(dataset_id)
        replay_instance.dataset_id = dataset_id
        replay_instance.start_time = query.get('start_time', None)
        replay_instance.end_time = query.get('end_time', None)
        replay_instance.parameters = query.get('parameters',None)
        replay_instance.container = self.container

        retrieve_data = replay_instance.execute_retrieve()

        if module and cls:
            return self._transform_data(retrieve_data, module, cls, kwargs or {})

        return retrieve_data

 def create_parameter_function(self, name='', parameter_function=None, description=''):
     validate_true(name, 'Name field may not be empty')
     validate_is_instance(parameter_function, dict, 'parameter_function field is not dictable.')
     parameter_function = self.numpy_walk(parameter_function)
     pf_res = ParameterFunctionResource(name=name, parameter_function=parameter_function, description=description)
     pf_id, ver = self.clients.resource_registry.create(pf_res)
     return pf_id

 def _check_response(response):
     validate_is_instance(response,dict,"Malformed response from ElasticSearch (%s)" % response, ServerError)
     if response.has_key('error'):
         raise ServerError("ElasticSearch error: %s" % response['error'])
     if response.has_key('ok') and not response['ok']:
         # Cant determine a better exception to throw, add a new one perhapse?
         raise NotFound("ElasticSearch Response: %s" % response)

 def shift(vector):
     validate_is_instance(vector,list)
     N = len(vector)
     x = vector[0]
     vector[0] = vector[N-1]
     vector[N-1] = x
     return vector

    def create_view(self, view_name='', description='', fields=None, order=None, filters=''):
        """Creates a view which has the specified search fields, the order in which the search fields are presented
        to a query and a term filter.
        @param view_name Name of the view
        @param description Simple descriptive sentence
        @param fields Search fields
        @param order List of fields to determine order of precendence in which the results are presented
        @param filter Simple term filter

        @param view_name    str
        @param description    str
        @param fields    list
        @param order    list
        @param filters    str
        @retval view_id    str
        """
        res, _ = self.clients.resource_registry.find_resources(name=view_name, id_only=True)
        if len(res) > 0:
            raise BadRequest('The view resource with name: %s, already exists.' % view_name)

        #======================
        # Arg Validations
        #======================
        validate_is_instance(fields,list, 'Specified fields must be a list.')
        validate_true(len(fields)>0, 'Specfied fields must be a list.')
        if order is not None:
            validate_is_instance(order,list, 'Specified order must be a list of fields')
            for field in order:
                if not field in fields:
                    raise BadRequest('The specified ordering field was not part of the search fields.')

        fields = set(fields) # Convert fields to a set for aggregation across the catalogs
        #======================================================================================================
        # Priorty Queue Index Matching
        #======================================================================================================

        pq = [] # Priority queue for matching
        catalog_id = None
        catalogs, _ = self.clients.resource_registry.find_resources(restype=RT.Catalog, id_only=False)
        for catalog in catalogs:
            if set(catalog.catalog_fields).issubset(fields):
                index_num = len(self.clients.catalog_management.list_indexes(catalog._id))
                heapq.heappush(pq, (index_num,catalog))
        if pq:
            weight, catalog = heapq.heappop(pq)
            if weight < self.heuristic_cutoff:
                catalog_id = catalog._id

                
        if catalog_id is None:
            catalog_id = self.clients.catalog_management.create_catalog('%s_catalog'% view_name, keywords=list(fields))

        view_res = View(name=view_name, description=description)
        view_res.order = order
        view_res.filters = filters
        view_id, _ = self.clients.resource_registry.create(view_res)
        self.clients.resource_registry.create_association(subject=view_id, predicate=PRED.hasCatalog,object=catalog_id)
        return view_id

    def query_term(
        self, source_id="", field="", value="", fuzzy=False, match=False, order=None, limit=0, offset=0, id_only=False
    ):
        """
        Elasticsearch Query against an index
        > discovery.query_index('indexID', 'name', '*', order={'name':'asc'}, limit=20, id_only=False)
        """
        if not self.use_es:
            raise BadRequest("Can not make queries without ElasticSearch, enable system.elasticsearch to make queries.")

        validate_true(source_id, "Unspecified source_id")
        validate_true(field, "Unspecified field")
        validate_true(value, "Unspecified value")

        es = ep.ElasticSearch(host=self.elasticsearch_host, port=self.elasticsearch_port)

        source = self.clients.resource_registry.read(source_id)

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # If source is a view, catalog or collection go through it and recursively call query_range on all the results in the indexes
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        iterate = self._multi(
            self.query_term, source, field=field, value=value, order=order, limit=limit, offset=offset, id_only=id_only
        )
        if iterate is not None:
            return iterate

        index = source
        validate_is_instance(index, ElasticSearchIndex, "%s does not refer to a valid index." % index)
        if order:
            validate_is_instance(order, dict, "Order is incorrect.")
            es.sort(**order)

        if limit:
            es.size(limit)

        if offset:
            es.from_offset(offset)

        if field == "*":
            field = "_all"

        if fuzzy:
            query = ep.ElasticQuery.fuzzy_like_this(value, fields=[field])
        elif match:
            match_query = ep.ElasticQuery.match(field=field, query=value)
            query = {"match_phrase_prefix": match_query["match"]}

        elif "*" in value:
            query = ep.ElasticQuery.wildcard(field=field, value=value)
        else:
            query = ep.ElasticQuery.field(field=field, query=value)

        response = IndexManagementService._es_call(es.search_index_advanced, index.index_name, query)

        IndexManagementService._check_response(response)

        return self._results_from_response(response, id_only)

    def query_geo_bbox(
        self, source_id="", field="", top_left=None, bottom_right=None, order=None, limit=0, offset=0, id_only=False
    ):
        validate_true(isinstance(top_left, (list, tuple)), "Top Left is not a list or a tuple")
        validate_true(len(top_left) == 2, "Top Left is not of the right size: (2)")
        validate_true(isinstance(bottom_right, (list, tuple)), "Bottom Right is not a list or a tuple")
        validate_true(len(bottom_right) == 2, "Bottom Right is not of the right size: (2)")

        if not self.use_es:
            raise BadRequest("Can not make queries without ElasticSearch, enable in res/config/pyon.yml")

        es = ep.ElasticSearch(host=self.elasticsearch_host, port=self.elasticsearch_port)
        source = self.clients.resource_registry.read(source_id)

        iterate = self._multi(
            self.query_geo_bbox,
            source=source,
            field=field,
            top_left=top_left,
            bottom_right=bottom_right,
            order=order,
            limit=limit,
            offset=offset,
            id_only=id_only,
        )
        if iterate is not None:
            return iterate

        index = source
        validate_is_instance(index, ElasticSearchIndex, "%s does not refer to a valid index." % index)

        sorts = ep.ElasticSort()
        if order is not None and isinstance(order, dict):
            sort_field = order.keys()[0]
            value = order[sort_field]
            sorts.sort(sort_field, value)
            es.sorted(sorts)

        if limit:
            es.size(limit)

        if offset:
            es.from_offset(offset)

        if field == "*":
            field = "_all"

        filter = ep.ElasticFilter.geo_bounding_box(field, top_left, bottom_right)

        es.filtered(filter)

        query = ep.ElasticQuery.match_all()

        response = IndexManagementService._es_call(es.search_index_advanced, index.index_name, query)
        IndexManagementService._check_response(response)

        return self._results_from_response(response, id_only)

    def update_data_product(self, data_product=None):
        """
        @todo document this interface!!!

        @param data_product    DataProduct
        @throws NotFound    object with specified id does not exist
        """
        validate_is_instance(data_product, DataProduct)

        self.data_product.update_one(data_product)

 def __init__(self, stream_id, stream_route):
     '''
     Creates a new StandaloneStreamPublisher
     @param stream_id    The stream identifier
     @param stream_route The StreamRoute to publish on.
     '''
     super(StandaloneStreamPublisher, self).__init__()
     self.stream_id = stream_id
     validate_is_instance(stream_route, StreamRoute, 'stream route is not valid')
     self.stream_route = stream_route

    def query_range(
        self, source_id="", field="", from_value=None, to_value=None, order=None, limit=0, offset=0, id_only=False
    ):

        if not self.use_es:
            raise BadRequest("Can not make queries without ElasticSearch, enable in res/config/pyon.yml")

        if from_value is not None:
            validate_true(
                isinstance(from_value, int) or isinstance(from_value, float), "from_value is not a valid number"
            )
        if to_value is not None:
            validate_true(isinstance(to_value, int) or isinstance(to_value, float), "to_value is not a valid number")
        validate_true(source_id, "source_id not specified")

        es = ep.ElasticSearch(host=self.elasticsearch_host, port=self.elasticsearch_port)

        source = self.clients.resource_registry.read(source_id)

        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        # If source is a view, catalog or collection go through it and recursively call query_range on all the results in the indexes
        # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        iterate = self._multi(
            self.query_range,
            source,
            field=field,
            from_value=from_value,
            to_value=to_value,
            order=order,
            limit=limit,
            offset=offset,
            id_only=id_only,
        )
        if iterate is not None:
            return iterate

        index = source
        validate_is_instance(index, ElasticSearchIndex, "%s does not refer to a valid index." % source_id)
        if order:
            validate_is_instance(order, dict, "Order is incorrect.")
            es.sort(**order)

        if limit:
            es.size(limit)

        if field == "*":
            field = "_all"

        query = ep.ElasticQuery.range(field=field, from_value=from_value, to_value=to_value)
        response = IndexManagementService._es_call(es.search_index_advanced, index.index_name, query)

        IndexManagementService._check_response(response)

        return self._results_from_response(response, id_only)

    def read_data_product(self, data_product_id=''):
        """
        method docstring
        """
        # Retrieve all metadata for a specific data product
        # Return data product resource

        data_product = self.data_product.read_one(data_product_id)
        validate_is_instance(data_product,DataProduct)

        return data_product

 def find_associations_mult(self, subjects, id_only=False):
     ds, datastore_name = self._get_datastore()
     validate_is_instance(subjects, list, 'subjects is not a list of resource_ids')
     view_args = dict(keys=subjects, include_docs=True)
     results = self.query_view(self._get_viewname("association","by_bulk"), view_args)
     ids = list([i['value'] for i in results])
     assocs = list([i['doc'] for i in results])
     if id_only:
         return ids, assocs
     else:
         return self.read_mult(ids), assocs

 def ingest(self, msg, stream_id):
     '''
     Actual ingestion mechanism
     '''
     if msg == {}:
         log.error('Received empty message from stream: %s', stream_id)
         return
     # Message validation
     validate_is_instance(msg,Granule,'Incoming message is not compatible with this ingestion worker')
     granule = msg
     self.add_granule(stream_id, granule)
     self.persist_meta(stream_id, granule)

    def add_indexes(self, catalog_id='', index_ids=None):
        """Add an index to the specified catalog

        @param index_ids    list
        @retval success    bool
        """
        validate_is_instance(index_ids,list, "A list of index IDs was not provided.")
        
        for index_id in index_ids:
            self.clients.resource_registry.create_association(subject=catalog_id, predicate=PRED.hasIndex,object=index_id)

        return True

    def dct(vector):
        import numpy as np
        validate_is_instance(vector,np.ndarray)
        N = vector.size

        x = list()
        for k in xrange(N):
            if k == 0:
                x.append( float(np.sqrt(1./N) * np.sum(vector)))
            else:
                v = np.vectorize(lambda n : np.cos(np.pi * k *  (2. * n + 1.)  / (2. * N)))
                x.append(float(np.sqrt(2. / N) * np.sum(v(np.arange(0,N)) * vector)))
        return x