def test_complex_version(self):

        io_serializer = IonObjectSerializer()
        obj = IonObject('SampleComplexEvent', {'num': 9, 'other_field': 'test value'})
        obj_dict = io_serializer.serialize(obj,True)
        self.assertEquals(obj_dict['persisted_version'], 1)
        # simulate a previous version data of SampleComplexEvent_V2
        obj_dict['type_'] = 'SampleComplexEvent_V2'

        # verify that the simulated previous version data has resource
        self.assertEquals('resource' in obj_dict, True)
        # verify that the simulated previous version data does not have new_attribute
        self.assertEquals('new_resource' in obj_dict, False)
        # simulate reading the previous version that does not have new_attribute
        io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
        obj = io_deserializer.deserialize(obj_dict)
        # verify that new attribute is added and initialized with default value
        self.assertEquals(obj.new_resource.new_attribute['key'], 'value')
        # verify that old attributes are still there
        self.assertEquals(obj.num, 9)
        # verify that old attributes are still there
        self.assertEquals(obj.other_field, 'test value')
        # verify that on read version is not yet updated
        self.assertEquals(obj_dict['persisted_version'], 1)

        # simulate create/update
        obj_dict = io_serializer.serialize(obj,True)
        # verify that version is updated
        self.assertEquals(obj_dict['persisted_version'], 2)

    def _build_stream_config(self):
        """
        """
        # Create a pubsub client to create streams.
        pubsub_client = PubsubManagementServiceClient(node=self.container.node)
        dataset_management = DatasetManagementServiceClient()

        encoder = IonObjectSerializer()

        # Create streams and subscriptions for each stream named in driver.
        self._stream_config = {}

        stream_name = 'ctdpf_parsed'
        param_dict_name = 'ctdpf_parsed'
        pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
        stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
        stream_def = pubsub_client.read_stream_definition(stream_def_id)
        stream_def_dict = encoder.serialize(stream_def)
        pd = stream_def.parameter_dictionary
        stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
                                                exchange_point='science_data',
                                                stream_definition_id=stream_def_id)
        stream_config = dict(routing_key=stream_route.routing_key,
                                 exchange_point=stream_route.exchange_point,
                                 stream_id=stream_id,
                                 parameter_dictionary=pd,
                                 stream_def_dict=stream_def_dict)
        self._stream_config[stream_name] = stream_config

    def test_persisted_version(self):

        # create an initial version of SampleResource
        io_serializer = IonObjectSerializer()
        obj = IonObject('SampleResource', {'num': 9, 'other_field': 'test value'})
        obj_dict = io_serializer.serialize(obj,True)
        self.assertEquals(obj_dict['persisted_version'], 1)
        # verify that the simulated previous version does not have new_attribute
        self.assertEquals('new_attribute' in obj_dict, False)

        # simulate version increment to SampleResource that adds new_attribute
        obj_dict['type_'] = 'SampleResource_V2'
        # simulate reading the previous version after version increment
        io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
        obj = io_deserializer.deserialize(obj_dict)
        # verify that “new_attribute” is added and initialized with default value
        self.assertEquals(obj.new_attribute['key'], 'value')
        # verify that old attributes are still there and retain values
        self.assertEquals(obj.num, 9)
        # verify that old attributes are still there and retain values
        self.assertEquals(obj.other_field, 'test value')
        # verify that persisted_version is not updated at read
        self.assertEquals(obj_dict['persisted_version'], 1)

        # simulate update
        obj_dict = io_serializer.serialize(obj,True)
        # verify that version is updated
        self.assertEquals(obj_dict['persisted_version'], 2)

def _process_gateway_request(resource_id, operation, json_request, requester):

    if requester is not None:
        json_request["agentRequest"]["requester"] = requester

    
    decoder = IonObjectSerializer()
    decoded_msg = decoder.serialize(json_request)
    payload = simplejson.dumps(decoded_msg)

    response = _agent_gateway_request(resource_id + '/' + operation,   payload)

    if response['data'].has_key(GATEWAY_ERROR):
        log.error(response['data'][GATEWAY_ERROR][GATEWAY_ERROR_MESSAGE])
        #raise BadRequest(response['data'][GATEWAY_ERROR][GATEWAY_ERROR_MESSAGE])
        ex_cls = response['data'][GATEWAY_ERROR][GATEWAY_ERROR_EXCEPTION]
        ex_msg = response['data'][GATEWAY_ERROR][GATEWAY_ERROR_MESSAGE]
        if hasattr(pyex, ex_cls):
            raise getattr(pyex, ex_cls)(ex_msg)
        
        else:
            raise Exception(ex_msg)

    try:
        if "type_" in response['data'][GATEWAY_RESPONSE]:
            del response['data'][GATEWAY_RESPONSE]["type_"]
    except Exception, e:
        pass

 def obj_to_tree(definition):
     from pyon.core.object import IonObjectSerializer
     if not isinstance(definition,StreamDefinitionContainer):
         return
     serializer = IonObjectSerializer()
     definition = serializer.serialize(definition)
     tree = DefinitionTree.traverse(definition,definition['data_stream_id'])
     return tree

    def _generate_stream_config(self):
        log.debug("_generate_stream_config for %s", self.agent_instance_obj.name)
        dsm = self.clients.dataset_management
        psm = self.clients.pubsub_management

        agent_obj  = self._get_agent()
        device_obj = self._get_device()

        streams_dict = {}
        for stream_cfg in agent_obj.stream_configurations:
            #create a stream def for each param dict to match against the existing data products
            streams_dict[stream_cfg.stream_name] = {'param_dict_name':stream_cfg.parameter_dictionary_name,
                                                    #'stream_def_id':stream_def_id,
                                                    'records_per_granule': stream_cfg.records_per_granule,
                                                    'granule_publish_rate':stream_cfg.granule_publish_rate,
                                                     }

        #retrieve the output products
        device_id = device_obj._id
        data_product_objs = self.RR2.find_data_products_of_instrument_device_using_has_output_product(device_id)

        stream_config = {}
        for d in data_product_objs:
            stream_def_id = self.RR2.find_stream_definition_id_of_data_product_using_has_stream_definition(d._id)
            for model_stream_name, stream_info_dict  in streams_dict.items():
                # read objects from cache to be compared
                pdict = self.RR2.find_resource_by_name(RT.ParameterDictionary, stream_info_dict.get('param_dict_name'))
                stream_def_id = self._meet_in_the_middle(d._id, pdict._id)

                if stream_def_id:
                    #model_param_dict = self.RR2.find_resources_by_name(RT.ParameterDictionary,
                    #                                         stream_info_dict.get('param_dict_name'))[0]
                    #model_param_dict = self._get_param_dict_by_name(stream_info_dict.get('param_dict_name'))
                    #stream_route = self.RR2.read(product_stream_id).stream_route
                    product_stream_id = self.RR2.find_stream_id_of_data_product_using_has_stream(d._id)
                    stream_def = psm.read_stream_definition(stream_def_id)
                    stream_route = psm.read_stream_route(stream_id=product_stream_id)
                    
                    from pyon.core.object import IonObjectSerializer
                    stream_def_dict = IonObjectSerializer().serialize(stream_def)
                    sdtype = stream_def_dict.pop('type_')

                    if model_stream_name in stream_config:
                        log.warn("Overwiting stream_config[%s]", model_stream_name)

                    stream_config[model_stream_name] = {'routing_key'           : stream_route.routing_key,
                                                        'stream_id'             : product_stream_id,
                                                        'stream_definition_ref' : stream_def_id,
                                                        'stream_def_dict'       : stream_def_dict,
                                                        'exchange_point'        : stream_route.exchange_point,
                                                        'parameter_dictionary'  : stream_def.parameter_dictionary,
                                                        'records_per_granule'   : stream_info_dict.get('records_per_granule'),
                                                        'granule_publish_rate'  : stream_info_dict.get('granule_publish_rate'),
                    }

        log.debug("Stream config generated")
        log.trace("generate_stream_config: %s", str(stream_config) )
        return stream_config

    def _generate_stream_config(self):
        log.debug("_generate_stream_config for %s", self.agent_instance_obj.name)
        dsm = self.clients.dataset_management
        psm = self.clients.pubsub_management

        agent_obj  = self._get_agent()
        device_obj = self._get_device()

        streams_dict = {}
        for stream_cfg in agent_obj.stream_configurations:
            #create a stream def for each param dict to match against the existing data products
            streams_dict[stream_cfg.stream_name] = {'param_dict_name':stream_cfg.parameter_dictionary_name}

        #retrieve the output products
        # TODO: What about platforms? other things?
        device_id = device_obj._id
        data_product_objs = self.RR2.find_data_products_of_instrument_device_using_has_output_product(device_id)

        stream_config = {}
        for dp in data_product_objs:
            stream_def_id = self.RR2.find_stream_definition_id_of_data_product_using_has_stream_definition(dp._id)
            for stream_name, stream_info_dict in streams_dict.items():
                # read objects from cache to be compared
                pdict = self.RR2.find_resource_by_name(RT.ParameterDictionary, stream_info_dict.get('param_dict_name'))
                stream_def_id = self._find_streamdef_for_dp_and_pdict(dp._id, pdict._id)

                if stream_def_id:
                    #model_param_dict = self.RR2.find_resources_by_name(RT.ParameterDictionary,
                    #                                         stream_info_dict.get('param_dict_name'))[0]
                    #model_param_dict = self._get_param_dict_by_name(stream_info_dict.get('param_dict_name'))
                    #stream_route = self.RR2.read(product_stream_id).stream_route
                    product_stream_id = self.RR2.find_stream_id_of_data_product_using_has_stream(dp._id)
                    stream_def = psm.read_stream_definition(stream_def_id)
                    stream_route = psm.read_stream_route(stream_id=product_stream_id)

                    from pyon.core.object import IonObjectSerializer
                    stream_def_dict = IonObjectSerializer().serialize(stream_def)
                    stream_def_dict.pop('type_')

                    if stream_name in stream_config:
                        log.warn("Overwriting stream_config[%s]", stream_name)

                    stream_config[stream_name] = {  'routing_key'           : stream_route.routing_key,  # TODO: Serialize stream_route together
                                                    'stream_id'             : product_stream_id,
                                                    'stream_definition_ref' : stream_def_id,
                                                    'stream_def_dict'       : stream_def_dict,  # This is very large
                                                    'exchange_point'        : stream_route.exchange_point,
                                                    # This is redundant and very large - the param dict is in the stream_def_dict
                                                    #'parameter_dictionary'  : stream_def.parameter_dictionary,

                    }
        if len(stream_config) < len(streams_dict):
            log.warn("Found only %s matching streams by stream definition (%s) than %s defined in the agent (%s).",
                     len(stream_config), stream_config.keys(), len(streams_dict), streams_dict.keys())

        log.debug("Stream config generated")
        log.trace("generate_stream_config: %s", stream_config)
        return stream_config

    def _serialize_port_assigments(self, port_assignments=None):
        serializer = IonObjectSerializer()
        serialized_port_assignments = {}
        if isinstance(port_assignments, dict):
            for device_id, platform_port in port_assignments.iteritems():
                flatpp = serializer.serialize(platform_port)
                serialized_port_assignments[device_id] = flatpp

        return serialized_port_assignments

 def size(self):
     '''
     Truly poor way to calculate the size of a granule...
     returns the size in bytes.
     '''
     granule = self.to_granule()
     serializer = IonObjectSerializer()
     flat = serializer.serialize(granule)
     byte_stream = msgpack.packb(flat, default=encode_ion)
     return len(byte_stream)

def _process_gateway_request(service_name, operation, json_request, requester):

    if requester is not None:
        json_request["serviceRequest"]["requester"] = requester


    decoder = IonObjectSerializer()
    decoded_msg = decoder.serialize(json_request)
    payload = simplejson.dumps(decoded_msg)

    response = _service_gateway_request(service_name + '/' + operation,   payload)

    return response

    def test_attribute_version(self):

        io_serializer = IonObjectSerializer()

        # verify that extraneous fields given while creating an IonObject raises an error.
        with self.assertRaises(AttributeError):
            IonObject('SampleComplexEvent_V2', {'num': 9, 'other_field': 'test value','more_new_resource':
                {'key':'value'}})

        obj = IonObject('SampleComplexEvent_V2', {'num': 9, 'other_field': 'test value','new_resource':
            {'num': 9, 'other_field': 'test value','new_attribute':{'key':'value'}}})
        obj_dict = io_serializer.serialize(obj,True)
        self.assertEquals(obj_dict['persisted_version'], 2)

        # verify that the simulated previous version data does have new_resource
        self.assertEquals('new_resource' in obj_dict, True)
        # verify that the new_resource has type SampleResource_V2
        self.assertEquals(obj_dict['new_resource']['type_'],"SampleResource_V2")

        # set type to SampleComplexEvent_V3
        obj_dict['type_']="SampleComplexEvent_V3"
        obj_dict['persisted_version']=3
        # set new_resource's type to SampleResource_V3
        # so we pretend that version, not the type, of the attribute has been changed
        obj_dict['new_resource']['type_']="SampleResource_V3"

        # simulate reading SampleComplexEvent_V3 after a new version of new_resource has been introduced
        io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
        obj = io_deserializer.deserialize(obj_dict)

        # verify that new resource is not deleted
        self.assertTrue('new_resource' in obj)
        # verify that new resource does not have new_attribute
        self.assertFalse('new_attribute' in obj.new_resource)
        # verify that the next version of new_resource has default data in the another_new_attribute
        self.assertEquals(obj.new_resource.another_new_attribute['key'], 'new_value')
        # verify that old attributes values of new_resource have not been thrown away
        self.assertEquals(obj.new_resource.num, 9)
        # verify that values from old attributes of SampleComplexEvent_V2 are still there
        self.assertEquals(obj.num, 9)
        self.assertEquals(obj.other_field, 'test value')

        # verify that on read version is not yet updated for the subsumed object
        self.assertEquals(obj.new_resource.persisted_version, 2)

        # simulate create/update
        obj_dict = io_serializer.serialize(obj,True)
        # verify that versions are unchanged
        self.assertEquals(obj_dict['persisted_version'], 3)
        # verify that versions are updated in the subsumed object
        self.assertEquals(obj_dict['new_resource']['persisted_version'], 3)

    def test_complex_version_del_attrib(self):

        io_serializer = IonObjectSerializer()
        # verify that extraneous fields given while creating an IonObject raises an error.
        with self.assertRaises(AttributeError):
            IonObject('SampleComplexEvent_V2', {'num': 9, 'other_field': 'test value','more_new_resource': {'key':'value'}})

        obj = IonObject('SampleComplexEvent_V2', {'num': 9, 'other_field': 'test value','new_resource': {'num': 9, 'other_field': 'test value','new_attribute':{'key':'value'}}})
        # create simulated saved data
        obj_dict = io_serializer.serialize(obj,True)
        self.assertEquals(obj_dict['persisted_version'], 2)
        # simulate a next version data of SampleComplexEvent_V2
        obj_dict['type_'] = 'SampleComplexEvent_V3'

        # verify that the simulated previous version data does have new_resource
        self.assertEquals('new_resource' in obj_dict, True)
        # note the schema version of new_resource
        self.assertEquals(obj_dict['new_resource']['persisted_version'], 2)

        # simulate reading the next version that has a new type of new_resource
        io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
        obj = io_deserializer.deserialize(obj_dict)

        # verify that new_resource exists
        self.assertTrue('new_resource' in obj)
        # however, verify that new_resource does not have new_attribute since type of new_resource has changed
        self.assertFalse('new_attribute' in obj.new_resource)
        # verify that the new type of new_resource has another_new_attribute that is initialized to default data
        self.assertEquals(obj.new_resource.another_new_attribute['key'], 'new_value')
        # verify on read that the schema version of new_resource replaces the old persisted_version
        self.assertEquals(obj.new_resource.persisted_version, 3)

        # verify that old attributes values of new_resource have been thrown away
        self.assertNotEquals(obj.new_resource.num, 9)
        # verify that attributes values of new_resource have been initialized to default values
        self.assertEquals(obj.new_resource.num, 0)

        # However, verify that old attributes of the resource (SampleComplexEvent) are still there
        self.assertEquals(obj.num, 9)
        # verify that old attributes are still there
        self.assertEquals(obj.other_field, 'test value')
        # verify that on read, version is not yet updated
        self.assertEquals(obj.persisted_version, 2)


        # simulate create/update
        obj_dict = io_serializer.serialize(obj,True)
        # verify that version is updated
        self.assertEquals(obj_dict['persisted_version'], 3)
        # verify that version is updated fo the subsumed object
        self.assertEquals(obj_dict['new_resource']['persisted_version'], 3)

class CodecInterceptor(Interceptor):
    """
    Transforms IonObject <-> dict
    """
    def __init__(self):
        Interceptor.__init__(self)
        self._io_serializer = IonObjectSerializer()
        self._io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())

    def outgoing(self, invocation):
        #log.debug("CodecInterceptor.outgoing: %s", invocation)

        #log.debug("Payload, pre-transform: %s", invocation.message)
        invocation.message = self._io_serializer.serialize(invocation.message)
        #log.debug("Payload, post-transform: %s", invocation.message)

        return invocation

    def incoming(self, invocation):
        #log.debug("CodecInterceptor.incoming: %s", invocation)

        payload = invocation.message
        #log.debug("Payload, pre-transform: %s", payload)

        invocation.message = self._io_deserializer.deserialize(payload)
        #log.debug("Payload, post-transform: %s", invocation.message)

        return invocation

    def __init__(self, container, datastore_name=""):
        self.container = container
        self.datastore_name = datastore_name

        # Object serialization/deserialization
        self._io_serializer = IonObjectSerializer()
        self._io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())

    def __init__(self, datastore_name='prototype'):
        self.datastore_name = datastore_name
        log.debug('Creating in-memory dict of dicts that will simulate data stores')
        self.root = {}

        # serializers
        self._io_serializer     = IonObjectSerializer()
        self._io_deserializer   = IonObjectDeserializer(obj_registry=get_obj_registry())

    def __init__(self):
        BaseIngestionManagementService.__init__(self)

        xs_dot_xp = CFG.core_xps.science_data
        try:
            self.XS, xp_base = xs_dot_xp.split('.')
            self.XP = '.'.join([bootstrap.get_sys_name(), xp_base])
        except ValueError:
            raise StandardError('Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure' % xs_dot_xp)

        self.serializer = IonObjectSerializer()

class IonSerializerDictionaryRepresentation(Representation):
    def __init__(self, id_factory):
        self.encoder = IonObjectSerializer()
        self.decoder = IonObjectDeserializer(obj_registry=get_obj_registry())
        self.id_factory = id_factory
    def encode(self, obj, add_id=False):
        out = self.encoder.serialize(obj)
        if add_id and '_id' not in out.keys():
            out['_id'] = self.id_factory.create_id()
        return out
    def decode(self, data):
        return self.decoder.deserialize(data)

    def test_version_del_attrib(self):

        io_serializer = IonObjectSerializer()

        # verify that extraneous fields given while creating an IonObject raises an error.
        with self.assertRaises(AttributeError):
            IonObject('SampleResource_V2', {'num': 9, 'other_field': 'test value','more_new_attribute': {'key':'value'}})
        # simulate creating a version 2 of SampleResource that has "new_attribute"
        obj = IonObject('SampleResource_V2', {'num': 9, 'other_field': 'test value','new_attribute': {'key':'value'}})
        obj_dict = io_serializer.serialize(obj,True)
        # verify that version is 2
        self.assertEquals(obj_dict['persisted_version'], 2)
        # verify that the simulated version 2 data does have new_attribute
        self.assertEquals('new_attribute' in obj_dict, True)


        # simulate incrementing to version 3 that does not have "new_attribute"
        obj_dict['type_'] = 'SampleResource_V3'

        # simulate reading after version increment to 3
        io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
        obj = io_deserializer.deserialize(obj_dict)

        # verify that new attribute is deleted
        self.assertFalse('new_attribute' in obj)
        # verify that the simulated next version data does have more_new_attribute
        self.assertEquals(obj.another_new_attribute['key'], 'new_value')

        # verify that old attributes are still there and retain their data
        self.assertEquals(obj.num, 9)
        # verify that old attributes are still there and retain their data
        self.assertEquals(obj.other_field, 'test value')
        # verify that persisted_version is not yet updated i.e. it is still 2
        self.assertEquals(obj_dict['persisted_version'], 2)

        # simulate update
        obj_dict = io_serializer.serialize(obj,True)
        # verify that version is updated
        self.assertEquals(obj_dict['persisted_version'], 3)

    def __init__(self, datastore_name=None, config=None, scope=None, profile=None):
        """
        @param datastore_name  Name of datastore within server. May be scoped to sysname
        @param config  A server config dict with connection params
        @param scope  Prefix for the datastore name (e.g. sysname) to separate multiple systems
        """

        PostgresDataStore.__init__(self, datastore_name=datastore_name,
                                     config=config or CFG.get_safe("server.postgresql"),
                                     profile=profile or DataStore.DS_PROFILE.BASIC,
                                     scope=scope)

        # IonObject Serializers
        self._io_serializer = IonObjectSerializer()
        self._io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())

    def test_event_version_del_attrib(self):

        io_serializer = IonObjectSerializer()

        # verify that extraneous fields given while creating an IonObject raises an error.
        with self.assertRaises(AttributeError):
            IonObject('SampleEvent_V2', {'num': 9, 'other_field': 'test value','more_new_attribute': {'key':'value'}})

        obj = IonObject('SampleEvent_V2', {'num': 9, 'other_field': 'test value','new_attribute': {'key':'value'}})
        obj_dict = io_serializer.serialize(obj,True)
        self.assertEquals(obj_dict['persisted_version'], 2)
        # simulate a next version data of SampleEvent_V2
        obj_dict['type_'] = 'SampleEvent_V3'

        # verify that the simulated previous version data does have new_attribute
        self.assertEquals('new_attribute' in obj_dict, True)
        # simulate reading the next version that does not have new_attribute
        io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
        obj = io_deserializer.deserialize(obj_dict)

        # verify that new attribute is deleted
        self.assertFalse('new_attribute' in obj)
        # verify that the simulated next version data does have more_new_attribute
        self.assertEquals(obj.another_new_attribute['key'], 'new_value')

        # verify that old attributes are still there
        self.assertEquals(obj.num, 9)
        # verify that old attributes are still there
        self.assertEquals(obj.other_field, 'test value')
        # verify that on read version is not yet updated
        self.assertEquals(obj_dict['persisted_version'], 2)

        # simulate create/update
        obj_dict = io_serializer.serialize(obj,True)
        # verify that version is updated
        self.assertEquals(obj_dict['persisted_version'], 3)