def test_pickle(self):
        a = T.scalar()  # the a is for 'anonymous' (un-named).
        x, s = T.scalars('xs')

        f = function([x, In(a, value=1.0, name='a'), In(s, value=0.0, update=s+a*x, mutable=True)], s+a*x)

        try:
            # Note that here we also test protocol 0 on purpose, since it
            # should work (even though one should not use it).
            g = pickle.loads(pickle.dumps(f, protocol=0))
            g = pickle.loads(pickle.dumps(f, protocol=-1))
        except NotImplementedError as e:
            if e[0].startswith('DebugMode is not picklable'):
                return
            else:
                raise
        # if they both return, assume  that they return equivalent things.
        # print [(k,id(k)) for k in f.finder.keys()]
        # print [(k,id(k)) for k in g.finder.keys()]

        self.assertFalse(g.container[0].storage is f.container[0].storage)
        self.assertFalse(g.container[1].storage is f.container[1].storage)
        self.assertFalse(g.container[2].storage is f.container[2].storage)
        self.assertFalse(x in g.container)
        self.assertFalse(x in g.value)

        self.assertFalse(g.value[1] is f.value[1])  # should not have been copied
        self.assertFalse(g.value[2] is f.value[2])  # should have been copied because it is mutable.
        self.assertFalse((g.value[2] != f.value[2]).any())  # its contents should be identical

        self.assertTrue(f(2, 1) == g(2))  # they should be in sync, default value should be copied.
        self.assertTrue(f(2, 1) == g(2))  # they should be in sync, default value should be copied.
        f(1, 2)  # put them out of sync
        self.assertFalse(f(1, 2) == g(1, 2))  # they should not be equal anymore.

    def start(self, cache, job_specs, callback):
        """Run jobs on the backend, blocking until their completion.

        Args:
            cache: The persistent cache which should be set on the backend
            job_specs: The job specification (see
                owls_parallel.backends.ParallelizationBackend)
            callback: The job notification callback, not used by this backend
        """
        # Create the result list
        results = []

        # Go through each job and create a batch job for it
        for spec in itervalues(job_specs):
            # Create the job content
            batch_script = _BATCH_TEMPLATE.format(**{
                "cache": b64encode(dumps(cache)),
                "job": b64encode(dumps(spec)),
            })

            # Create an on-disk handle
            script_name = '{0}.py'.format(uuid4().hex)
            script_path = join(self._path, script_name)

            # Write it to file
            with open(script_path, 'w') as f:
                f.write(batch_script)

            # Submit the batch job and record the job id
            results.append(self._submit(self._path, script_name))

        # All done
        return results

    def save(self, path, items):
        # TODO: purge old cache
        with atomic_file(path) as f:
            c = 0
            f.write(struct.pack("I", c))
            # check is marshalable and compatible with broadcast
            can_marshal = marshalable(items)
            for v in items:
                if can_marshal:
                    try:
                        r = 0, marshal.dumps(v)
                    except Exception:
                        r = 1, cPickle.dumps(v, -1)
                        can_marshal = False
                else:
                    r = 1, cPickle.dumps(v, -1)
                f.write(msgpack.packb(r))
                c += 1
                yield v

            bytes = f.tell()
            if bytes > 10 << 20:
                logger.warning("cached result is %dMB (larger than 10MB)", bytes >> 20)
            # count
            f.seek(0)
            f.write(struct.pack("I", c))

 def test_pickling(self):
     """intbitset - pickling"""
     from six.moves import cPickle
     for set1 in self.sets + [[]]:
         self.assertEqual(self.intbitset(set1), cPickle.loads(cPickle.dumps(self.intbitset(set1), -1)))
     for set1 in self.sets + [[]]:
         self.assertEqual(self.intbitset(set1, trailing_bits=True), cPickle.loads(cPickle.dumps(self.intbitset(set1, trailing_bits=True), -1)))

def writePickleZip (outputFile, data, log=None) :
    '''Utility to write a pickle to disk.

       NOTE: This first attempts to use protocol 2 for greater portability 
             across python versions. If that fails it, and we're using py3,
             we attempt again with the highest protocol available. Advances in
             py3.x allow large datasets to be pickled.

       outputFile : Name of the file to write. The extension should be pkl.gz
       data       : Data to write to the file
       log        : Logger to use
    '''
    if not outputFile.endswith('pkl.gz') :
        raise Exception('The file must end in the pkl.gz extension.')
    if log is not None :
        log.info('Compressing to [' + outputFile + ']')

    # attempt to pickle with protocol 2 --
    # protocol 2 is the highest protocol supported in py2.x. If we can
    # get away with pickling with this protocol, it will provide better
    # portability across python releases.
    try :
        with gzip.open(outputFile, 'wb') as f :
            f.write(cPickle.dumps(data, protocol=2))

    # TODO: find exact error thrown while pickling large networks
    except Exception as ex :
        import sys
        # large objects cannot be pickled in py2.x, so if we're using py3.x,
        # let's attempt the pickle again with the current highest.
        if sys.version_info >= (3, 0) :
            with gzip.open(outputFile.replace('pkl', 'pkl3'), 'wb') as f :
                f.write(cPickle.dumps(data, protocol=cPickle.HIGHEST_PROTOCOL))
        else : raise ex

def _make_pickleable(fn):
    # return a pickleable function followed by a tuple of initial arguments
    # could use partial but this is more efficient
    try:
        cPickle.dumps(fn, protocol=0)
    except TypeError:
        pass
    else:
        return fn, ()
    if inspect.ismethod(fn):
        name, self_ = fn.__name__, fn.__self__
        if self_ is None:  # Python 2 unbound method
            self_ = fn.im_class
        return _invoke_method, (self_, name)
    elif inspect.isfunction(fn) and fn.__module__ in sys.modules:
        cls, name = _find_class_from_staticmethod(fn)
        if (cls, name) != (None, None):
            try:
                cPickle.dumps((cls, name), protocol=0)
            except cPickle.PicklingError:
                pass
            else:
                return _invoke_method, (cls, name)
    # Fall back to sending the source code
    return _evaluate_fn_source, (textwrap.dedent(inspect.getsource(fn)),)

    def __init__(self, data=None, pickle=True):
        """ Initialise the instance.
        """
        QtCore.QMimeData.__init__(self)

        # Keep a local reference to be returned if possible.
        self._local_instance = data

        if pickle:
            if data is not None:
                # We may not be able to pickle the data.
                try:
                    pdata = dumps(data)
                    # This format (as opposed to using a single sequence) allows
                    # the type to be extracted without unpickling the data.
                    self.setData(self.MIME_TYPE, dumps(data.__class__) + pdata)
                except (PickleError, TypeError, AttributeError):
                    # if pickle fails, still try to create a draggable
                    warnings.warn(("Could not pickle dragged object %s, " +
                            "using %s mimetype instead") % (repr(data),
                            self.NOPICKLE_MIME_TYPE), RuntimeWarning)
                    self.setData(self.NOPICKLE_MIME_TYPE, str2bytes(str(id(data))))

        else:
            self.setData(self.NOPICKLE_MIME_TYPE, str2bytes(str(id(data))))

 def test_subclass_coerce_pymimedata(self):
     md = PyMimeData(data=0)
     md2 = PMDSubclass.coerce(md)
     self.assertTrue(isinstance(md2, PMDSubclass))
     self.assertTrue(md2.hasFormat(PyMimeData.MIME_TYPE))
     self.assertFalse(md2.hasFormat(PyMimeData.NOPICKLE_MIME_TYPE))
     self.assertEqual(md2.data(PyMimeData.MIME_TYPE).data(), dumps(int)+dumps(0))

def test_run_in_runclassmethod():
    class C(object):
        @classmethod
        def fn(cls, *args, **kwargs):
            return cls, args, kwargs
    C.__name__ = 'C_' + str(uuid4()).replace('-', '_')
    C.__module__ = 'pytest_shutil.run'
    with patch('pytest_shutil.run.execnet') as execnet:
        gw = execnet.makegateway.return_value
        chan = gw.remote_exec.return_value
        chan.receive.return_value = cPickle.dumps(sentinel.ret)
        c = C()
        with patch.object(run, C.__name__, C, create=True):
            run.run_in_subprocess(c.fn, python='sentinel.python')(ARG, kw=KW)
            ((s,), _) = chan.send.call_args
            if sys.version_info < (3, 0, 0):
                # Class methods are not pickleable in Python 2.
                assert cPickle.loads(s) == (run._invoke_method, (C, 'fn', ARG), {'kw': KW})
            else:
                # Class methods are pickleable in Python 3.
                assert cPickle.loads(s) == (c.fn, (ARG,), {'kw': KW})
            ((remote_fn,), _) = gw.remote_exec.call_args
            ((chan.receive.return_value,), _) = chan.send.call_args
            remote_fn(chan)
            chan.send.assert_called_with(cPickle.dumps((C, (ARG,), {'kw': KW}), protocol=0))

def test_store_update_context():
    src.val.nested = 'nested{{src.ccdid}}'
    src.val['ccdid'] = 2
    src['srcdir'] = 'obs{{ src.obsid }}/{{src.nested}}'
    files['srcdir'] = '{{ src.srcdir }}'
    src['obsid'] = 123
    src.val['ccdid'] = 2
    src['ra'] = 1.4343256789
    src['ra'].format = '%.4f'
    files['evt2'] = 'obs{{ src.obsid }}/{{src.nested}}/acis_evt2'
    src.val.nested = 'nested{{src.ccdid}}'

    tmp = pickle.dumps(src)
    tmp2 = pickle.dumps(files)

    src.clear()
    files.clear()

    assert src['ra'].val is None
    assert files['evt2'].val is None

    src.update(pickle.loads(tmp))
    files.update(pickle.loads(tmp2))

    assert str(src['ra']) == '1.4343'
    assert str(src['srcdir']) == 'obs123/nested2'
    assert files['srcdir'].rel == 'data/obs123/nested2'
    assert files.rel.srcdir == 'data/obs123/nested2'
    assert str(files['evt2.fits']) == 'data/obs123/nested2/acis_evt2.fits'

 def test_pickle(self):
     md = PyMimeData(data=0)
     self.assertEqual(md._local_instance, 0)
     self.assertTrue(md.hasFormat(PyMimeData.MIME_TYPE))
     self.assertFalse(md.hasFormat(PyMimeData.NOPICKLE_MIME_TYPE))
     self.assertEqual(md.data(PyMimeData.MIME_TYPE).data(),
                      dumps(int)+dumps(0))

def hash_update(M, elems):
    '''
    M = hash_update(M, elems)

    Update the hash object ``M`` with the sequence ``elems``.

    Parameters
    ----------
    M : hashlib object
        An object on which the update method will be called
    elems : sequence of 2-tuples

    Returns
    -------
    M : hashlib object
        This is the same object as the argument
    '''
    from six.moves import cPickle as pickle
    from six.moves import map
    import six

    try:
        import numpy as np
    except ImportError:
        np = None
    for n,e in elems:
        M.update(pickle.dumps(n))
        if hasattr(e, '__jug_hash__'):
            M.update(e.__jug_hash__())
        elif type(e) in (list, tuple):
            M.update(repr(type(e)).encode('utf-8'))
            hash_update(M, enumerate(e))
        elif type(e) == set:
            M.update('set')
            # With randomized hashing, different runs of Python might result in
            # different orders, so sort. We cannot trust that all the elements
            # in the set will be comparable, so we convert them to their hashes
            # beforehand.
            items = list(map(hash_one, e))
            items.sort()
            hash_update(M, enumerate(items))
        elif type(e) == dict:
            M.update(six.b('dict'))
            items = [(hash_one(k),v) for k,v in e.items()]
            items.sort(key=(lambda k_v:k_v[0]))

            hash_update(M, items)
        elif np is not None and type(e) == np.ndarray:
            M.update(six.b('np.ndarray'))
            M.update(pickle.dumps(e.dtype))
            M.update(pickle.dumps(e.shape))
            try:
                buffer = e.data
                M.update(buffer)
            except:
                M.update(e.copy().data)
        else:
            M.update(pickle.dumps(e))
    return M

 def test_coerce_list_pymimedata(self):
     md = PyMimeData(data=0)
     md2 = PyMimeData.coerce([md])
     self.assertEqual(md2._local_instance, [0])
     self.assertTrue(md2.hasFormat(PyMimeData.MIME_TYPE))
     self.assertFalse(md2.hasFormat(PyMimeData.NOPICKLE_MIME_TYPE))
     self.assertEqual(md2.data(PyMimeData.MIME_TYPE).data(),
                      dumps(list)+dumps([0]))

    def test_get(self):
        #mock the pick and set it to the data variable
        test_pickle = pickle.dumps(
            {pickle.dumps(self.test_key): self.test_value}, protocol=2)
        self.test_cache.data = pickle.loads(test_pickle)

        #assert
        self.assertEquals(self.test_cache.get(self.test_key), self.test_value)
        self.assertEquals(self.test_cache.get(self.bad_key), None)

 def test_get(self):
     env = Envelope('[email protected]', ['[email protected]', '[email protected]'])
     envelope_raw = cPickle.dumps(env)
     delivered_indexes_raw = cPickle.dumps([0])
     self.storage.redis.hmget('test:asdf', 'envelope', 'attempts', 'delivered_indexes').AndReturn((envelope_raw, 13, delivered_indexes_raw))
     self.mox.ReplayAll()
     get_env, attempts = self.storage.get('asdf')
     self.assertEqual('[email protected]', get_env.sender)
     self.assertEqual(['[email protected]'], get_env.recipients)
     self.assertEqual(13, attempts)

    def test_serialization(self):
        e = smqtk.representation.classification_element.memory\
            .MemoryClassificationElement('test', 0)

        e2 = cPickle.loads(cPickle.dumps(e))
        self.assertEqual(e, e2)

        e.set_classification(a=0, b=1)
        e2 = cPickle.loads(cPickle.dumps(e))
        self.assertEqual(e, e2)

 def _encodeMetadata(self, metadata):
     # metadata format is:
     #    - first line with trailing \x00: comment or empty comment
     #    - then: pickled metadata (incl. comment)
     try:
         comment = metadata['sys_metadata']['comment']
         comment = dumps(comment)
     except KeyError:
         comment = ''
     return b'\x00\n'.join((comment, dumps(metadata, HIGHEST_PROTOCOL)))

def assert_pickle_idempotent(obj):
    '''Assert that obj does not change (w.r.t. ==) under repeated picklings
    '''
    from six.moves.cPickle import dumps, loads
    obj1 = loads(dumps(obj))
    obj2 = loads(dumps(obj1))
    obj3 = loads(dumps(obj2))
    assert_equivalent(obj, obj1)
    assert_equivalent(obj, obj2)
    assert_equivalent(obj, obj3)
    assert type(obj) is type(obj3)

def save_sesh(dict_objs, file='skrfSesh.p', module='skrf', exclude_prefix='_'):
    '''
    Save all `skrf` objects in the local namespace.

    This is used to save current workspace in a hurry, by passing it the
    output of :func:`locals` (see Examples). Note this can be
    used for other modules as well by passing a different `module` name.

    Parameters
    ------------
    dict_objs : dict
        dictionary containing `skrf` objects. See the Example.
    file : str or file-object, optional
        the file to save all objects to
    module : str, optional
        the module name to grep for.
    exclude_prefix: str, optional
        dont save objects which have this as a prefix.

    See Also
    ----------
    read : read a skrf object
    write : write skrf object[s]
    read_all : read all skrf objects in a directory
    write_all : write dictionary of skrf objects to a directory


    Examples
    ---------
    Write out all skrf objects in current namespace.

    >>> rf.write_all(locals(), 'mysesh.p')


    '''
    objects = {}
    print('pickling: ')
    for k in dict_objs:
        try:
            if module  in inspect.getmodule(dict_objs[k]).__name__:
                try:
                    pickle.dumps(dict_objs[k])
                    if k[0] != '_':
                        objects[k] = dict_objs[k]
                        print(k+', ')
                finally:
                    pass

        except(AttributeError, TypeError):
            pass
    if len (objects ) == 0:
        print('nothing')

    write(file, objects)

 def setUp(self):
     """Set up hopefully unique universes."""
     # _n marks named universes/atomgroups/pickled strings
     self.universe = mda.Universe(PDB_small, PDB_small, PDB_small)
     self.universe_n = mda.Universe(PDB_small, PDB_small, PDB_small,
                                    anchor_name="test1")
     self.ag = self.universe.atoms[:20]  # prototypical AtomGroup
     self.ag_n = self.universe_n.atoms[:10]
     self.pickle_str = cPickle.dumps(self.ag,
                                     protocol=cPickle.HIGHEST_PROTOCOL)
     self.pickle_str_n = cPickle.dumps(self.ag_n,
                                       protocol=cPickle.HIGHEST_PROTOCOL)