def Deserializer(stream_or_string):
    if isinstance(stream_or_string, basestring):
        stream = StringIO(stream_or_string)
    else:
        stream = stream_or_string
    for obj in PythonDeserializer(msgpack.unpack(stream, object_hook=DjangoMsgPackDecoder().decode)):
        yield obj

    def _decode(self, response):
        ct = response.getheader('Content-Type')
        ce = response.getheader('Content-Encoding')
        cl = response.getheader('Content-Length')

        if cl and not int(cl):
            LOG.debug("Empty response body")
            return ""

        if ce and ce == "gzip":
            # this reads the whole response in memory, but json.load() would do the same anyway...
            data = response.read()
            response = gzip.GzipFile(fileobj=StringIO(data))

        if ct and 'application/json' in ct:
            LOG.debug("decoding %s", ct)
            return json.load(response)
        elif ct and 'application/x-msgpack' in ct:
            if msgpack:
                LOG.debug("decoding %s", ct)
                return msgpack.unpack(response)
            else:
                LOG.debug("not decoding %s, decoder is unavailable", ct)
                return response.read()

        return response.read()

 def getPiecemap(self, inner_path):
     file_info = self.site.content_manager.getFileInfo(inner_path)
     piecemap_inner_path = helper.getDirname(file_info["content_inner_path"]) + file_info["piecemap"]
     self.site.needFile(piecemap_inner_path, priority=20)
     piecemap = msgpack.unpack(self.site.storage.open(piecemap_inner_path))[helper.getFilename(inner_path)]
     piecemap["piece_size"] = file_info["piece_size"]
     return piecemap

 def fetch_todays_schedule(self, rooms):
     """ Fetch the schedules for the given rooms. """
     room_ids = self.fetch_room_ids(rooms)
     inst_id = self.fetch_inst_id()
     ret = {}
     now = datetime.datetime.now()
     day = DAYS[now.isoweekday() - 1]
     for room_name in room_ids:
         ret[room_name] = []
         events = msgpack.unpack(urllib2.urlopen(
             "%s/snapshot/head/%s/location/%s?format=msgpack" % (
                 self.url, inst_id, room_ids[room_name])))['events']
         for event in events:
             starttime = datetime.datetime.strptime(
                     event['starttime'], '%H:%M:%S').time()
             endtime = datetime.datetime.strptime(
                     event['endtime'], '%H:%M:%S').time()
             if event['day'] != day:
                 continue
             ok = False
             for period in event['eventperiods']:
                 startdate = datetime.datetime.strptime(
                         period['startdate'], '%Y-%m-%d %H:%M:%SZ').date()
                 enddate = datetime.datetime.strptime(
                         period['enddate'], '%Y-%m-%d %H:%M:%SZ').date()
                 if (startdate <= now.date() and now.date() <= enddate):
                     ok = True
                     break
             if not ok:
                 continue
             ret[room_name].append((starttime, endtime,
                     event['course']['name']))
     return ret

 def fetch_inst_id(self):
     """ Fetches the institute id of the RU """
     for d in msgpack.unpack(urllib2.urlopen(
             "%s/list/institutes?format=msgpack" % self.url)):
         if d['name'] == 'Radboud Universiteit Nijmegen':
             return d['id']
     assert False

  def __init__(self, data_path, use_msgpack=False):
    self.net_as = SubnetTree.SubnetTree()
    self.d_asn = {}
    self.net_as_map = {}
    net_as_raw = None

    net_as_file = open(data_path, 'rb')
    if use_msgpack:
      import msgpack
      net_as_raw = msgpack.unpack(net_as_file)
    else:
      net_as_raw = json.load(net_as_file)
    net_as_file.close()

    for asn, v in net_as_raw.items():
      '''
       "11542": {
          "name": "EYEMG - EYEMG - interactive media group", "cc": "US", "timestamp": "20070524", "rir": "arin",
          "nets": {"208.79.156.0/22": {"cc": "US", "timestamp": "20070621", "rir": "arin"}
                  },
          "asn": "11542"}
      '''
      self.d_asn[asn] = ASN(asn)
      self.d_asn[asn].fromdict(v)

      for net in self.d_asn[asn].nets.keys():
        '''A single net may be announced in various ASNs'''
        self.net_as.insert(str(net), net)

        if not net in self.net_as_map:
          self.net_as_map[net] = []

        if not asn in self.net_as_map[net]:
          self.net_as_map[net].append(asn)

    def _rpc_all(self, cmd):
        clients = {}
        poller = zmq.Poller()

        ret = []
        for k, d in self.workers.items():
            socket = self.context.socket(zmq.REQ)
            socket.setsockopt(zmq.LINGER, 0)
            socket.connect(d['rpc_address'])
            poller.register(socket, zmq.POLLIN)
            socket.send(cmd)
            clients[socket] = k

        t = time.time()
        while time.time() - t < 1.000:
            socks = dict(poller.poll(50)) # wait 0.05s
            for socket in socks.keys():
                status = unpack(socket.recv())
                self.workers[clients[socket]] = status
                ret.append(status)
                del clients[socket]
            if not clients:
                break

        # timeouted sockets
        for k, socket in clients.items():
            status = {"status":"no response"} 
            if k in self.workers:
                status.update( self.workers[k] )
            ret.append(status)

        return ret

 def test_1chain_simplerestore(self):
     m = andrey.Markov(1, 1)
     m.teach('alpha beta conky delta')
     dump = msgpack.dumps(m.todict())
     dic = msgpack.unpack(io.BytesIO(dump), encoding='utf-8')
     m2 = andrey.Markov.fromdict(dic)
     self.assertEqual('beta conky', m2.choose('alpha', continued=1))

 def _wait(self, unpacker, sock):
     while True:
         data = sock.read()
         unpacker.feed(data)
         try:
             return msgpack.unpack()
         except StopIteration:
             pass

	def load(self):
		self.data = {'version' : self._version}
		if os.path.exists(self.db_path):
			with open(self.db_path) as f:
				self.data = msgpack.unpack(f)
		else:
			print 'Initializing replay database...'
			self.update()

 def prepare_txn(self, transaction_id, do_cleanup=True):
     self._active_txn = True
     if not self.lock.got_exclusive_lock():
         if self.exclusive is not None:
             # self.exclusive is either True or False, thus a new client is active here.
             # if it is False and we get here, the caller did not use exclusive=True although
             # it is needed for a write operation. if it is True and we get here, something else
             # went very wrong, because we should have a exclusive lock, but we don't.
             raise AssertionError("bug in code, exclusive lock should exist here")
         # if we are here, this is an old client talking to a new server (expecting lock upgrade).
         # or we are replaying segments and might need a lock upgrade for that.
         try:
             self.lock.upgrade()
         except (LockError, LockErrorT):
             # if upgrading the lock to exclusive fails, we do not have an
             # active transaction. this is important for "serve" mode, where
             # the repository instance lives on - even if exceptions happened.
             self._active_txn = False
             raise
     if not self.index or transaction_id is None:
         try:
             self.index = self.open_index(transaction_id, False)
         except RuntimeError:
             self.check_transaction()
             self.index = self.open_index(transaction_id, False)
     if transaction_id is None:
         self.segments = {}  # XXX bad name: usage_count_of_segment_x = self.segments[x]
         self.compact = FreeSpace()  # XXX bad name: freeable_space_of_segment_x = self.compact[x]
     else:
         if do_cleanup:
             self.io.cleanup(transaction_id)
         hints_path = os.path.join(self.path, 'hints.%d' % transaction_id)
         index_path = os.path.join(self.path, 'index.%d' % transaction_id)
         try:
             with open(hints_path, 'rb') as fd:
                 hints = msgpack.unpack(fd)
         except (msgpack.UnpackException, msgpack.ExtraData, FileNotFoundError) as e:
             logger.warning('Repository hints file missing or corrupted, trying to recover')
             if not isinstance(e, FileNotFoundError):
                 os.unlink(hints_path)
             # index must exist at this point
             os.unlink(index_path)
             self.check_transaction()
             self.prepare_txn(transaction_id)
             return
         if hints[b'version'] == 1:
             logger.debug('Upgrading from v1 hints.%d', transaction_id)
             self.segments = hints[b'segments']
             self.compact = FreeSpace()
             for segment in sorted(hints[b'compact']):
                 logger.debug('Rebuilding sparse info for segment %d', segment)
                 self._rebuild_sparse(segment)
             logger.debug('Upgrade to v2 hints complete')
         elif hints[b'version'] != 2:
             raise ValueError('Unknown hints file version: %d' % hints[b'version'])
         else:
             self.segments = hints[b'segments']
             self.compact = FreeSpace(hints[b'compact'])

 def fetch_inst_id(self):
     """ Fetches the institute id of the RU """
     try:
         for d in msgpack.unpack(urllib2.urlopen(
                 "%s/list/institutes?format=msgpack" % self.url)):
             if d['name'] == 'Radboud Universiteit Nijmegen':
                 return d['id']
     except IOError, e: # urllib2 exceptions are a subclass of IOError
         raise RuusterError(e)

 def __iter__(self):
     while True:
         try:
             pos = self.buf.tell()
             yield unpack(self.buf)
         except umsgpack.InsufficientDataException:
             self.buf.seek(pos)
             self.buf = io.BytesIO(self.buf.read())
             raise StopIteration

def load_clips():
    """
    Load previous clips from DATA_FILE
    """
    try:
        with open(DATA_FILE, 'r') as f:
            return msgpack.unpack(f, encoding='utf-8')
    except IOError:
        return {}

def load_graph(filename):
    with open(filename, 'r+b') as outfile:
        psi, rV, phi, rE = msgpack.unpack(outfile, use_list=False)
    G, E = {}, {}
    for u, v, s in zip(rE[::3], rE[1::3], rE[2::3]):
        E[(u, v)] = bool(s)
        pg.add_edge(G, u, v)
    nodes_sign = list(rV)
    return psi, phi, nodes_sign, G, E

    def __init__(self, data):
        if isinstance(data, bytes):
            if data[:2] != b'\x1f\x8b': # gzip magic number
                self._data = msgpack.unpackb(data)
                return

            import io, gzip
            data = gzip.GzipFile(fileobj=io.BytesIO(data))

        self._data = msgpack.unpack(data)

 def fetch_room_ids(self, names):
     """ Fetches the ids of the rooms with the given names """
     ret = {}
     names_set = set(names)
     for d in msgpack.unpack(urllib2.urlopen(
             "%s/list/locations?format=msgpack" % self.url)):
         name = d['name'].upper() # normalize: Hg -> HG
         if name in names_set:
             ret[name] = d['id']
     return ret

    def load_from_stream(self, stream, to_container, **opts):
        """
        Load JSON config from given stream `stream`.

        :param stream: Stream will provide MessagePack-ed config content string
        :param to_container: callble to make a container object
        :param opts: keyword options passed to `msgpack.unpack`

        :return: Dict-like object holding configuration
        """
        return msgpack.unpack(stream, object_hook=to_container, **opts)

 def request(self, request):
     """ processing request
 
     {"op":"all_stats"} -> a dict of stats
     """
     response = {}
     response.update({"status":"ok"})
     try:
         req = unpack(request)
     except Exception, e:
         response.update({"status":"error","message":e.message})

def load_msgpack_gc(_file):
    output = open(_file, 'rb')

    # disable garbage collector
    gc.disable()

    mydict = msgpack.unpack(output)

    # enable garbage collector again
    gc.enable()
    output.close()
    return mydict