def loop(self, csrcn):
        self.csrcn = csrcn
        self.setup_screen()

        sslctx = ssl.SSLContext(protocol=ssl.PROTOCOL_TLS)
        sslctx.load_cert_chain('certificates/client.crt', 'certificates/client.key')
        sslctx.load_verify_locations('certificates/ca.crt')

        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        ssl_sock = sslctx.wrap_socket(sock)
        ssl_sock.connect(('127.0.0.1', 6161))
        msgpack.pack(['CONNECT', 'brokermon'], ssl_sock)
        msgpack.pack(['LOGMON'], ssl_sock)
        unpacker = msgpack.Unpacker(raw=False)

        while True:
            data = ssl_sock.read(10000)
            if not data:
                return
            unpacker.feed(data)
            for pkt in unpacker:
                if pkt[0] == 'PING':
                    self.csrcn.insstr(8, 7, '{:15}'.format(pkt[1]))
                elif pkt[0] == 'PONG':
                    self.csrcn.insstr(9, 7, '{:15}'.format(pkt[1]))
                elif pkt[0] == 'TICKER':
                    self.csrcn.insstr(7, 7, pkt[1])
                elif pkt[0] == 'LOG':
                    self.log_print(pkt[1], pkt[2])
            self.csrcn.refresh()

def QueueSender(reader: asyncio, writer: asyncio.StreamWriter, queue_name: str):
    """
    A coroutine for pulling items from the Queue to the streams.
    """
    client = writer.get_extra_info("peername")
    sclient = ':'.join(str(_) for _ in client)
    while True:
        try:
            data = yield from reader.read(65536)
        except ConnectionResetError:
            rlogger.info("Client {} closed connection".format(sclient))
            return
        if not data:
            slogger.info("Client {} closed connection".format(sclient))
            return
        # Unpack data
        try:
            sub_data = msgpack.unpackb(data, encoding='utf-8')
        except (msgpack.UnpackException, ValueError) as e:
            slogger.error("Recieved non-msgpack pull from {}".format(sclient))
            continue
        action = sub_data.get("action", -1)
        if not action == 1:
            slogger.error("Recieved non-pull action on pull channel from client (action: {})"
                          .format(sclient, action))
            continue
        queue = queues[queue_name][1]
        assert isinstance(queue, asyncio.Queue)
        data = yield from queue.get()
        slogger.debug("Packing data {} for queue {}".format(data[1], queue_name))
        response = {"status": 0, "data": data[1], "msgnum": data[0]}
        msgpack.pack(response, writer)

 def commit(self):
     """Commit transaction
     """
     if not self.txn_active:
         return
     if self.files is not None:
         ttl = int(os.environ.get('BORG_FILES_CACHE_TTL', 20))
         with SaveFile(os.path.join(self.path, 'files'), binary=True) as fd:
             for path_hash, item in self.files.items():
                 # Only keep files seen in this backup that are older than newest mtime seen in this backup -
                 # this is to avoid issues with filesystem snapshots and mtime granularity.
                 # Also keep files from older backups that have not reached BORG_FILES_CACHE_TTL yet.
                 entry = FileCacheEntry(*msgpack.unpackb(item))
                 if entry.age == 0 and bigint_to_int(entry.mtime) < self._newest_mtime or \
                    entry.age > 0 and entry.age < ttl:
                     msgpack.pack((path_hash, entry), fd)
     self.config.set('cache', 'manifest', self.manifest.id_str)
     self.config.set('cache', 'timestamp', self.manifest.timestamp)
     self.config.set('cache', 'key_type', str(self.key.TYPE))
     self.config.set('cache', 'previous_location', self.repository._location.canonical_path())
     with SaveFile(os.path.join(self.path, 'config')) as fd:
         self.config.write(fd)
     self.chunks.write(os.path.join(self.path, 'chunks').encode('utf-8'))
     os.rename(os.path.join(self.path, 'txn.active'),
               os.path.join(self.path, 'txn.tmp'))
     shutil.rmtree(os.path.join(self.path, 'txn.tmp'))
     self.txn_active = False

 def write_index(self):
     hints = {b'version': 2,
              b'segments': self.segments,
              b'compact': self.compact}
     transaction_id = self.io.get_segments_transaction_id()
     hints_file = os.path.join(self.path, 'hints.%d' % transaction_id)
     with open(hints_file + '.tmp', 'wb') as fd:
         msgpack.pack(hints, fd)
         fd.flush()
         os.fsync(fd.fileno())
     os.rename(hints_file + '.tmp', hints_file)
     self.index.write(os.path.join(self.path, 'index.tmp'))
     os.rename(os.path.join(self.path, 'index.tmp'),
               os.path.join(self.path, 'index.%d' % transaction_id))
     if self.append_only:
         with open(os.path.join(self.path, 'transactions'), 'a') as log:
             print('transaction %d, UTC time %s' % (transaction_id, datetime.utcnow().isoformat()), file=log)
     # Remove old auxiliary files
     current = '.%d' % transaction_id
     for name in os.listdir(self.path):
         if not name.startswith(('index.', 'hints.')):
             continue
         if name.endswith(current):
             continue
         os.unlink(os.path.join(self.path, name))
     self.index = None

def set_cache(key, val):
    path = _path_from_key(key)
    dirname = os.path.dirname(path)
    mkdirp(dirname)
    log.info("Saving cache to {0}".format(dirname))
    with open(path, 'w') as f:
        msgpack.pack(val, f)

 def commit(self):
     """Commit transaction
     """
     if not self.txn_active:
         return
     self.security_manager.save(self.manifest, self.key)
     pi = ProgressIndicatorMessage(msgid='cache.commit')
     if self.files is not None:
         if self._newest_cmtime is None:
             # was never set because no files were modified/added
             self._newest_cmtime = 2 ** 63 - 1  # nanoseconds, good until y2262
         ttl = int(os.environ.get('BORG_FILES_CACHE_TTL', 20))
         pi.output('Saving files cache')
         with IntegrityCheckedFile(path=os.path.join(self.path, 'files'), write=True) as fd:
             for path_hash, item in self.files.items():
                 # Only keep files seen in this backup that are older than newest cmtime seen in this backup -
                 # this is to avoid issues with filesystem snapshots and cmtime granularity.
                 # Also keep files from older backups that have not reached BORG_FILES_CACHE_TTL yet.
                 entry = FileCacheEntry(*msgpack.unpackb(item))
                 if entry.age == 0 and bigint_to_int(entry.cmtime) < self._newest_cmtime or \
                    entry.age > 0 and entry.age < ttl:
                     msgpack.pack((path_hash, entry), fd)
         self.cache_config.integrity['files'] = fd.integrity_data
     pi.output('Saving chunks cache')
     with IntegrityCheckedFile(path=os.path.join(self.path, 'chunks'), write=True) as fd:
         self.chunks.write(fd)
     self.cache_config.integrity['chunks'] = fd.integrity_data
     pi.output('Saving cache config')
     self.cache_config.save(self.manifest, self.key)
     os.rename(os.path.join(self.path, 'txn.active'),
               os.path.join(self.path, 'txn.tmp'))
     shutil.rmtree(os.path.join(self.path, 'txn.tmp'))
     self.txn_active = False
     pi.finish()

    def store_to_stream(self, stream):
        """ Stores the Safe to `stream'.

            This is done automatically if opened with `open'. """
        start_time = time.time()
        l.debug("Packing ...")
        stream.write(SAFE_MAGIC)
        msgpack.pack(self.data, stream)
        l.debug(" packed in %.2fs", time.time() - start_time)

def create_random_data(dataset, n_rep=300):
    import shazoo_exps as se
    data = {}
    g_adj, g_ew, gold_signs, phi = se.load_real_graph(dataset)
    g_adj = {int(u): {int(v) for v in adj} for u, adj in g_adj.items()}
    n = len(g_adj)
    nodes = list((range(n)))
    gold = np.array([gold_signs[u] for u in nodes])
    inv_ew = {(int(e[0]), int(e[1])): 1/w for e, w in se.sz.iteritems(g_ew)}
    g_ew = {(int(e[0]), int(e[1])): w for e, w in se.sz.iteritems(g_ew)}

    rst = []
    for _ in range(n_rep):
        se.sz.GRAPH, se.sz.EWEIGHTS = g_adj, g_ew
        se.sz.get_rst(None)
        adj, ew = se.sz.TREE_ADJ, se.sz.TWEIGHTS
        rst.append(tuple(set(ew)))
    data['rst'] = tuple(rst)

    trees = []
    for i in range(n_rep):
        adj, ew = se.get_mst(inv_ew)
        trees.append(tuple(set(ew)))
        if i == 2:
            res = []
            for j, s in enumerate(trees):
                for t in trees[j+1:]:
                    res.append(set(s) == set(t))
            if any(res):
                break
    data['mst'] = tuple(trees)

    nodes_order = []
    for _ in range(n_rep):
        random.shuffle(nodes)
        nodes_order.append(tuple(nodes))
    data['nodes_order'] = tuple(nodes_order)

    batch_order = []
    for ts in train_size:
        level = []
        max_index = int(ts*n)
        indices = list(range(max_index))
        for _ in range(n_rep):
            random.shuffle(indices)
            level.append(tuple(indices))
        batch_order.append(tuple(level))
    data['batch_order'] = tuple(batch_order)

    ones = np.ones(n, dtype=int)
    changed_signs = []
    for ts in pertubations:
        changed_signs.append(create_random_perturbations(ts, ones, nodes, gold, n_rep))
    data['changed_signs'] = tuple(changed_signs)

    with open('{}.random'.format(dataset), 'w+b') as outfile:
        msgpack.pack(data, outfile)

def pack(response, io):
    if isinstance(response, types.StringTypes):
        io.write(response)
    elif isinstance(response, dict):
        msgpack.pack(response, io)
    elif isinstance(response, types.GeneratorType) or isinstance(response, Iterable):
        [pack(chunk, io) for chunk in response]
    elif response is not None:
        msgpack.pack(response, io)

def generate_msgpack():
    """
    Generate 'postcodes_X.mp' msg pack files, this is a utility and shouldn't be required as files are included in the
    repo.

    To use it you need to download a full list of uk postcodes csv file from
    from http://www.freemaptools.com/download-uk-postcode-lat-lng.htm
    and http://www.doogal.co.uk/UKPostcodes.php
    """
    # loaded locally as only required here
    from math import radians, sin, cos, sqrt, asin
    import csv

    def haversine(lat1, lon1, lat2, lon2):
        R = 6372.8 * 1000 # Earth radius in meters
        dLat = radians(lat2 - lat1)
        dLon = radians(lon2 - lon1)
        lat1 = radians(lat1)
        lat2 = radians(lat2)
        a = sin(dLat/2)**2 + cos(lat1)*cos(lat2)*sin(dLon/2)**2
        c = 2*asin(sqrt(a))
        return R * c

    all_pcs = []
    with open('freemaptools_postcodes.csv', 'rb') as f:
        csv_reader = csv.reader(f)
        next(csv_reader)  # heading
        for i, row in enumerate(csv_reader):
            pc = row[1]
            pc = pc.lower().replace(' ', '')
            lat = float(row[2])
            lng = float(row[3])
            all_pcs.append((pc, lat, lng))
    with open('doogle_postcodes.csv', 'rb') as f:
        csv_reader = csv.DictReader(f)
        for i, row in enumerate(csv_reader):
            if row['Terminated']:
                continue
            pc = row['Postcode'].lower().replace(' ', '')
            lat = float(row['Latitude'])
            lng = float(row['Longitude'])
            all_pcs.append((pc, lat, lng))

    pcs1 = {}
    pcs2 = {}
    for pc, lat, lng in all_pcs:
        error = haversine(lat, lng, round(lat, 3), round(lng, 3))
        assert error < 100
        if pc[0] in FILE_1_PREF:
            pcs1[pc] = '%0.3f %0.3f' % (lat - 49.5, lng + 8.5)
        else:
            pcs2[pc] = '%0.3f %0.3f' % (lat - 49.5, lng + 8.5)
    msgpack.pack(pcs1, open(PC_FILE1, 'wb'))
    msgpack.pack(pcs2, open(PC_FILE2, 'wb'))
    print 'saved %d and %d postcodes to %s and %s respectively' % (len(pcs1), len(pcs2), PC_FILE1, PC_FILE2)

 def write(self, out):
   if self.version != 1:
     msgpack.pack(self.version, out)
   msgpack.pack(self.klasses, out)
   msgpack.pack(self.stores, out)
   doc = self.doc_packed
   msgpack.pack(len(doc), out)
   out.write(doc)
   for insts in self.instances_packed:
     msgpack.pack(len(insts), out)
     out.write(insts)

 def send_multipart(self, target, *args):
     if type(target) is str:
         target = self.servers[target]
     #safe_args = [(a.encode('utf-8') if type(a) is str else a) for a in args]
     self.set_indent('{:12} <- {:6} '.format(target.ident, args[0]))
     if (args[0] == 'PING'):
         if self.mon:
             msgpack.pack(['PING', target.ident], self.mon.writer)
     else:
         self.log(target.ident, repr(args))
     msgpack.pack(args, target.writer)

 def test_unknown_integrity_version(self):
     # For now an unknown integrity data version is ignored and not an error.
     integrity_path = os.path.join(self.repository.path, 'integrity.1')
     with open(integrity_path, 'r+b') as fd:
         msgpack.pack({
             # Borg only understands version 2
             b'version': 4.7,
         }, fd)
         fd.truncate()
     with self.repository:
         # No issues accessing the repository
         assert len(self.repository) == 1
         assert self.repository.get(H(0)) == b'foo'

def _parse_mirteFile(path):
    """ Open and parses the mirteFile at <path>. """
    cache_path = os.path.join(os.path.dirname(path),
                CACHE_FILENAME_TEMPLATE % os.path.basename(path))
    if (os.path.exists(cache_path) and
                os.path.getmtime(cache_path) >= os.path.getmtime(path)):
        with open(cache_path) as f:
            return msgpack.unpack(f)
    with open(path) as f:
        ret = yaml.load(f)
    with open(cache_path, 'w') as f:
        msgpack.pack(ret, f)
    return ret

    def save(self, filename):
        tfn = '%s.inprog-%d' % (filename, random.randint(1, 10000000))
        fh = open(tfn, 'wb')

        try:
            me_as_dict = self.todict()
            msgpack.pack(me_as_dict, encoding='utf-8', stream=fh)
        finally:
            fh.close()

            if os.path.exists(filename):
                os.rename(filename, '%s.bak' % filename)
            os.rename(tfn, filename)

 def _do_serialize(self):
     if self.__magic__ is None or self.__fields__ is None:
         raise RuntimeError(
             "Serialization can only be performed on classes implementing "
             "__fields__ and __magic__")
     buf = io.BytesIO()
     msgpack.pack(PROTOCOL_VERSION, buf)
     msgpack.pack(self.__magic__, buf)
     for name, type_info in self.__fields__:
         value = getattr(self, name)
         type_info.validate(value, name)
         type_info.pack(value, buf)
     return buf

def save_object(object_, file_path):

    def ndarrray_to_list(o, _warned=[False]): # Use a mutlable default arg to hold a fn interal temp var.
        if isinstance(o, np.ndarray):
            if not _warned[0]:
                logger.warning("numpy array will be serialized as list. Invoked at:\n"+''.join(tb.format_stack()))
                _warned[0] = True
            return o.tolist()
        return o

    file_path = os.path.expanduser(file_path)
    with open(file_path, 'wb') as fh:
        msgpack.pack(object_, fh, use_bin_type=True,default=ndarrray_to_list)

def send_packet(packet, dev_id):
    (length, data, timestamp) = packet
    if length > 20000:
        print "WARNING: Large packet: {0} (sniffing loop?)".format(length)
    message = {"ln": length, "ts": timestamp, "if": devs[dev_id]["eth"], "id": dev_id, "dt": data}
    clients_copy = clients
    for client in clients_copy:
        try:
            msgpack.pack(message, client)
            client.flush()
        except eventlet.green.socket.error:
            print "client disconnected (fd: {0})".format(client.fileno())
            clients.remove(client)

    def hashFile(self, dir_inner_path, file_relative_path, optional=False):
        inner_path = dir_inner_path + file_relative_path

        file_size = self.site.storage.getSize(inner_path)
        # Only care about optional files >1MB
        if not optional or file_size < 1 * 1024 * 1024:
            return super(ContentManagerPlugin, self).hashFile(dir_inner_path, file_relative_path, optional)

        back = {}
        content = self.contents.get(dir_inner_path + "content.json")

        hash = None
        piecemap_relative_path = None
        piece_size = None

        # Don't re-hash if it's already in content.json
        if content and file_relative_path in content.get("files_optional", {}):
            file_node = content["files_optional"][file_relative_path]
            if file_node["size"] == file_size:
                self.log.info("- [SAME SIZE] %s" % file_relative_path)
                hash = file_node.get("sha512")
                piecemap_relative_path = file_node.get("piecemap")
                piece_size = file_node.get("piece_size")

        if not hash or not piecemap_relative_path:  # Not in content.json yet
            if file_size < 5 * 1024 * 1024:  # Don't create piecemap automatically for files smaller than 5MB
                return super(ContentManagerPlugin, self).hashFile(dir_inner_path, file_relative_path, optional)

            self.log.info("- [HASHING] %s" % file_relative_path)
            merkle_root, piece_size, piecemap_info = self.hashBigfile(self.site.storage.open(inner_path, "rb"), file_size)
            if not hash:
                hash = merkle_root

            if not piecemap_relative_path:
                file_name = helper.getFilename(file_relative_path)
                piecemap_relative_path = file_relative_path + ".piecemap.msgpack"
                piecemap_inner_path = inner_path + ".piecemap.msgpack"

                msgpack.pack({file_name: piecemap_info}, self.site.storage.open(piecemap_inner_path, "wb"))

                back.update(super(ContentManagerPlugin, self).hashFile(dir_inner_path, piecemap_relative_path, optional=True))

        piece_num = int(math.ceil(float(file_size) / piece_size))

        # Add the merkle root to hashfield
        hash_id = self.site.content_manager.hashfield.getHashId(hash)
        self.optionalDownloaded(inner_path, hash_id, file_size, own=True)
        self.site.storage.piecefields[hash].fromstring("1" * piece_num)

        back[file_relative_path] = {"sha512": hash, "size": file_size, "piecemap": piecemap_relative_path, "piece_size": piece_size}
        return back

 def pack(stream):
     """Wraps pack of msgpack."""
     return msgpack.pack(
         stream,
         encoding="UTF-8",
         use_bin_type=True
     )