def kill_workers(self, timeout=5):
        """
        Send a suicide message to all workers, with some kind of timeout.
        """
        logging.info('Killing workers, taking up to %d seconds.', int(timeout))
        poller = zmq.Poller()
        poller.register(self.results_pull, zmq.POLLIN)

        while True:
            # Seems to get stuck gevent-blocking in the work_push.send() after
            # all the workers have died.  Also, gevent.Timeout() doesn't seem
            # to work here?!
            signal.alarm(int(timeout))
            self.work_push.send(msgpack.dumps([{'type': 'PING'}]))
            socks = dict(poller.poll(timeout * 1500))
            if self.results_pull in socks \
                    and socks[self.results_pull] == zmq.POLLIN:
                result_packed = self.results_pull.recv()
                result = msgpack.loads(result_packed)
                logging.info('Heard from worker id=%d; sending SUICIDE',
                            result['worker_id'])
                self.work_push.send(msgpack.dumps([{'type': 'SUICIDE'}]))
                gevent.sleep(0.1)
            else:
                break
            signal.alarm(0)

 def post_row(self, row, params, files):
     if files:
         bottle.abort(400)
     params = {k: base64.b64decode(v) for k, v in params.items()}
     action = params['action']
     with thrift_lock() as thrift:
         manager = PicarusManager(thrift=thrift)
         print(params)
         model_key = params['model']
         print('ModelKey[%r]' % model_key)
         # TODO: Allow io/ so that we can write back to the image too
         if action == 'i/link':
             self._row_validate(row, 'r')
             # TODO: Get this directly from model
             chain_input, model_link = _takeout_input_model_link_from_key(manager, model_key)
             binary_input = thrift.get(self.table, row, chain_input)[0].value  # TODO: Check val
             model = picarus_takeout.ModelChain(msgpack.dumps([model_link]))
             bottle.response.headers["Content-type"] = "application/json"
             return json.dumps({base64.b64encode(params['model']): base64.b64encode(model.process_binary(binary_input))})
         elif action == 'i/chain':
             self._row_validate(row, 'r')
             # TODO: Get this directly from model
             chain_inputs, model_chain = zip(*_takeout_input_model_chain_from_key(manager, model_key))
             binary_input = thrift.get(self.table, row, chain_inputs[0])[0].value  # TODO: Check val
             model_chain = list(model_chain)
             model = picarus_takeout.ModelChain(msgpack.dumps(model_chain))
             bottle.response.headers["Content-type"] = "application/json"
             v = base64.b64encode(model.process_binary(binary_input))
             return json.dumps({base64.b64encode(params['model']): v})
         else:
             bottle.abort(400)

    def dumps(self, msg):
        '''
        Run the correct dumps serialization format
        '''
        if self.serial == 'pickle':
            return pickle.dumps(msg)
        else:
            try:
                return msgpack.dumps(msg)
            except TypeError:
                if msgpack.version >= (0, 2, 0):
                    # Should support OrderedDict serialization, so, let's
                    # raise the exception
                    raise

                # msgpack is < 0.2.0, let's make it's life easier
                # Since OrderedDict is identified as a dictionary, we can't
                # make use of msgpack custom types, we will need to convert by
                # hand.
                # This means iterating through all elements of a dictionary or
                # list/tuple
                def odict_encoder(obj):
                    if isinstance(obj, dict):
                        for key, value in obj.copy().iteritems():
                            obj[key] = odict_encoder(value)
                        return dict(obj)
                    elif isinstance(obj, (list, tuple)):
                        obj = list(obj)
                        for idx, entry in enumerate(obj):
                            obj[idx] = odict_encoder(entry)
                        return obj
                    return obj
                return msgpack.dumps(odict_encoder(msg))

    def header(self, length):
        self._sync = self.conn.generate_sync()
        header = msgpack.dumps({IPROTO_CODE: self.request_type,
                                IPROTO_SYNC: self._sync,
                                IPROTO_SCHEMA_ID: self.conn.schema_version})

        return msgpack.dumps(length + len(header)) + header

def fake_multisignal_mainLoop(stop_flag, stream,  precomputed):
    import zmq
    pos = 0
    abs_pos = pos2 = 0
    
    context = zmq.Context()
    socket = context.socket(zmq.PUB)
    socket.bind("tcp://*:{}".format(stream['port']))
    
    socket.send(msgpack.dumps(abs_pos))
    
    packet_size = stream['packet_size']
    sampling_rate = stream['sampling_rate']
    np_arr = stream['shared_array'].to_numpy_array()
    half_size = np_arr.shape[1]/2
    while True:
        t1 = time.time()
        #~ print 'pos', pos, 'abs_pos', abs_pos
        #double copy
        np_arr[:,pos2:pos2+packet_size] = precomputed[:,pos:pos+packet_size] 
        np_arr[:,pos2+half_size:pos2+packet_size+half_size] = precomputed[:,pos:pos+packet_size]
        pos += packet_size
        pos = pos%precomputed.shape[1]
        abs_pos += packet_size
        pos2 = abs_pos%half_size
        socket.send(msgpack.dumps(abs_pos))
        
        if stop_flag.value:
            print 'will stop'
            break
        t2 = time.time()
        #~ time.sleep(packet_size/sampling_rate-(t2-t1))
        
        time.sleep(packet_size/sampling_rate)

    def dumps(self, msg):
        '''
        Run the correct dumps serialization format
        '''
        try:
            return msgpack.dumps(msg)
        except TypeError:
            if msgpack.version >= (0, 2, 0):
                # Should support OrderedDict serialization, so, let's
                # raise the exception
                raise

            # msgpack is < 0.2.0, let's make its life easier
            # Since OrderedDict is identified as a dictionary, we can't
            # make use of msgpack custom types, we will need to convert by
            # hand.
            # This means iterating through all elements of a dictionary or
            # list/tuple
            def odict_encoder(obj):
                if isinstance(obj, dict):
                    for key, value in obj.copy().iteritems():
                        obj[key] = odict_encoder(value)
                    return dict(obj)
                elif isinstance(obj, (list, tuple)):
                    obj = list(obj)
                    for idx, entry in enumerate(obj):
                        obj[idx] = odict_encoder(entry)
                    return obj
                return obj
            return msgpack.dumps(odict_encoder(msg))
        except SystemError as exc:
            log.critical('Unable to serialize message! Consider upgrading msgpack. '
                         'Message which failed was {failed_message} '
                         'with exception {exception_message}').format(msg, exc)

    def test_encode(self):
        """Test %s() can be encoded into msgpack serialised binary data."""

        # Test message can be encoded.
        msgA = self.message(self.items)
        serialised = msgA.encode()

        # Test message can be decoded.
        msgB = self.message(serialised)
        self.compare(msgA, msgB)

        # Ensure timestamp was copied.
        self.assertEqual(msgA['timestamp'], msgB['timestamp'])

        # Ensure additional fields can be sent.
        msgA['newfieldA'] = 'A'
        msgA['newfieldB'] = 'B'
        msgB.update(msgA.encode())
        self.compare(msgA, msgB)

        # Test arbitrary serialised data raises exceptions.
        with self.assertRaises(Exception):
            self.message(msgpack.dumps('invalid_type'))

        # Test serialised incomplete dictionaries raise exceptions.
        dct = self.items.copy()
        dct.popitem()
        serialised = msgpack.dumps(dct)
        with self.assertRaises(TypeError):
            self.message(serialised)

    def run_server(self):
        logging.basicConfig(level=logging.WARN,format='%(asctime)s [%(process)s] %(levelname)s %(message)s')

        socket = nanomsg.Socket(nanomsg.REP)
        socket.bind(self.endpoint)
        socket.recv_timeout = 1000

        last_check_ttr = time.time()
        while True:
            if time.time() - last_check_ttr > 1:
                self.check_ttr()

            try:
                msg = msgpack.loads(socket.recv())
                command = msg.pop(0)+'_cmd'
                msg = [ (_ if _ != '' else None) for _ in msg ]
                logger.debug("Command %s %s", command, msg)
                if hasattr(self,command):
                    result = getattr(self,command)(*msg)
                    socket.send(msgpack.dumps(result))
                else:
                    socket.send_multipart(msgpack.dumps('ERR','Unknown command "%s"' % command[0:-4]))
            except nanomsg.NanoMsgAPIError as error:
                if error.errno != nanomsg.EAGAIN:
                    raise

    def handle(self, command):
        handle_dict = {}
        handle_dict['login'] = login
        handle_dict['publish_notice'] = publish_notice
        handle_dict['get_notices'] = get_notices
        handle_dict['publish_proposal'] = publish_proposal
        handle_dict['get_proposal_list'] = get_proposal_list
        handle_dict['add_comment'] = add_comment
        handle_dict['del_comment'] = del_comment
        handle_dict['get_comments'] = get_comments
        handle_dict['get_proposal'] = get_proposal
        handle_dict['get_reconsider_list'] = get_reconsider_list
        handle_dict['publish_reconsider'] = publish_reconsider

        try:
            command_dict = msgpack.loads(command)
            print "command_dict %r" % command_dict
            if "command" in command_dict:
            	#TODO auto import command file call the process function
                print "command"
                if command_dict['command'] in handle_dict:
                    ##print command_dict['command']
                    reply = handle_dict[command_dict['command']].process(command_dict)
                else:
                    print command_dict['command']

            	return msgpack.dumps(reply)
            else:
            	 print "Unsupport command"
            	 return msgpack.dumps({"result":False})
            return msgpack.dumps({"result":False})
        except Exception,e:
            print e
            return msgpack.dumps({"result":False})

def test_request(req_header, req_body):
    query_header = msgpack.dumps(req_header)
    query_body = msgpack.dumps(req_body)
    packet_len = len(query_header) + len(query_body)
    query = msgpack.dumps(packet_len) + query_header + query_body
    try:
        s.send(query)
    except OSError as e:
        print '   => ', 'Failed to send request'
    resp_len = ''
    resp_headerbody = ''
    resp_header = {}
    resp_body = {}
    try:
        resp_len = s.recv(5)
        resp_len = msgpack.loads(resp_len)
        resp_headerbody = s.recv(resp_len)
        unpacker = msgpack.Unpacker(use_list = True)
        unpacker.feed(resp_headerbody)
        resp_header = unpacker.unpack()
        resp_body = unpacker.unpack()
    except OSError as e:
        print '   => ', 'Failed to recv response'
    res = {}
    res['header'] = resp_header
    res['body'] = resp_body
    return res

def redis_store(input_dir, name, server, port, **kw):
    import redis
    r = redis.StrictRedis(server, port)
    times = set()
    sensor_types = {}
    fn_to_time = lambda x: int(x.rsplit('/', 1)[-1].split('.', 1)[0])
    r.sadd('users', name)
    for fn in sorted(glob.glob(input_dir + '/*'), key=fn_to_time):
        fn_time = fn_to_time(fn) / 1000.
        if fn.endswith('.jpg'):
            times.add(sample[1])
            r.zadd(name + ':images', fn_time, os.path.basename(fn))
        else:
            try:
                data = msgpack.load(open(fn))
            except ValueError:
                print('Could not parse [%s]' % fn)
                continue
            print(data)
            for sensor_name, type_num in data[2].items():
                sensor_types[sensor_name] = msgpack.dumps(type_num)
            for sensor_name, samples in data[3].items():
                for sample in samples:
                    times.add(sample[1])
                    r.zadd(name + ':sensor:' + sensor_name, sample[1], msgpack.dumps(sample))
    r.hmset(name + ':sensors', sensor_types)
    r.zadd(name + ':times', **{msgpack.dumps(x): x for x in times})

 def handle_intern_message(self, addr, message):
     self._logger.info("received intern message %s", message)
     try:
         if message["type"] == "ssh_key":
             # copy the dict manually to ensure the corectness of the message
             yield from self.write_stdout({"type": "ssh_key", "ssh_key": message["ssh_key"]})
             yield from self.intern.send_multipart([addr, b'', msgpack.dumps({"type": "ok"}, encoding="utf8", use_bin_type=True)])
             return False
         if message["type"] == "run_student":
             # copy the dict manually to ensure the corectness of the message
             self.running_student_container[message["socket_id"]] = addr
             yield from self.write_stdout({"type": "run_student", "environment": message["environment"],
                                      "time_limit": message["time_limit"], "hard_time_limit": message["hard_time_limit"],
                                      "memory_limit": message["memory_limit"], "share_network": message["share_network"],
                                      "socket_id": message["socket_id"]})
             return False
         if message["type"] == "run_student_ask_retval":
             # ignore, just a dummy message
             return False
         if message["type"] == "done":
             yield from self.intern.send_multipart([addr, b'', msgpack.dumps({"type": "ok"}, encoding="utf8", use_bin_type=True)])
             return True
         return False
     except:
         self._logger.exception("Exception occured while handling an internal message")

def format_tags(tags, fmt):
    verbose = None not in tags.values()

    if fmt == "msgpack":
        import msgpack
        if not verbose:
            return msgpack.dumps(tags.keys())
        else:
            return msgpack.dumps(
                { tag:count for tag,count in tags.items() }
            )

    if fmt == "text":
        if not verbose:
            result =  '\n'.join(map(str,tags))
        else:
            result = '\n'.join("{} {}".format(tag, count)
                               for tag,count in tags.items())

    elif fmt == "json":
        import json
        if not verbose:
            result = json.dumps(tags.keys(), indent=4)
        else:
            result = json.dumps({ tag:count for tag,count in tags.items() },
                                indent=4)

    elif fmt == "web":
        print("Error: web output not supported for tag listing")

    return (result + "\n").encode("utf-8")

 def handle(self):
     print '%s: connected' % self.client_address[0]
     unpacker = msgpack.Unpacker()
     self.wfile.write(msgpack.dumps(self.server.get_next_bits()))
     self.wfile.flush()
     try:
         while True:
             try:
                 d = unpacker.unpack()
             except msgpack.OutOfData:
                 tmp = self.request.recv(4096)
                 if not tmp:
                     break
                 unpacker.feed(tmp)
                 continue
             if (not isinstance(d, list) or len(d) != 3 or
                         not isinstance(d[0], basestring) or
                         not isinstance(d[1], int) or
                         not isinstance(d[2], list)):
                 print '%s: invalid input %s' % (
                                 self.client_address[0], repr(d))
                 break
             client, bits, res = d
             self.server.register(client, bits, res)
             self.wfile.write(msgpack.dumps(self.server.get_next_bits()))
             self.wfile.flush()
     except IOError, e:
         print '%s: %s' % (self.client_address[0], e)

    def test_to_dict_msgpack_with_data_token(self):
        token = DataToken('Host', 'www.w3af.com', ('Host',))
        headers = Headers([('Host', token)])
        freq = FuzzableRequest(URL("http://www.w3af.com/"), headers=headers)

        req = HTTPRequest.from_fuzzable_request(freq)

        msgpack.dumps(req.to_dict())

def my_dumps(obj, context=None):
    if type(obj).__name__ == 'MyObject':
        header = {'serializer': 'my-ser'}
        frames = [msgpack.dumps(obj.__dict__, use_bin_type=True),
                  msgpack.dumps(context, use_bin_type=True)]
        return header, frames
    else:
        raise NotImplementedError()

def array_sha256(a):
    dtype = msgpack.dumps(str(a.dtype))
    shape = msgpack.dumps(a.shape)
    bdata = a.flatten().view(numpy.uint8)
    sha = hashlib.sha256()
    sha.update(dtype)
    sha.update(shape)
    sha.update(bdata)
    return sha.hexdigest()

def test_request(req_header, req_body):
    query_header = msgpack.dumps(req_header)
    query_body = msgpack.dumps(req_body)
    packet_len = len(query_header) + len(query_body)
    query = msgpack.dumps(packet_len) + query_header + query_body
    try:
        s.send(query)
    except OSError as e:
        print '   => ', 'Failed to send request'
    return receive_response()

def chunker(request, response):
    chunks = yield request.read()
    try:
        chunks = int(msgpack.loads(chunks))
    except ValueError:
        chunks = int(chunks)

    for num in xrange(chunks):
        response.write(msgpack.dumps('{0:-<1024}'.format(num)))
    response.write(msgpack.dumps('Done'))
    response.close()

def brainvisionsocket_mainLoop(stop_flag, streams, brain_host, brain_port, resolutions):
    import zmq
    abs_pos = pos2 = 0
    
    context = zmq.Context()
    
    stream0 = streams[0]
    socket0 = context.socket(zmq.PUB)
    socket0.bind("tcp://*:{}".format(stream0['port']))
    socket0.send(msgpack.dumps(abs_pos))
    
    stream1 = streams[1]
    socket1 = context.socket(zmq.PUB)
    socket1.bind("tcp://*:{}".format(stream1['port']))
    
    brain_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    brain_socket.connect((brain_host, brain_port))
    
    packet_size = stream0['packet_size']
    sampling_rate = stream0['sampling_rate']
    np_arr = stream0['shared_array'].to_numpy_array()
    half_size = np_arr.shape[1]/2
    while not stop_flag.value :
        buf_header = recv_data(brain_socket, 24)
        (id1, id2, id3, id4, msgsize, msgtype) = struct.unpack('<llllLL', buf_header)
        rawdata = recv_data(brain_socket,  msgsize - 24)
        if msgtype == 1:
            pass
        elif msgtype == 4:
            block, chunk, markers = get_signal_and_markers(rawdata, stream0.nb_channel)
            
            # Signals
            chunk *= resolutions[np.newaxis, :]
            packet_size = chunk.shape[0]
            #~ print 'packet_size', packet_size
            np_arr[:,pos2:pos2+packet_size] = chunk.transpose() 
            np_arr[:,pos2+half_size:pos2+packet_size+half_size] = chunk.transpose()
            if pos2+packet_size>half_size:
                pass
                #TODO : check packet_size
            abs_pos += packet_size
            pos2 = abs_pos%half_size
            socket0.send(msgpack.dumps(abs_pos))
            
            #Triggers
            markers['pos'] += (abs_pos-packet_size)
            for marker in markers:
                socket1.send(marker.tostring())
            

        elif msgtype == 3:
            break
    
    brain_socket.close()