def main(args):
    logging.basicConfig(level=logging.INFO)
    model = json.load(args.ifile)
    fn = fn_from_args(args)
    if fn:
        weight_layers = [layer for layer in model
                         if layer['layerName'] in weight_first_list]
        if fn.needs_two_step:
            for weights in [layer['parameters'][0] for layer in weight_layers]:
                fn.consume(weights)
            fn.done()
        for i, layer in enumerate(weight_layers):
            layer['parameters'][0] = transform(layer['parameters'][0], fn)
        if fn.needs_two_step:
            model = {
                'codebook' : fn.serialize_codebook(),
                'model' : model
            }
    if args.ubjson_format:
        args.ofile.write(simpleubjson.encode(model))
    elif args.json:
        args.ofile.write(json.dumps(model).encode('utf8'))
    else:
        msgpack.dump(model, args.ofile, use_bin_type=True,
                     use_single_float=args.single_precision_float)
    args.ofile.close()
    args.ifile.close()

def freqs_to_cBpack(in_filename, out_filename, cutoff=-600):
    """
    Convert a csv file of words and their frequencies to a file in the
    idiosyncratic 'cBpack' format.

    Only words with a frequency greater than `cutoff` centibels will be
    written to the new file.

    This cutoff should not be stacked with a cutoff in `read_freqs`; doing
    so would skew the resulting frequencies.
    """
    freqs = read_freqs(in_filename, cutoff=0, lang=None)
    cBpack = []
    for token, freq in freqs.items():
        cB = round(math.log10(freq) * 100)
        if cB <= cutoff:
            continue
        neg_cB = -cB
        while neg_cB >= len(cBpack):
            cBpack.append([])
        cBpack[neg_cB].append(token)

    for sublist in cBpack:
        sublist.sort()

    # Write a "header" consisting of a dictionary at the start of the file
    cBpack_data = [{'format': 'cB', 'version': 1}] + cBpack

    with gzip.open(out_filename, 'wb') as outfile:
        msgpack.dump(cBpack_data, outfile)

    def dump(data, filepath):
        '''
        Write data as as type self.ext to filepath. json or msgpack
        '''
        if ' ' in filepath:
            raise raeting.KeepError("Invalid filepath '{0}' "
                                    "contains space".format(filepath))

        root, ext = os.path.splitext(filepath)
        if ext == '.json':
            with aiding.ocfn(filepath, "w+") as f:
                json.dump(data, f, indent=2, encoding='utf-8')
                f.flush()
                os.fsync(f.fileno())
        elif ext == '.msgpack':
            if not msgpack:
                raise raeting.KeepError("Invalid filepath ext '{0}' "
                            "needs msgpack installed".format(filepath))
            with aiding.ocfn(filepath, "w+b", binary=True) as f:
                msgpack.dump(data, f, encoding='utf-8')
                f.flush()
                os.fsync(f.fileno())
        else:
            raise raeting.KeepError("Invalid filepath ext '{0}' "
                        "not '.json' or '.msgpack'".format(filepath))

def crawler_baidu_person_list():
    # 从百度人气榜上获取人名
    all_person_list = set()
    for index in range(50):
        url = 'http://baike.baidu.com/operation/api/starflowerstarlist?' \
              'rankType=thisWeek&pg=%d' % index
        req = urllib2.Request(url, None)
        response = urllib2.urlopen(req)
        html_doc = response.read()
        content = json.loads(html_doc)
        this_page_list = content.get('data').get('thisWeek')
        for person_content in this_page_list:
            all_person_list.add(person_content.get('name'))
    print len(all_person_list)

    for index in range(50):
        url = 'http://baike.baidu.com/operation/api/starflowerstarlist?' \
              'rankType=lastWeek&pg=%d' % index
        req = urllib2.Request(url, None)
        response = urllib2.urlopen(req)
        html_doc = response.read()
        content = json.loads(html_doc)
        this_page_list = content.get('data').get('lastWeek')
        for person_content in this_page_list:
            all_person_list.add(person_content.get('name'))
    print len(all_person_list)
    all_person_set = list(all_person_list)
    msgpack.dump(all_person_set, open('baidu_fans.p', 'w'))

def crawler_fans():
    all_person_set = set()
    f_fans = open('fans.txt', 'w')
    for page_index in range(1, 345, 1):
        for class_index in [1, 2, 4, 5, 6]:
            start = time()
            try:
                url = 'https://123fans.cn/results.php?qi=%d&c=%d'\
                      % (page_index, class_index)
                req = urllib2.Request(url, None)
                response = urllib2.urlopen(req)
                html_doc = response.read()
                soup = BeautifulSoup(html_doc)
                children = [k for k in soup.children][1]
                lis = children.select('.odd')
                for k in lis:
                    try:
                        name = k.select('.name')[0].string
                        all_person_set.add(name)
                        f_fans.write(name+'\n')
                    except:
                        traceback.print_exc()
                        continue
                print page_index, class_index, \
                    len(all_person_set), time()-start
            except:
                print 'error', page_index, class_index
        sleep(5)
    all_person_set = list(all_person_set)
    msgpack.dump(all_person_set, open('fans.p', 'w'))

def load_check_result_url(dic_file, check_url_file):
    person_result_dic = {}   # {person:([](right_set),[](wrong_set))} # 肯定正确和肯定错的的图片
    right_url_count = wrong_url_count = error_format_count = no_baike_count = no_meaning_count = 0
    if os.path.exists(dic_file):
        person_result_dic = msgpack.load(open(dic_file, 'rb'))
    for line in open(check_url_file):
        tmp = line.rstrip().split('\t')
        # [person_name, pic_index, pic_url, baike_name, baike_sim, newbaike_sim, guess_info]
        person_name = tmp[0]
        right_list, wrong_list = person_result_dic.get(person_name, ([], []))
        if len(tmp) == 7:
            if tmp[3] not in no_meaning_list:
                if tmp[3] == no_find_baike:
                    no_baike_count += 1
                    continue
                else:
                    if get_newbaike_sim(tmp[4]) > sim_threshold:
                        if tmp[0] == tmp[3]:
                            right_list.append(tmp[1])
                            right_url_count += 1
                        else:
                            wrong_url_count += 1
                            wrong_list.append(tmp[1])
                    else:   # 小于某概率时结果不可信,需要标注
                        no_baike_count += 1
                        continue
            else:
                no_meaning_count += 1
                continue
        else:
            error_format_count += 1
            continue
        person_result_dic[person_name] = (right_list, wrong_list)
    print right_url_count, wrong_url_count, no_baike_count, no_meaning_count, error_format_count
    msgpack.dump(person_result_dic, open('person_result_dic.p', 'w'))

def main():
    dialect_name, working_dir = sys.argv[1:]
    
    executor = sqlexecutor.executor(dialect_name, working_dir)
    
    print "Ready"
    sys.stdout.flush()
    try:
        for message in msgpack.Unpacker(sys.stdin, read_size=1):
            command = message[0]
            args = message[1:]
            
            if command == "execute":
                (creation_sql, query, ) = args
                result = executor.execute(creation_sql, query)
                if result.table is None:
                    column_names = None
                    rows = None
                else:
                    column_names = result.table.column_names
                    rows = result.table.rows
                
                
                msgpack.dump((result.error, column_names, rows), sys.stdout)
                sys.stdout.flush()
            else:
                return
            
    finally:
        executor.close()

    def dump(data, filepath):
        '''
        Write data as as type self.ext to filepath. json or msgpack
        '''
        if ' ' in filepath:
            raise raeting.KeepError("Invalid filepath '{0}' "
                                    "contains space".format(filepath))

        if hasattr(data, 'get'):
            for key, val in data.items():  # P3 json.dump no encoding parameter
                if isinstance(val, (bytes, bytearray)):
                    data[key] = val.decode('utf-8')

        root, ext = os.path.splitext(filepath)
        if ext == '.json':
            with ocfn(filepath, "w+") as f:
                json.dump(data, f, indent=2)
                f.flush()
                os.fsync(f.fileno())
        elif ext == '.msgpack':
            if not msgpack:
                raise raeting.KeepError("Invalid filepath ext '{0}' "
                            "needs msgpack installed".format(filepath))
            with ocfn(filepath, "w+b", binary=True) as f:
                msgpack.dump(data, f, encoding='utf-8')
                f.flush()
                os.fsync(f.fileno())
        else:
            raise raeting.KeepError("Invalid filepath ext '{0}' "
                        "not '.json' or '.msgpack'".format(filepath))

def get_or_build(path, build_fn, *args, **kwargs):
    """
    Load from serialized form or build an object, saving the built
    object.

    Remaining arguments are provided to `build_fn`.
    """

    save = False
    obj = None

    if path is not None and os.path.isfile(path):
        with open(path, 'rb') as obj_f:
            obj = msgpack.load(obj_f, use_list=False, encoding='utf-8')
    else:
        save = True

    if obj is None:
        obj = build_fn(*args, **kwargs)

        if save and path is not None:
            with open(path, 'wb') as obj_f:
                msgpack.dump(obj, obj_f)

    return obj

def wedge(cluster_set, report_cluster_status=False, force_wedge_thrsh=False):
    # The lower bound of the edges being processed by the wedge algorithm.
    global edge_cut_prob
    global wedge_thrsh

    if not force_wedge_thrsh:
        edge_cut_prob = bconfig.WEDGE_THRESHOLD / 4.
        wedge_thrsh = bconfig.WEDGE_THRESHOLD
    else:
        edge_cut_prob = force_wedge_thrsh / 4.
        wedge_thrsh = force_wedge_thrsh

    matr = ProbabilityMatrix(cluster_set.last_name)
    matr.load()

    global h5file
    h5filepath = bconfig.TORTOISE_FILES_PATH+'wedge_cache_'+str(PID())
    h5file = h5py.File(h5filepath)

    convert_cluster_set(cluster_set, matr)
    del matr # be sure that this is the last reference!

    do_wedge(cluster_set)

    report = []
    if bconfig.DEBUG_WEDGE_PRINT_FINAL_CLUSTER_COMPATIBILITIES or report_cluster_status:
        msg = []
        for cl1 in cluster_set.clusters:
            for cl2 in cluster_set.clusters:
                if cl2 > cl1:
                    id1 = cluster_set.clusters.index(cl1)
                    id2 = cluster_set.clusters.index(cl2)
                    c12 = _compare_to(cl1,cl2)
                    c21 = _compare_to(cl2,cl1)
                    report.append((id1,id2,c12+c21))
                    msg.append( ' %s vs %s : %s + %s = %s -- %s' %  (id1, id2, c12, c21, c12+c21, cl1.hates(cl2)))
        msg = 'Wedge final clusters for %s: \n' % str(wedge_thrsh) + '\n'.join(msg)
        if not bconfig.DEBUG_WEDGE_OUTPUT and bconfig.DEBUG_WEDGE_PRINT_FINAL_CLUSTER_COMPATIBILITIES:
            print
            print msg
            print
        wedge_print(msg)


    restore_cluster_set(cluster_set)

    if bconfig.DEBUG_CHECKS:
        assert cluster_set._debug_test_hate_relation()
        assert cluster_set._debug_duplicated_recs()

    if report_cluster_status:
        destfile = '/tmp/baistats/cluster_status_report_pid_%s_lastname_%s_thrsh_%s' % (str(PID()),str(cluster_set.last_name),str(wedge_thrsh))
        f = filehandler.open(destfile, 'w')
        SER.dump([wedge_thrsh,cluster_set.last_name,report,cluster_set.num_all_bibs],f)
        f.close()
    gc.collect()

    h5file.close()
    os.remove(h5filepath)

 def dump(self, outfile):
     """Write a serialized version of the database to filehandle."""
     db_dict = {
         'meta_prints': self.meta_prints,
         'content_prints': self.content_prints,
         'series_id': self.series_id,
     }
     msgpack.dump(db_dict, outfile)

 def serializeToFile(self, fname, annotations):
     """
     Overwritten to write Msgpack files.
     """
     # TODO make all image filenames relative to the label file
     import msgpack
     f = open(fname, "w")
     msgpack.dump(annotations, f)

 def __call__(self):
   if self.filename == None:
     self.filename = msgpackmemoized_basedir + '/' + self.f.__name__ + '.msgpack'
   if path.exists(self.filename):
     return msgpack.load(open(self.filename))
   result = self.f()
   msgpack.dump(result, open(self.filename, 'w'))
   return result

    def dump(cls, records, filepath=None):
        """Dump in the same fashion as load."""

        if filepath is None:
            filepath = os.path.join(config['current_snapshot'], 'repos.msgpack')

        with utils.FaultTolerantFile(filepath) as f:
            msgpack.dump(records, f, default=cls._dumper)

def match_one_file(midi_filename, embed_fn, hash_fn, msd_embeddings,
                   msd_sequences, msd_feature_paths, msd_ids, output_filename):
    """
    Match one MIDI file to the million song dataset by computing its CQT,
    pruning by matching its embedding, re-pruning by matching its downsampled
    hash sequence, and finally doing DTW on CQTs on the remaining entries.

    Parameters
    ----------
    midi_filename : str
        Path to a MIDI file to match to the MSD
    embed_fn : function
        Function which takes in a CQT and produces a fixed-length embedding
    hash_fn : function
        Function which takes in a CQT and produces a sequence of binary vectors
    msd_embeddings : np.ndarray
        (# MSD entries x embedding dimension) matrix of all embeddings for all
        entries from the MSD
    msd_sequences : list of np.ndarray
        List of binary vector sequences (represented as ints) for all MSD
        entries
    msd_feature_paths : list of str
        Path to feature files (containing CQT) for each MSD entry
    msd_ids : list of str
        MSD ID of each corresponding entry in the above lists
    output_filename : str
        Where to write the results file, which includes the DTW scores for all
        of the non-pruned MSD entries
    """
    # Try to compute a CQT for the MIDI file
    try:
        m = pretty_midi.PrettyMIDI(midi_filename)
    except Exception as e:
        print 'Could not parse {}: {}'.format(
            os.path.split(midi_filename)[1], traceback.format_exc(e))
        return
    try:
        midi_gram = feature_extraction.midi_cqt(m)
    except Exception as e:
        print "Error creating CQT for {}: {}".format(
            os.path.split(midi_filename)[1], traceback.format_exc(e))
        return
    # Skip this file if the MIDI gram is very long, to avoid memory issues
    if midi_gram.shape[0] > MAX_FRAMES:
        return
    # Compute the embedding of the CQT
    midi_embedding = embed_fn(midi_gram.reshape(1, 1, *midi_gram.shape))
    # Compute the hash sequence
    midi_hash_sequence = hash_fn(midi_gram.reshape(1, 1, *midi_gram.shape))
    # Convert to sequence of integers
    midi_hash_sequence = dhs.vectors_to_ints(midi_hash_sequence > 0)
    midi_hash_sequence = midi_hash_sequence.astype(np.uint32)
    matches = match_one_midi(
        midi_gram, midi_embedding, midi_hash_sequence, msd_embeddings,
        msd_sequences, msd_feature_paths, msd_ids)
    # Write out the result
    with open(output_filename, 'wb') as f:
        msgpack.dump(matches, f)

def SavePermResults(path,name,method,*args):
    data = []
    for thing in enumerate(args):
        data.append(thing)
    if method=='msgpack':
        with open(os.path.join(path,name+'.mpac'),'wb') as f:
            msgpack.dump(data,f)
    else:
        with open(os.path.join(path,name+'.pickle'),'wb') as f:
            pickle.dump(data,f)

def make_hanzi_converter(table_in, msgpack_out):
    table = {}
    with open(table_in, encoding="utf-8") as infile:
        for line in infile:
            hexcode, char = line.rstrip("\n").split("\t")
            codept = int(hexcode, 16)
            assert len(char) == 1
            if chr(codept) != char:
                table[codept] = char
    with gzip.open(msgpack_out, "wb") as outfile:
        msgpack.dump(table, outfile, encoding="utf-8")

def make_hanzi_converter(table_in, msgpack_out):
    table = {}
    with open(table_in, encoding='utf-8') as infile:
        for line in infile:
            hexcode, char = line.rstrip('\n').split('\t')
            codept = int(hexcode, 16)
            assert len(char) == 1
            if chr(codept) != char:
                table[codept] = char
    with gzip.open(msgpack_out, 'wb') as outfile:
        msgpack.dump(table, outfile, raw=False)

def create_train_valid_data(folder='/data/liubo/face/research_feature_self'):
    # 根据已经存在的数据训练人脸验证模型
    person_list = os.listdir(folder)
    path_feature_dic = {}  #
    for person in person_list:
        person_path = os.path.join(folder, person)
        pic_feature_list = os.listdir(person_path)
        for pic_feature_path in pic_feature_list:
            pic_feature_path = os.path.join(person_path, pic_feature_path)
            pic_feature = msgpack_numpy.load(open(pic_feature_path, 'rb'))
            path_feature_dic[pic_feature_path] = pic_feature
    msgpack.dump(path_feature_dic, open('research_feature.p', 'wb'))

def write_http_response_to_temp_file(http_response):
    """
    Write an HTTPResponse instance to a temp file using msgpack

    :param http_response: The HTTP response
    :return: The name of the file
    """
    temp = get_temp_file('http')
    data = http_response.to_dict()
    msgpack.dump(data, temp, use_bin_type=True)
    temp.close()
    return temp.name