def generate_training_set(follow, followed, ratio, solution_file, data_file):
    """ Uses the solution file to generate training set to train
    the model, hoping this method can get better result.
    Ratio controls the fraction of pos and neg data sets, if ratio is -1,
    the fraction is the origion fraction."""

    raw_solution = utilities.read_file(solution_file, False)
    dict_solution = {}
    for i in range(len(raw_solution)):
        row = raw_solution[i]
        dict_solution[int(row[0])] = set(int(n) for n in row[1 : :])

    x_train = [['spring brother is a true man']]
    for node in dict_solution.keys():
        nodes_pos = dict_solution[node]
        for n in nodes_pos:
            features = rank.get_features(follow, followed, node, n)
            x_train.append([1] + features)

        nodes_neg = candidate.get_candidates(follow, followed, node)
        nodes_neg.difference_update(nodes_pos)
        nodes_neg = list(nodes_neg)
        perm = random.permutation(len(nodes_neg))
        if ratio != -1:
            num = min(int(len(nodes_pos) * ratio), len(nodes_neg))
        else:
            num = len(nodes_neg)
        for i in range(num):
            node = nodes_neg[perm[i]]
            features = rank.get_features(follow, followed, node, n)
            x_train.append([0] + features)

    utilities.write_file(data_file, x_train)

 def compile_and_write(self, idl_filename):
     interface_name = idl_filename_to_interface_name(idl_filename)
     definitions = self.reader.read_idl_definitions(idl_filename)
     output_code_list = self.code_generator.generate_code(
         definitions, interface_name)
     for output_path, output_code in output_code_list:
         write_file(output_code, output_path, self.only_if_changed)

def main():
    # Check input
    if len(sys.argv) != 5:
        print"usage: python run_ML_FW.py [train file] [setting file] [model folder] [test data folder]"
        exit()
    # Get environment variables
    train_file = sys.argv[1]
    setting_file = sys.argv[2]
    model_folder = sys.argv[3]
    test_data_folder = sys.argv[4]
    tops = 10#int(sys.argv[5])    
    # Create model folder if it doesn't exist
    if os.path.exists(model_folder):
        shutil.rmtree(model_folder)
    os.makedirs(model_folder)
    # Read settings
    print'reading setting ...'
    ddict = utilities.read_setting(setting_file)
    print'write setting ...'
    file_name = '%s/setting.txt'%(model_folder)
    utilities.write_setting(ddict, file_name)
    # Read data for computing perplexities
    print'read data for computing perplexities ...'
    (wordids_1, wordcts_1, wordids_2, wordcts_2) = \
    utilities.read_data_for_perpl(test_data_folder)
    # Initialize the algorithm
    print'initialize the algorithm ...'
    ml_fw = ML_FW.MLFW(ddict['num_terms'], ddict['num_topics'], ddict['tau0'], ddict['kappa'], ddict['iter_infer'])
    # Start
    print'start!!!'
    i = 0
    while i < ddict['iter_train']:
        i += 1
        print'\n***iter_train:%d***\n'%(i)
        datafp = open(train_file, 'r')
        j = 0
        while True:
            j += 1
            (wordids, wordcts) = utilities.read_minibatch_list_frequencies(datafp, ddict['batch_size'])
            # Stop condition
            if len(wordids) == 0:
                break
            # 
            print'---num_minibatch:%d---'%(j)
            (time_e, time_m, theta) = ml_fw.static_online(ddict['batch_size'], wordids, wordcts)
            # Compute sparsity
            sparsity = utilities.compute_sparsity(theta, theta.shape[0], theta.shape[1], 't')
            # Compute perplexities
            LD2 = utilities.compute_perplexities_fw(ml_fw.beta, ddict['iter_infer'], \
                               wordids_1, wordcts_1, wordids_2, wordcts_2)
            # Search top words of each topics
            list_tops = utilities.list_top(ml_fw.beta, tops)
            # Write files
            utilities.write_file(i, j, ml_fw.beta, time_e, time_m, theta, sparsity, LD2, list_tops, tops, model_folder)
        datafp.close()
    # Write final model to file
    file_name = '%s/beta_final.dat'%(model_folder)
    utilities.write_topics(ml_fw.beta, file_name)
    # Finish
    print'done!!!'        

def write_event_interfaces_file(event_idl_files, destination_filename, only_if_changed):
    def extended_attribute_string(name, value):
        if name == 'RuntimeEnabled':
            value += 'Enabled'
        return name + '=' + value

    def interface_line(full_path):
        relative_path_local, _ = os.path.splitext(os.path.relpath(full_path, source_dir))
        relative_path_posix = relative_path_local.replace(os.sep, posixpath.sep)

        idl_file_contents = get_file_contents(full_path)
        extended_attributes = get_interface_extended_attributes_from_idl(idl_file_contents)
        extended_attributes_list = [
            extended_attribute_string(name, extended_attributes[name])
            for name in EXPORTED_EXTENDED_ATTRIBUTES
            if name in extended_attributes]

        return '%s %s\n' % (relative_path_posix,
                            ', '.join(extended_attributes_list))

    lines = ['namespace="Event"\n',
             '\n']
    interface_lines = [interface_line(event_idl_file)
                       for event_idl_file in event_idl_files]
    interface_lines.sort()
    lines.extend(interface_lines)
    write_file(''.join(lines), destination_filename, only_if_changed)

 def generate_callback_function_impl(output_directory, component):
     generator = CodeGeneratorCallbackFunction(
         component_info_providers[component], cache_dir=None,
         output_dir=output_directory, target_component=component)
     outputs = generator.generate_code()
     for output_path, output_code in outputs:
         write_file(output_code, output_path)

 def generate_union_type_containers(output_directory, component):
     generator = CodeGeneratorUnionType(
         component_info_providers[component], cache_dir=None,
         output_dir=output_directory, target_component=component)
     outputs = generator.generate_code()
     for output_path, output_code in outputs:
         write_file(output_code, output_path)

def generate_conditional_features(info_provider, options, idl_filenames):
    reader = IdlReader(info_provider.interfaces_info, options.cache_directory)
    jinja_env = initialize_jinja_env(options.cache_directory)

    # Extract the bidirectional mapping of conditional features <-> interfaces
    # from the global info provider and the supplied list of IDL files.
    feature_info = conditional_features_info(info_provider,
                                             reader, idl_filenames,
                                             options.target_component.lower())

    # Convert that mapping into the context required for the Jinja2 templates.
    template_context = conditional_features_context(
        MODULE_PYNAME, feature_info)

    # Generate and write out the header file
    header_text = render_template(jinja_env.get_template(
        "ConditionalFeaturesFor%s.h.tmpl" % options.target_component.title()), template_context)
    header_path = posixpath.join(options.output_directory,
                                 "ConditionalFeaturesFor%s.h" % options.target_component.title())
    write_file(header_text, header_path)

    # Generate and write out the implementation file
    cpp_text = render_template(jinja_env.get_template(
        "ConditionalFeaturesFor%s.cpp.tmpl" % options.target_component.title()), template_context)
    cpp_path = posixpath.join(options.output_directory,
                              "ConditionalFeaturesFor%s.cpp" % options.target_component.title())
    write_file(cpp_text, cpp_path)

 def compile_and_write(self, idl_filename):
     interface_name = idl_filename_to_interface_name(idl_filename)
     definitions = self.reader.read_idl_definitions(idl_filename)
     target_definitions = definitions[self.target_component]
     output_code_list = self.code_generator.generate_code(
         target_definitions, interface_name)
     for output_path, output_code in output_code_list:
         write_file(output_code, output_path)

 def _save_previous_version(self):
     """
     Save current installed mysql version to cache file.
     It will be previous version after new installation.
     """
     version = mysql_version()
     if version:
         write_file(self.CACHE_VERSION_FILE, version)

    def compile_file(self, idl_filename):
        definitions = self.reader.read_idl_definitions(idl_filename)

        files = self.code_generator.generate_code(definitions)

        for file_data in files:
            file_name = os.path.join(self.output_directory,
                                     file_data['file_name'])
            write_file(file_data['contents'], file_name)

def write_global_constructors_partial_interface(interface_name, destination_filename, constructor_attributes_list, only_if_changed):
    # FIXME: replace this with a simple Jinja template
    lines = (['[\n'] +
             ['    NoHeader,\n'] +
             [']\n'] +
             ['partial interface %s {\n' % interface_name] +
             ['    %s;\n' % constructor_attribute
              # FIXME: sort by interface name (not first by extended attributes)
              for constructor_attribute in sorted(constructor_attributes_list)] +
             ['};\n'])
    write_file(''.join(lines), destination_filename, only_if_changed)

def write_global_constructors_partial_interface(interface_name, idl_filename, constructor_attributes_list):
    # FIXME: replace this with a simple Jinja template
    lines = (['partial interface %s {\n' % interface_name] +
             ['    %s;\n' % constructor_attribute
              # FIXME: sort by interface name (not first by extended attributes)
              for constructor_attribute in sorted(constructor_attributes_list)] +
             ['};\n'])
    write_file(''.join(lines), idl_filename)
    header_filename = os.path.splitext(idl_filename)[0] + '.h'
    idl_basename = os.path.basename(idl_filename)
    write_file(HEADER_FORMAT.format(idl_basename=idl_basename), header_filename)

    def compile_and_write(self, idl_filename, output_filenames):
        interface_name = idl_filename_to_interface_name(idl_filename)
        idl_pickle_filename = os.path.join(self.output_directory,
                                           '%s_globals.pickle' % interface_name)
        definitions = self.reader.read_idl_definitions(idl_filename)
        output_code_list = self.code_generator.generate_code(definitions,
                                                             interface_name,
                                                             idl_pickle_filename,
                                                             self.only_if_changed)

        for output_code, output_filename in zip(output_code_list, output_filenames):
            write_file(output_code, output_filename, self.only_if_changed)

    def set_mysql_version(self, version):
        """
        Set new mysql version for next install
        """
        # check available versions
        versions = ["auto"] + self.REPO_NAMES.keys()
        if version not in versions:
            print >> sys.stderr, "Invalid mysql version."
            print >> sys.stderr, "Available versions: %s" % ", ".join(versions)
            sys.exit(2)

        write_file(self.NEW_VERSION_FILE, version)

def main():
    opts, _ = parse_args()
    # TODO(peria): get rid of |info_provider|
    info_provider = create_component_info_provider(
        opts.info_dir, opts.target_component)
    generator = ExternalReferenceTableGenerator(opts, info_provider)

    idl_files = utilities.read_idl_files_list_from_file(opts.idl_files_list, False)
    for idl_file in idl_files:
        generator.process_idl_file(idl_file)
    output_code = generator.generate()
    output_path = opts.output
    write_file(output_code, output_path)

def generate_test_set(follow, followed, test_file, validation_file,
    solution_file, num,  max_remove_num):
    """ Generates the test set for analysis. """

    nodes_exclude = utilities.read_nodes_list(test_file)

    print 'Generating test nodes...'
    nodes_test = generate_test_nodes(follow, nodes_exclude, num)
    writable_nodes_test = [[n] for n in nodes_test]
    solution = generate_solution(follow, followed, nodes_test, max_remove_num)

    utilities.write_file(validation_file, writable_nodes_test)
    utilities.write_file(solution_file, solution)

def generate_union_type_containers(options):
    info_provider = create_component_info_provider(
        options.info_dir, options.target_component)
    if not info_provider.interfaces_info:
        raise Exception('Interfaces info is required to generate '
                        'union types containers')
    generator = CodeGeneratorUnionType(
        info_provider,
        options.cache_directory,
        options.output_directory,
        options.target_component)
    output_code_list = generator.generate_code()
    for output_path, output_code in output_code_list:
        write_file(output_code, output_path, options.write_file_only_if_changed)

def generate_union_type_containers(options):
    if not (options.interfaces_info_file and options.component_info_file):
        raise Exception('Interfaces info is required to generate '
                        'union types containers')
    with open(options.interfaces_info_file) as interfaces_info_file:
        interfaces_info = pickle.load(interfaces_info_file)
    with open(options.component_info_file) as component_info_file:
        component_info = pickle.load(component_info_file)
    generator = CodeGeneratorUnionType(
        interfaces_info,
        options.cache_directory,
        options.output_directory,
        options.target_component)
    output_code_list = generator.generate_code(component_info['union_types'])
    for output_path, output_code in output_code_list:
        write_file(output_code, output_path, options.write_file_only_if_changed)

 def _get_mysqlup():
     """
     ? Set value for panel update MYSQLUP option
     """
     if os.path.exists("/etc/cpupdate.conf"):
         shutil.copy2("/etc/cpupdate.conf", "/etc/cpupdate.conf.governor")
         is_mysqlup = grep("/etc/cpupdate.conf", "MYSQLUP")
         if is_mysqlup:
             if not grep(is_mysqlup, "never$", True):
                 replace_lines("/etc/cpupdate.conf", "".join(is_mysqlup),
                               "MYSQLUP=never")
         else:
             add_line("/etc/cpupdate.conf", "\nMYSQLUP=never\n")
     else:
         write_file("/etc/cpupdate.conf.governor", "")
         write_file("/etc/cpupdate.conf", "MYSQLUP=never\n")

def write_event_interfaces_file(event_idl_files, destination_filename, suffix):
    def interface_line(full_path):
        relative_path_local, _ = os.path.splitext(os.path.relpath(full_path, source_dir))
        relative_path_posix = relative_path_local.replace(os.sep, posixpath.sep)

        idl_file_contents = get_file_contents(full_path)
        extended_attributes = get_interface_extended_attributes_from_idl(idl_file_contents)
        extended_attributes_list = [
            (name, extended_attributes[name])
            for name in EXPORTED_EXTENDED_ATTRIBUTES
            if name in extended_attributes]

        return (relative_path_posix, extended_attributes_list)

    lines = [
        '{',
        'metadata: {',
        '  namespace: "Event",'
    ]
    if suffix:
        lines.append('  suffix: "' + suffix + '",')
        lines.append('  export: "%s_EXPORT",' % suffix.upper())
    else:
        lines.append('  export: "CORE_EXPORT",')
    lines.extend([
        '},',
        'data: ['
    ])
    interface_lines = [interface_line(event_idl_file)
                       for event_idl_file in event_idl_files]
    interface_lines.sort()
    for name, attributes in interface_lines:
        lines.extend([
            '  {',
            '    name: "%s",' % name
        ])
        for param, value in attributes:
            if param == 'RuntimeEnabled':
                value += 'Enabled'
            lines.append('    %s: "%s",' % (param, value))
        lines.append('  },')
    lines.extend([
        ']',
        '}'
    ])
    write_file('\n'.join(lines), destination_filename)