def save_html(data, filename, key):
    path = join(settings.LOG_DIR, key, filename + '.html')
    try:
        write(data, path)
    except IOError:
        mkdir_p(dirname(path))
        write(data, path)

def make_pull_plot_gen(category, misIDRatios, catRatios):
  pull_plot = TH1D(category, category, 6, 0, 6 )
  others_plot = TH1D(category+"others", category+"others", 6, 0, 6 )
  true_plot = TH1D(category+"true", category+"true", 6, 0, 6 )
  bin_names = get_all_containers(category)
  for b in range(1, len(bin_names)+1):
    pull_plot.GetXaxis().SetBinLabel(b,bin_names[b-1])
    #pull_plot.SetAxisRange(-0.006, 0.006,"Y")
    #others_plot.SetAxisRange(-0.006, 0.006,"Y")
    others_plot.GetXaxis().SetBinLabel(b,bin_names[b-1])
    true_plot.GetXaxis().SetBinLabel(b,bin_names[b-1])
    (value, err, err_plus) = catRatios[bin_names[b-1]]
    pull_plot.SetBinContent(b, value)
    pull_plot.SetBinError(b, err)

    other = get_other_component(category, bin_names[b-1])
    (valueO, errO, err0_plus) = misIDRatios[other]
    others_plot.SetBinContent(b, valueO)
    others_plot.SetBinError(b, errO)

    true_plot.SetBinContent(b, misIDRatios[category][0])
    #print bin_names[b-1], value, valueO  
  pull_plot.Add(others_plot, -1)
  c = TCanvas("Plot", "Plot", 1920,1080)
  ROOT.gStyle.SetOptStat(0)
  true_plot.SetLineColor(ROOT.kRed)
  true_plot.SetLineWidth(3)
  true_plot.GetYaxis().SetRangeUser(-0.006, 0.006)
  true_plot.Draw()
  pull_plot.SetLineWidth(3)
  pull_plot.Draw("SAME")
  mydir = "pull_plots_gen/"
  mkdir_p(mydir)
  c.SaveAs("%s/%s_pulls.pdf" % (mydir, category))
  c.SaveAs("%s/%s_pulls.png" % (mydir, category))

def build_gdb13_data():
    atom_idxs = {'H': 0, 'C': 1, 'N': 2, 'O': 3, 'F': 4}
    base_path = os.path.join(DATA_BASE_DIR, "gdb13")
    mkdir_p(base_path)

    energies = []
    atom_counts = []
    for name in sorted(os.listdir(os.path.join(base_path, "xyz"))):
        xyz_path = os.path.join(base_path, "xyz", name)
        out_path = xyz_path.replace("xyz", "out")

        natoms = 0
        energy = None
        counts = [0 for _ in atom_idxs]
        with open(xyz_path, 'r') as xyz_f, open(out_path, 'w') as out_f:
            for i, line in enumerate(xyz_f):
                line = line.strip()
                if not i:
                    natoms = int(line)
                elif i == 1:
                    energy = float(line.split()[-3])
                elif i - 2 < natoms:
                    line = line.replace("*^", "e")
                    ele, x, y, z, _ = line.split()
                    counts[atom_idxs[ele]] += 1
                    out_f.write("%s %.8f %.8f %.8f\n" % (ele, float(x), float(y), float(z)))
        energies.append(energy)
        atom_counts.append(counts)
    atom_counts = numpy.matrix(atom_counts)
    atomization = calculate_atomization_energies(atom_counts, numpy.matrix(energies).T)
    atomization *= HARTREE_TO_KCAL
    numpy.savetxt(os.path.join(base_path, "energies.txt"), atomization)
    numpy.savetxt(os.path.join(base_path, "heavy_counts.txt"), atom_counts.sum(1))

def saveTime(time):

    dir_ = './.Scores/'
    utils.mkdir_p(dir_)
    f = open(dir_ + '.Scores.txt', 'a')
    f.write(str(time) + '\n')
    f.close()

  def sweep():
    to_check = []

    bioguide = utils.flags().get('bioguide', None)
    if bioguide:
      possibles = [bioguide]
    else:
      possibles = current_bioguide.keys()

    for bioguide in possibles:
      if media_bioguide.get(bioguide, None) is None:
        to_check.append(bioguide)
      elif media_bioguide[bioguide]["social"].get(service, None) is None:
        to_check.append(bioguide)
      else:
        pass

    utils.mkdir_p("cache/social_media")
    writer = csv.writer(open("cache/social_media/%s_candidates.csv" % service, 'w'))
    writer.writerow(["bioguide", "official_full", "website", "service", "candidate", "candidate_url"])

    if len(to_check) > 0:
      email_body = "Social media leads found:\n\n"
      for bioguide in to_check:
        candidate = candidate_for(bioguide)
        if candidate:
          url = current_bioguide[bioguide]["terms"][-1].get("url", None)
          candidate_url = "https://%s.com/%s" % (service, candidate)
          row = [bioguide, current_bioguide[bioguide]['name']['official_full'].encode('utf-8'), url, service, candidate, candidate_url]
          writer.writerow(row)
          print "\tWrote: %s" % candidate
          email_body += ("%s\n" % row)

      if email_enabled:
        utils.send_email(email_body)

def main():
  utils.Initialize()
  e_name_list = gflags.FLAGS.extractors

  new_experiment = experiments.StartNewExperiment(e_name_list)
  experiment_id = new_experiment.GetID()
  utils.mkdir_p(gflags.FLAGS.reports_dir)
  utils.mkdir_p(gflags.FLAGS.models_dir)
  report_loc = path.join(gflags.FLAGS.reports_dir, "%.3d.html" % experiment_id)
  model_loc = path.join(gflags.FLAGS.models_dir, "%.3d.model" % experiment_id)

  print "Experiment ID: %d. Detailed report at %s. Model at %s\n" % (
      experiment_id,
      report_loc,
      model_loc,
  )
  
  cv_data = LoadCVData()
  hd_data = LoadHDData()

  new_experiment.RunCrossValidation(cv_data)

  model = models.BuildModel(e_name_list, cv_data)
  model.Save(model_loc)
  hd_result = model.EvaluateOn(hd_data)
  new_experiment.RecordHeldoutDataEval(hd_result)

  new_experiment.Save()
  new_experiment.PrintSummary()
  new_experiment.ExportReport(report_loc)

def process_clip(clip_name, paths, img_type, negative_images, overwrite=True):
    # overwrite: overwrite the training of the FFLD model.
    print(clip_name)
    frames_path = paths['clips'] + frames + clip_name + sep
    if not check_path_and_landmarks(frames_path, clip_name, paths['in_bb'] + clip_name + sep):  # check that paths, landmarks exist
        return
    list_frames = sorted(os.listdir(frames_path))
    save_model = paths['out_model'] + clip_name + '.model'
    if (not os.path.exists(save_model)) or overwrite:
        # build the detector
        training_pos = load_images(list_frames, frames_path, paths['in_bb'], clip_name, max_images=400)
        if len(training_pos) == 0:
            print('No positives found for the clip {}, skipping it.'.format(clip_name))
            return
        ps_model = train_ffld2_detector(training_pos, negative_images, n_components=1, n_relabel=6)
        ps_model.save(save_model)
    else:
        print('The model {} already exists and was loaded from disk.'.format(save_model))
        ps_model = load_model(save_model)
    global detector
    detector = FFLD2Detector(ps_model)

    p_det_bb = mkdir_p(paths['out_bb'] + clip_name + sep)
    p_det_landm = mkdir_p(paths['out_lns'] + clip_name + sep)
    clip = Clip(clip_name, paths['clips'], frames, write_ln=[p_det_bb, p_det_landm])
    # TODO: Try parallel model
    [predict_in_frame(frame_name, clip, img_type) for frame_name in list_frames]

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--image_list', required=True)
    parser.add_argument('--config', required=True)
    parser.add_argument('--dump_prefix', required=True)
    parser.add_argument('--output_dir', required=True)
    args = parser.parse_args()

    conf_mod = imp.load_source('config', args.config)
    config = conf_mod.get()

    model = config['model']
    utils.load_model(model, args.dump_prefix)

    X, _ = conf_mod.get_data(args.image_list)

    utils.mkdir_p(args.output_dir)
    image_list = utils.read_image_list(args.image_list)

    logger.info('compiling model ...')
    model.compile(loss='mean_squared_error', optimizer=Adam())

    for x, (input_path, _) in ProgressBar()(zip(X, image_list)):
        y = model.predict(np.array([x], dtype='float32'),
                               batch_size=1, verbose=False)
        img = np.round(y.reshape(y.shape[2:]) * 255.0).astype('uint8')

        # FIXME: we assume that basenames of images are distinct
        fname = os.path.basename(input_path)
        output_path = os.path.join(args.output_dir, fname)
        cv2.imwrite(output_path, img)

 def test_mkdir_p(self):
     d = mkdtemp()
     path = os.path.join(d, 'a/b/c')
     mkdir_p(path)
     self.assertTrue(os.path.isdir(path))
     mkdir_p(path)  # Already exists test
     rmtree(d)

    def create_test_case(self):
        """
        Create a test case.
        """
        testyml_template = default_testyml_template.replace(
            "%role", self.normalized_role)
        testyml_template = testyml_template.replace(
            "%year", str(date.today().year))
        testyml_template = testyml_template.replace(
            "%author", self.config["author_name"])
        testyml_template = testyml_template.replace(
            "%email", self.config["author_email"])

        utils.mkdir_p(os.path.join(self.output_path,
                                   "tests", "inventory", "group_vars"))

        utils.mkdir_p(os.path.join(self.output_path,
                                   "tests", "inventory", "host_vars"))

        hosts = "placeholder_fqdn\n"
        utils.string_to_file(os.path.join(self.output_path,
                                          "tests", "inventory", "hosts"),
                             hosts)

        test_file = os.path.join(self.output_path, "tests", "test")
        utils.string_to_file(test_file, testyml_template)

        os.chmod(test_file, 0755)

 def user_login(self, user):
     """
     Called immediately after a user authenticates successfully.  Saves
     session information in the user's directory.  Expects *user* to be a
     dict containing a 'upn' value representing the username or
     userPrincipalName. e.g. '[email protected]' or just 'someuser'.  Any additional
     values will be attached to the user object/cookie.
     """
     logging.debug("user_login(%s)" % user["upn"])
     user.update(additional_attributes(user))
     # Make a directory to store this user's settings/files/logs/etc
     user_dir = os.path.join(self.settings["user_dir"], user["upn"])
     if not os.path.exists(user_dir):
         logging.info(_("Creating user directory: %s" % user_dir))
         mkdir_p(user_dir)
         os.chmod(user_dir, 0o700)
     session_file = os.path.join(user_dir, "session")
     session_file_exists = os.path.exists(session_file)
     if session_file_exists:
         session_data = open(session_file).read()
         try:
             session_info = tornado.escape.json_decode(session_data)
         except ValueError:  # Something wrong with the file
             session_file_exists = False  # Overwrite it below
     if not session_file_exists:
         with open(session_file, "w") as f:
             # Save it so we can keep track across multiple clients
             session_info = {"session": generate_session_id()}
             session_info.update(user)
             session_info_json = tornado.escape.json_encode(session_info)
             f.write(session_info_json)
     self.set_secure_cookie("gateone_user", tornado.escape.json_encode(session_info))

def download_cfo(options):
    if options.get('loglevel', None):
        log.setLevel(options['loglevel'])

    OUT_DIR = os.path.join(CACHE_DIR, 'cfo')
    if not os.path.exists(OUT_DIR):
        mkdir_p(OUT_DIR)

    base_url = 'https://www.campaignfinanceonline.state.pa.us/pages/CFAnnualTotals.aspx'

    def _get_response_loc_pair(dl_info):
        filer_id = dl_info
        loc = os.path.join(OUT_DIR, '{}.html'.format(filer_id))
        response = requests.get(base_url, params={'Filer': filer_id})
        return (response, loc)

    filer_ids = set([])

    for loc in iglob(os.path.join(
            CACHE_DIR, 'dos', '*', '*', '[fF]iler.[Tt]xt')):
        with open(loc, 'r') as fin:
            for row in csv.reader(fin):
                if row[0]:
                    filer_ids.add(row[0])

    download_all(list(filer_ids), _get_response_loc_pair, options)

def processResults(df, features, output_path, filename):
    TeamObjects = {TeamsDict['Team_Id'][k] : {'Team_Name' : TeamsDict['Team_Name'][k]} for k in TeamsDict['Team_Id']}
    for season in range(2003,2016):
        for k in TeamObjects:
            for f in features:
                TeamObjects[k][f] = 0
            TeamObjects[k]['GameCount'] = 0

        for index, game in df[df.Season == season].iterrows():
            d = game.to_dict()
            Wteam = d['Wteam']
            Lteam = d['Lteam']

            for f in features:
                if f.startswith('W'):
                    TeamObjects[Wteam][f] += d[f.replace('Avg', '')]
                if f.startswith('L'):
                    TeamObjects[Lteam][f] += d[f.replace('Avg', '')]

            TeamObjects[Wteam]['GameCount'] += 1
            TeamObjects[Lteam]['GameCount'] += 1

        for k in TeamObjects:
            for f in features:
                if TeamObjects[k]['GameCount'] > 0:
                    TeamObjects[k][f] /= TeamObjects[k]['GameCount']

        TeamStats = pandas.DataFrame.from_dict(TeamObjects, orient='index')
        mkdir_p(output_path)
        TeamStats.to_csv(output_path + filename + str(season) + '.csv')
        print('Wrote out ' + output_path + filename + str(season) + '.csv')

 def user_login(self, user):
     """
     Called immediately after a user authenticates successfully.  Saves
     session information in the user's directory.  Expects *user* to be a
     string containing the username or userPrincipalName. e.g. '[email protected]'
     or just 'someuser'.
     """
     logging.debug("user_login(%s)" % user)
     # Make a directory to store this user's settings/files/logs/etc
     user_dir = os.path.join(self.settings['user_dir'], user)
     if not os.path.exists(user_dir):
         logging.info(_("Creating user directory: %s" % user_dir))
         mkdir_p(user_dir)
         os.chmod(user_dir, 0o700)
     session_file = os.path.join(user_dir, 'session')
     session_file_exists = os.path.exists(session_file)
     if session_file_exists:
         session_data = open(session_file).read()
         try:
             session_info = tornado.escape.json_decode(session_data)
         except ValueError: # Something wrong with the file
             session_file_exists = False # Overwrite it below
     if not session_file_exists:
         with open(session_file, 'w') as f:
             # Save it so we can keep track across multiple clients
             session_info = {
                 'upn': user, # FYI: UPN == userPrincipalName
                 'session': generate_session_id()
             }
             session_info_json = tornado.escape.json_encode(session_info)
             f.write(session_info_json)
     self.set_secure_cookie(
         "gateone_user", tornado.escape.json_encode(session_info))

def backup(path, password_file=None):
    """
    Replaces the contents of a file with its decrypted counterpart, storing the
    original encrypted version and a hash of the file contents for later
    retrieval.
    """
    vault = VaultLib(get_vault_password(password_file))
    with open(path, 'r') as f:
        encrypted_data = f.read()

        # Normally we'd just try and catch the exception, but the
        # exception raised here is not very specific (just
        # `AnsibleError`), so this feels safer to avoid suppressing
        # other things that might go wrong.
        if vault.is_encrypted(encrypted_data):
            decrypted_data = vault.decrypt(encrypted_data)

            # Create atk vault files
            atk_path = os.path.join(ATK_VAULT, path)
            mkdir_p(atk_path)
            # ... encrypted
            with open(os.path.join(atk_path, 'encrypted'), 'wb') as f:
                f.write(encrypted_data)
            # ... hash
            with open(os.path.join(atk_path, 'hash'), 'wb') as f:
                f.write(hashlib.sha1(decrypted_data).hexdigest())

            # Replace encrypted file with decrypted one
            with open(path, 'wb') as f:
                f.write(decrypted_data)

def actually_create_android_project(package_name, sdk_version, java_package_name, is_library):
    path = os.path.join(os.getcwd(), package_name.lower())
    console.pretty_println("\nCreating android project ", console.bold)
    console.pretty_print("  Name      : ", console.cyan)
    console.pretty_println("%s" % package_name, console.yellow)
    console.pretty_print("  Sdk Ver   : ", console.cyan)
    console.pretty_println("%s" % sdk_version, console.yellow)
    console.pretty_print("  Java Name : ", console.cyan)
    console.pretty_println("%s" % java_package_name, console.yellow)
    if is_library:
        console.pretty_print("  Library   : ", console.cyan)
        console.pretty_println("yes\n", console.yellow)
        cmd = ['android', 'create', 'lib-project', '-n', package_name, '-p', path, '-k', java_package_name, '-t', 'android-' + sdk_version, ]
    else:
        activity_name = utils.camel_case(package_name)
        console.pretty_print("  Activity  : ", console.cyan)
        console.pretty_println("%s\n" % activity_name, console.yellow)
        cmd = ['android', 'create', 'project', '-n', package_name, '-p', path, '-k', java_package_name, '-t', 'android-' + sdk_version, '-a', activity_name]
    try:
        subprocess.check_call(cmd)
    except subprocess.CalledProcessError:
        raise subprocess.CalledProcessError("failed to create android project.")
    # This is in the old form, let's shovel the shit around to the new form
    utils.mkdir_p(os.path.join(path, 'src', 'main', 'java'))
    os.remove(os.path.join(path, 'local.properties'))
    os.remove(os.path.join(path, 'project.properties'))
    os.remove(os.path.join(path, 'ant.properties'))
    os.remove(os.path.join(path, 'proguard-project.txt'))
    os.remove(os.path.join(path, 'build.xml'))
    os.rmdir(os.path.join(path, 'bin'))
    os.rmdir(os.path.join(path, 'libs'))
    shutil.move(os.path.join(path, 'AndroidManifest.xml'), os.path.join(path, 'src', 'main'))
    shutil.move(os.path.join(path, 'res'), os.path.join(path, 'src', 'main'))
    if not is_library:
        shutil.move(os.path.join(path, 'src', java_package_name.split('.')[0]), os.path.join(path, 'src', 'main', 'java'))

    def setup(self):
        """
        Create a working directory and some test files
        """
        self.working_dir = tempfile.mkdtemp()
        self.file_contents = collections.OrderedDict.fromkeys([
            'file.test',
            '1/file.test',
            '2/2/file.test',
        ])
        self.file_timestamps = self.file_contents.copy()

        # create a key for the tests
        self.key = _create_key(None, write=False)

        # setup files in subdirectory
        for path in self.file_contents.keys():
            # create file content
            self.file_contents[path] = str(uuid.uuid4())

            abspath = os.path.join(self.working_dir, path)

            # create subdirs as necessary
            mkdir_p(os.path.dirname(abspath))

            # create test file in dir
            with open(abspath, 'w') as f:
                f.write(self.file_contents[path])

            # record file creation time
            self.file_timestamps[path] = os.stat(abspath).st_ctime

def main():
    """
    Main entry point for execution as a program (instead of as a module).
    """

    args = parse_args()
    completed_classes = []

    mkdir_p(PATH_CACHE, 0o700)
    if args.clear_cache:
        shutil.rmtree(PATH_CACHE)
    if args.on_demand:
        logging.warning(
            "--on-demand option is deprecated and is not required"
            " anymore. Do not use this option. It will be removed"
            "in the future."
        )

    for class_name in args.class_names:
        try:
            logging.info("Downloading class: %s", class_name)
            if download_class(args, class_name):
                completed_classes.append(class_name)
        except requests.exceptions.HTTPError as e:
            logging.error("HTTPError %s", e)
        except ClassNotFound as cnf:
            logging.error("Could not find class: %s", cnf)
        except AuthenticationFailed as af:
            logging.error("Could not authenticate: %s", af)

    if completed_classes:
        logging.info("Classes which appear completed: " + " ".join(completed_classes))

def make_pull_plot(category, misIDRatios, catRatios, datastring, fitname, fittype):
  pull_plot = TH1D(category, category, 6, 0, 6 );
  others_plot = TH1D(category+"others", category+"others", 6, 0, 6 );
  bin_names = get_all_containers(category)
  for b in range(1, len(bin_names)+1):
    pull_plot.GetXaxis().SetBinLabel(b,bin_names[b-1])
    others_plot.GetXaxis().SetBinLabel(b,bin_names[b-1])
    (value, err) = catRatios[get_bin_nr_composite(bin_names[b-1])]
    pull_plot.SetBinContent(b, value)
    pull_plot.SetBinError(b, err)

    other = get_other_component(category, bin_names[b-1])
    (valueO, errO) = misIDRatios[get_bin_nr_single(other)]
    others_plot.SetBinContent(b, valueO)
    others_plot.SetBinError(b, errO)
    #print bin_names[b-1], value, valueO  
  pull_plot.Add(others_plot, -1)
  c = TCanvas("Plot", "Plot", 800,600)
  ROOT.gStyle.SetOptStat(0)
  pull_plot.Draw()
  if len(fittype) == 0: fittype = "histograms"
  mydir = "pull_plots/%s/%s/%s/" % (fitname, fittype, datastring)
  mkdir_p(mydir)
  c.SaveAs("%s/%s_pulls.pdf" % (mydir, category))
  c.SaveAs("%s/%s_pulls.png" % (mydir, category))

  def sweep():
    to_check = []

    bioguide = utils.flags().get('bioguide', None)
    if bioguide:
      possibles = [bioguide]
    else:
      possibles = current_bioguide.keys()

    for bioguide in possibles:
      if media_bioguide.get(bioguide, None) is None:
        to_check.append(bioguide)
      elif media_bioguide[bioguide]["social"].get(service, None) is None:
        to_check.append(bioguide)
      else:
        pass

    utils.mkdir_p("cache/social_media")
    writer = csv.writer(open("cache/social_media/%s_candidates.csv" % service, 'w'))
    writer.writerow(["bioguide", "official_full", "website", "service", "candidate"])

    for bioguide in to_check:
      candidate = candidate_for(bioguide)
      if candidate:
        url = current_bioguide[bioguide]["terms"][-1].get("url", None)
        writer.writerow([bioguide, current_bioguide[bioguide]['name']['official_full'], url, service, candidate])
        print "\tWrote: %s" % candidate