def brickfind_crawl(brick, args):
    if brick.endswith("/"):
        brick = brick[0:len(brick)-1]

    working_dir = os.path.dirname(args.outfile)
    mkdirp(working_dir, exit_on_err=True, logger=logger)
    create_file(args.outfile, exit_on_err=True, logger=logger)

    with open(args.outfile, "a+") as fout:
        brick_path_len = len(brick)

        def output_callback(path, filter_result, is_dir):
            path = path.strip()
            path = path[brick_path_len+1:]

            if args.type == "both":
                output_write(fout, path, args.output_prefix,
                             encode=(not args.no_encode), tag=args.tag,
                             field_separator=args.field_separator)
            else:
                if (is_dir and args.type == "d") or (
                    (not is_dir) and args.type == "f"):
                    output_write(fout, path, args.output_prefix,
                    encode=(not args.no_encode), tag=args.tag,
                    field_separator=args.field_separator)

        ignore_dirs = [os.path.join(brick, dirname)
                       for dirname in
                       conf.get_opt("brick_ignore_dirs").split(",")]

        find(brick, callback_func=output_callback,
             ignore_dirs=ignore_dirs)

        fout.flush()
        os.fsync(fout.fileno())

    def __init__(self, root_folder, channel, version, compress, pretty_print, cached = False):
        self.data_folder = os.path.join(root_folder, channel, version)
        mkdirp(self.data_folder)
        self.compress = compress
        self.pretty_print = pretty_print
        self.max_filter_id = None
        self.cached = cached
        if cached:
            self.cache = {}

        # Load filter-tree
        self.filter_tree = self.json_from_file(
            "filter-tree.json",
            {'_id': 0, 'name': 'reason'}
        )

        # Load histogram definitions
        self.histograms = self.json_from_file("histograms.json", {})

        # Load histogram revision meta-data
        self.revisions = self.json_from_file("revisions.json", {})

        # Histograms.json cache
        self.histograms_json_cache = [(None, None)] * HGRAMS_JSON_CACHE_SIZE
        self.histograms_json_cache_next = 0

def mode_create(session_dir, args):
    validate_session_name(args.session)

    logger.debug("Init is called - Session: %s, Volume: %s"
                 % (args.session, args.volume))
    mkdirp(session_dir, exit_on_err=True, logger=logger)
    mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True,
           logger=logger)
    status_file = os.path.join(session_dir, args.volume, "status")

    if os.path.exists(status_file) and not args.force:
        fail("Session %s already created" % args.session, logger=logger)

    if not os.path.exists(status_file) or args.force:
        ssh_setup(args)
        enable_volume_options(args)

    # Add Rollover time to current time to make sure changelogs
    # will be available if we use this time as start time
    time_to_update = int(time.time()) + get_changelog_rollover_time(
        args.volume)

    run_cmd_nodes("create", args, time_to_update=str(time_to_update))

    if not os.path.exists(status_file) or args.reset_session_time:
        with open(status_file, "w") as f:
            f.write(str(time_to_update))

    sys.stdout.write("Session %s created with volume %s\n" %
                     (args.session, args.volume))

    sys.exit(0)

def mode_pre(session_dir, args):
    global gtmpfilename

    """
    Read from Session file and write to session.pre file
    """
    endtime_to_update = int(time.time()) - get_changelog_rollover_time(
        args.volume)
    status_file = os.path.join(session_dir, args.volume, "status")
    status_file_pre = status_file + ".pre"

    mkdirp(os.path.dirname(args.outfile), exit_on_err=True, logger=logger)

    # If Pre status file exists and running pre command again
    if os.path.exists(status_file_pre) and not args.regenerate_outfile:
        fail("Post command is not run after last pre, "
             "use --regenerate-outfile")

    start = 0
    try:
        with open(status_file) as f:
            start = int(f.read().strip())
    except ValueError:
        pass
    except (OSError, IOError) as e:
        fail("Error Opening Session file %s: %s"
             % (status_file, e), logger=logger)

    logger.debug("Pre is called - Session: %s, Volume: %s, "
                 "Start time: %s, End time: %s"
                 % (args.session, args.volume, start, endtime_to_update))

    prefix = datetime.now().strftime("%Y%m%d-%H%M%S-%f-")
    gtmpfilename = prefix + next(tempfile._get_candidate_names())

    run_cmd_nodes("pre", args, start=start, end=-1, tmpfilename=gtmpfilename)

    # Merger
    if args.full:
        cmd = ["sort", "-u"] + node_outfiles + ["-o", args.outfile]
        execute(cmd,
                exit_msg="Failed to merge output files "
                "collected from nodes", logger=logger)
    else:
        # Read each Changelogs db and generate finaldb
        create_file(args.outfile, exit_on_err=True, logger=logger)
        outfilemerger = OutputMerger(args.outfile + ".db", node_outfiles)
        write_output(args.outfile, outfilemerger, args.field_separator)

    try:
        os.remove(args.outfile + ".db")
    except (IOError, OSError):
        pass

    run_cmd_nodes("cleanup", args, tmpfilename=gtmpfilename)

    with open(status_file_pre, "w", buffering=0) as f:
        f.write(str(endtime_to_update))

    sys.stdout.write("Generated output file %s\n" % args.outfile)

def cook(path, caller_cwd):

    def delete_if_exists(path):
        if os.path.isfile(path):
            os.remove(path)

    local_cwd = os.getcwd()
    # Check if `path` is an absolute path to the recipe
    if os.path.isabs(path):
        recipe_path = os.path.realpath(path)
        recipe_basename = os.path.basename(recipe_path)
        mkdirp('.recipes')
        delete_if_exists(os.path.join(local_cwd, '.recipes', recipe_basename))
        shutil.copyfile(recipe_path, os.path.join(local_cwd, '.recipes', recipe_basename))
        recipe_path = os.path.join('/vagrant', '.recipes', recipe_basename)
    # Check if `path` is a relative path to the recipe (from the caller's perspective)
    elif os.path.isfile(os.path.realpath(os.path.join(caller_cwd, path))):
        recipe_path = os.path.realpath(os.path.join(caller_cwd, path))
        recipe_basename = os.path.basename(recipe_path)
        mkdirp('.recipes')
        delete_if_exists(os.path.join(local_cwd, '.recipes', recipe_basename))
        shutil.copyfile(recipe_path, os.path.join(local_cwd, '.recipes', recipe_basename))
        recipe_path = os.path.join('/vagrant', '.recipes', recipe_basename)
    # Check if `path + (.sh)` is a relative path to the recipe (from the dev-box's perspective)
    elif os.path.isfile(os.path.realpath(os.path.join(local_cwd, 'recipes', path + '.sh'))):
        recipe_path = os.path.realpath(os.path.join(local_cwd, 'recipes', path + '.sh'))
        recipe_basename = os.path.basename(recipe_path)
        recipe_path = os.path.join('/vagrant', 'recipes', recipe_basename)
    # Recipe file was not found
    else:
        print_error('Error: recipe was not found')
        return
    print_green('# DevBox is now cooking')
    return run('sh {0}'.format(recipe_path))

def main():
    args = _get_args()
    mkdirp(conf.get_opt("session_dir"), exit_on_err=True)

    if args.mode == "list":
        session_dir = conf.get_opt("session_dir")
    else:
        session_dir = os.path.join(conf.get_opt("session_dir"),
                                   args.session)

    if not os.path.exists(session_dir) and args.mode not in ["create", "list"]:
        fail("Invalid session %s" % args.session)

    vol_dir = os.path.join(session_dir, args.volume)
    if not os.path.exists(vol_dir) and args.mode not in ["create", "list"]:
        fail("Session %s not created with volume %s" %
            (args.session, args.volume))

    mkdirp(os.path.join(conf.get_opt("log_dir"), args.session, args.volume),
           exit_on_err=True)
    log_file = os.path.join(conf.get_opt("log_dir"),
                            args.session,
                            args.volume,
                            "cli.log")
    setup_logger(logger, log_file, args.debug)

    # globals() will have all the functions already defined.
    # mode_<args.mode> will be the function name to be called
    globals()["mode_" + args.mode](session_dir, args)

def changelog_crawl(brick, end, args):
    """
    Init function, prepares working dir and calls Changelog query
    """
    if brick.endswith("/"):
        brick = brick[0:len(brick)-1]

    # WORKING_DIR/BRICKHASH/OUTFILE
    working_dir = os.path.dirname(args.outfile)
    brickhash = hashlib.sha1(brick)
    brickhash = str(brickhash.hexdigest())
    working_dir = os.path.join(working_dir, brickhash)

    mkdirp(working_dir, exit_on_err=True, logger=logger)
    create_file(args.outfile, exit_on_err=True, logger=logger)
    create_file(args.outfile + ".gfids", exit_on_err=True, logger=logger)

    log_file = os.path.join(conf.get_opt("log_dir"),
                            args.session,
                            args.volume,
                            "changelog.%s.log" % brickhash)

    logger.info("%s Started Changelog Crawl. Start: %s, End: %s"
                % (brick, args.start, end))
    get_changes(brick, working_dir, log_file, end, args)

def mode_pre(session_dir, args):
    """
    Read from Session file and write to session.pre file
    """
    endtime_to_update = int(time.time()) - int(
        conf.get_opt("changelog_rollover_time"))
    status_file = os.path.join(session_dir, args.volume, "status")
    status_file_pre = status_file + ".pre"

    mkdirp(os.path.dirname(args.outfile), exit_on_err=True, logger=logger)

    start = 0
    try:
        with open(status_file) as f:
            start = int(f.read().strip())
    except ValueError:
        pass
    except (OSError, IOError) as e:
        fail("Error Opening Session file %s: %s"
             % (status_file, e), logger=logger)

    logger.debug("Pre is called - Session: %s, Volume: %s, "
                 "Start time: %s, End time: %s"
                 % (args.session, args.volume, start, endtime_to_update))

    run_in_nodes(args.volume, start, args)

    with open(status_file_pre, "w", buffering=0) as f:
        f.write(str(endtime_to_update))

    sys.stdout.write("Generated output file %s\n" % args.outfile)

def brickfind_crawl(brick, args):
    if brick.endswith("/"):
        brick = brick[0:len(brick)-1]

    working_dir = os.path.dirname(args.outfile)
    mkdirp(working_dir, exit_on_err=True, logger=logger)
    create_file(args.outfile, exit_on_err=True, logger=logger)

    with open(args.outfile, "a+") as fout:
        brick_path_len = len(brick)

        def output_callback(path, filter_result):
            path = path.strip()
            path = path[brick_path_len+1:]
            output_write(fout, path, args.output_prefix, encode=True)

        ignore_dirs = [os.path.join(brick, dirname)
                       for dirname in
                       conf.get_opt("brick_ignore_dirs").split(",")]

        find(brick, callback_func=output_callback,
             ignore_dirs=ignore_dirs)

        fout.flush()
        os.fsync(fout.fileno())

def main():
    global gtmpfilename

    args = None

    try:
        args = _get_args()
        mkdirp(conf.get_opt("session_dir"), exit_on_err=True)

        # force the default session name if mode is "query"
        if args.mode == "query":
            args.session = "default"

        if args.mode == "list":
            session_dir = conf.get_opt("session_dir")
        else:
            session_dir = os.path.join(conf.get_opt("session_dir"),
                                       args.session)

        if not os.path.exists(session_dir) and \
                args.mode not in ["create", "list", "query"]:
            fail("Invalid session %s" % args.session)

        # "default" is a system defined session name
        if args.mode in ["create", "post", "pre", "delete"] and \
                args.session == "default":
            fail("Invalid session %s" % args.session)

        vol_dir = os.path.join(session_dir, args.volume)
        if not os.path.exists(vol_dir) and args.mode not in \
                ["create", "list", "query"]:
            fail("Session %s not created with volume %s" %
                 (args.session, args.volume))

        mkdirp(os.path.join(conf.get_opt("log_dir"),
                            args.session,
                            args.volume),
               exit_on_err=True)
        log_file = os.path.join(conf.get_opt("log_dir"),
                                args.session,
                                args.volume,
                                "cli.log")
        setup_logger(logger, log_file, args.debug)

        # globals() will have all the functions already defined.
        # mode_<args.mode> will be the function name to be called
        globals()["mode_" + args.mode](session_dir, args)
    except KeyboardInterrupt:
        if args is not None:
            if args.mode == "pre" or args.mode == "query":
                # cleanup session
                if gtmpfilename is not None:
                    # no more interrupts until we clean up
                    signal.signal(signal.SIGINT, signal.SIG_IGN)
                    run_cmd_nodes("cleanup", args, tmpfilename=gtmpfilename)

        # Interrupted, exit with non zero error code
        sys.exit(2)

 def __init__(self, input_queue, output_queue,
                    work_folder, aws_cred):
     super(DownloaderProcess, self).__init__()
     self.input_queue = input_queue
     self.output_queue = output_queue
     self.work_folder = work_folder
     mkdirp(self.work_folder)
     self.input_bucket = "telemetry-published-v2"
     self.aws_cred = aws_cred
     self.s3 = S3Connection(**self.aws_cred)
     self.bucket = self.s3.get_bucket(self.input_bucket, validate = False)

def mode_post(args):
    session_dir = os.path.join(conf.get_opt("session_dir"), args.session)
    status_file = os.path.join(session_dir, args.volume,
                     "%s.status" % urllib.quote_plus(args.brick))

    mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True,
           logger=logger)
    status_file_pre = status_file + ".pre"

    if os.path.exists(status_file_pre):
        os.rename(status_file_pre, status_file)
        sys.exit(0)

def mode_create(session_dir, args):
    logger.debug("Init is called - Session: %s, Volume: %s"
                 % (args.session, args.volume))

    execute(["gluster", "volume", "info", args.volume],
            exit_msg="Unable to get volume details",
            logger=logger)

    mkdirp(session_dir, exit_on_err=True, logger=logger)
    mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True,
           logger=logger)
    status_file = os.path.join(session_dir, args.volume, "status")

    if os.path.exists(status_file) and not args.force:
        fail("Session %s already created" % args.session, logger=logger)

    if not os.path.exists(status_file) or args.force:
        ssh_setup(args)

        execute(["gluster", "volume", "set",
                 args.volume, "build-pgfid", "on"],
                exit_msg="Failed to set volume option build-pgfid on",
                logger=logger)
        logger.info("Volume option set %s, build-pgfid on" % args.volume)

        execute(["gluster", "volume", "set",
                 args.volume, "changelog.changelog", "on"],
                exit_msg="Failed to set volume option "
                "changelog.changelog on", logger=logger)
        logger.info("Volume option set %s, changelog.changelog on"
                    % args.volume)

        execute(["gluster", "volume", "set",
                 args.volume, "changelog.capture-del-path", "on"],
                exit_msg="Failed to set volume option "
                "changelog.capture-del-path on", logger=logger)
        logger.info("Volume option set %s, changelog.capture-del-path on"
                    % args.volume)

    # Add Rollover time to current time to make sure changelogs
    # will be available if we use this time as start time
    time_to_update = int(time.time()) + get_changelog_rollover_time(
        args.volume)

    run_cmd_nodes("create", args, time_to_update=str(time_to_update))

    if not os.path.exists(status_file) or args.reset_session_time:
        with open(status_file, "w", buffering=0) as f:
            f.write(str(time_to_update))

    sys.exit(0)

 def publish_results(self):
     # Create work folder for update process
     update_folder = os.path.join(self.work_folder, "update")
     shutil.rmtree(update_folder, ignore_errors = True)
     mkdirp(update_folder)
     # Update results
     updateresults(self.data_folder, update_folder, self.bucket_name,
                   self.prefix, self.cache_folder, self.region,
                   self.aws_cred, NB_WORKERS)
     self.put_file(self.files_processed_path, 'FILES_PROCESSED')
     self.put_file(self.files_missing_path, 'FILES_MISSING')
     # Clear data_folder
     shutil.rmtree(self.data_folder, ignore_errors = True)
     mkdirp(self.data_folder)

def mode_create(args):
    session_dir = os.path.join(conf.get_opt("session_dir"),
                               args.session)
    status_file = os.path.join(session_dir, args.volume,
                     "%s.status" % urllib.quote_plus(args.brick))

    mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True,
           logger=logger)

    if not os.path.exists(status_file) or args.reset_session_time:
        with open(status_file, "w", buffering=0) as f:
            f.write(args.time_to_update)

    sys.exit(0)

def check_coords(cubes, write_to="./.jazz/offending_cube"):
    """Check coordinates are matching.  If they are not this could be
    quite a problem!  However, some models' have files which read in with
    slightly different coordinates (CCSM4, for example).  In this case
    the difference is miniscule so we can safely replace the coordinates.
    This method replaces coordinates but also informs the user it is doing
    this. It also prints and optionally saves the summary of the offending
    cube.

    Args:
        cubes (iris.cube.CubeList): list of cubes to check
        write_to (Optional[str]): path to which to write warnings

    """
    # Remove attributes from auxiliary coordinates - these can sometimes
    # prevent merging and concatenation.
    for cube in cubes:
        for coord in cube.aux_coords:
            coord.attributes = {}

    # Get the names of the spatial coords
    coord_names = [coord.name() for coord in cubes[0].dim_coords]
    if "time" in coord_names:
        coord_names.remove("time")

    for coord_name in coord_names:
        # Make a list of the coordinates' points for each cube
        points_list = [cube.coord(coord_name).points for cube in cubes]

        # Loop over the list of points for all the cubes
        for p in xrange(len(points_list) - 1):

            # If the coordinates are different from the first set,
            # replace them with the first set
            if not (points_list[p + 1] == points_list[0]).all():
                cubes[p + 1].replace_coord(cubes[0].coord(coord_name))

                # Notify user
                warnings.warn(
                    "Replacing the coordinates of a cube. " "Offending cube is {}".format(cubes[p + 1].summary())
                )

                if write_to is not None:
                    utils.mkdirp(write_to)
                    utils.write_file(cubes[p + 1].summary(), "{0}_{1}_{2}".format(write_to, coord_name, p))

def render(pname):
    "renders the packer template and writes it's json to something like elife-builder/packer/pname.json"
    assert utils.mkdirp('packer'), "failed to create the 'packer' dir"
    out = json.dumps(render_template(pname), indent=4)
    fname = template_path(pname, '.json')
    open(fname, 'w').write(out)
    print out
    print 'wrote',fname
    return fname

def mode_create(session_dir, args):
    logger.debug("Init is called - Session: %s, Volume: %s"
                 % (args.session, args.volume))

    cmd = ["gluster", 'volume', 'info', args.volume, "--xml"]
    _, data, _ = execute(cmd,
                         exit_msg="Failed to Run Gluster Volume Info",
                         logger=logger)
    try:
        tree = etree.fromstring(data)
        statusStr = tree.find('volInfo/volumes/volume/statusStr').text
    except (ParseError, AttributeError) as e:
        fail("Invalid Volume: %s" % e, logger=logger)

    if statusStr != "Started":
        fail("Volume %s is not online" % args.volume, logger=logger)

    mkdirp(session_dir, exit_on_err=True, logger=logger)
    mkdirp(os.path.join(session_dir, args.volume), exit_on_err=True,
           logger=logger)
    status_file = os.path.join(session_dir, args.volume, "status")

    if os.path.exists(status_file) and not args.force:
        fail("Session %s already created" % args.session, logger=logger)

    if not os.path.exists(status_file) or args.force:
        ssh_setup(args)
        enable_volume_options(args)

    # Add Rollover time to current time to make sure changelogs
    # will be available if we use this time as start time
    time_to_update = int(time.time()) + get_changelog_rollover_time(
        args.volume)

    run_cmd_nodes("create", args, time_to_update=str(time_to_update))

    if not os.path.exists(status_file) or args.reset_session_time:
        with open(status_file, "w", buffering=0) as f:
            f.write(str(time_to_update))

    sys.stdout.write("Session %s created with volume %s\n" %
                     (args.session, args.volume))

    sys.exit(0)

def brickfind_crawl(brick, args):
    if brick.endswith("/"):
        brick = brick[0:len(brick)-1]

    working_dir = os.path.dirname(args.outfile)
    mkdirp(working_dir, exit_on_err=True, logger=logger)
    create_file(args.outfile, exit_on_err=True, logger=logger)

    with open(args.outfile, "a+") as fout:
        brick_path_len = len(brick)

        def mtime_filter(path):
            try:
                st = os.lstat(path)
            except (OSError, IOError) as e:
                if e.errno == ENOENT:
                    st = None
                else:
                    raise

            if st and (st.st_mtime > args.start or st.st_ctime > args.start):
                return True

            return False

        def output_callback(path):
            path = path.strip()
            path = path[brick_path_len+1:]
            output_write(fout, path, args.output_prefix)

        ignore_dirs = [os.path.join(brick, dirname)
                       for dirname in
                       conf.get_opt("brick_ignore_dirs").split(",")]

        if args.full:
            find(brick, callback_func=output_callback,
                 ignore_dirs=ignore_dirs)
        else:
            find(brick, callback_func=output_callback,
                 filter_func=mtime_filter,
                 ignore_dirs=ignore_dirs)

        fout.flush()
        os.fsync(fout.fileno())

def mode_cleanup(args):
    working_dir = os.path.join(conf.get_opt("working_dir"),
                               args.session,
                               args.volume)

    mkdirp(os.path.join(conf.get_opt("log_dir"), args.session, args.volume),
           exit_on_err=True)
    log_file = os.path.join(conf.get_opt("log_dir"),
                            args.session,
                            args.volume,
                            "changelog.log")

    setup_logger(logger, log_file)

    try:
        shutil.rmtree(working_dir, onerror=handle_rm_error)
    except (OSError, IOError) as e:
        logger.error("Failed to delete working directory: %s" % e)
        sys.exit(1)