def copy_proguard_mapping(flavor, version_name):
    folder_path = 'releases'
    sh.mkdir("-p", folder_path)
    output_file = '%s/wikipedia-%s.mapping.tar.gz' % (folder_path, version_name)
    input_file = 'wikipedia/build/outputs/mapping/%s/release/mapping.txt' % flavor
    sh.tar('czf', output_file, input_file)
    print ' proguard mapping: %s' % output_file

def download_package(destination, product, version, compiler):
  remove_existing_package(destination, product, version)

  label = get_release_label()
  file_name = "{0}-{1}-{2}-{3}.tar.gz".format(product, version, compiler, label)
  url_path="/{0}/{1}-{2}/{0}-{1}-{2}-{3}.tar.gz".format(product, version, compiler, label)
  download_path = HOST + url_path

  print "URL {0}".format(download_path)
  print "Downloading {0} to {1}".format(file_name, destination)
  # --no-clobber avoids downloading the file if a file with the name already exists
  sh.wget(download_path, directory_prefix=destination, no_clobber=True)
  print "Extracting {0}".format(file_name)
  sh.tar(z=True, x=True, f=os.path.join(destination, file_name), directory=destination)
  sh.rm(os.path.join(destination, file_name))

  if product == "kudu":
    # The Kudu tarball is actually a renamed parcel. Rename the contents to match the
    # naming convention.
    kudu_dirs = glob.glob("{0}/KUDU*{1}*".format(destination, version))
    if not kudu_dirs:
      raise Exception("Could not find contents of Kudu tarball")
    if len(kudu_dirs) > 1:
      raise Exception("Found too many Kudu folders: %s" % (kudu_dirs, ))
    new_dir = "{0}/{1}-{2}".format(destination, product, version)
    if os.path.exists(new_dir):
      shutil.rmtree(new_dir)
    os.rename(kudu_dirs[0], new_dir)

  write_version_file(destination, product, version, compiler, label)

def initialize():
    # noinspection PyUnresolvedReferences
    from sh import wget, tar, rm, shasum
    if not os.path.exists(prefix):
        os.makedirs(prefix)

    if (not os.path.exists(dirs['inputs'])) or (not os.path.exists(dirs['intermediates'])):
        try:
            if not os.path.exists(prefix):
                logger.info("Creating {DIR}".format(DIR=prefix))
                os.makedirs(prefix)
            logger.info("Downloading data from {URL} to {DIR}".format(URL=data_url, DIR=prefix))
            tar(wget(data_url, "-qO-", _piped=True), "xz", _cwd=prefix)
            logger.info("Checking checksums of downloaded files")
            for line in shasum("-c", _cwd=prefix, _in=checksums, _iter=True):
                logger.info(line)
        except Exception as e:
            logger.info("Error: {}".format(e.message))
            logger.info("Deleting {DIR}".format(DIR=dirs['inputs']))
            rm(dirs['inputs'], '-rf')
            logger.info("Deleting {DIR}".format(DIR=dirs['intermediates']))
            rm(dirs['intermediates'], '-rf')
            raise

    # make sure all those directories exist
    for d in (dirs['outputs'], dirs['plots']):
        if not os.path.exists(d):
            logger.info("Creating {DIR}".format(DIR=d))
            os.makedirs(d)

def packaging_lib(libmace_output_dir, project_name):
    print("* Package libs for %s" % project_name)
    tar_package_name = "libmace_%s.tar.gz" % project_name
    project_dir = "%s/%s" % (libmace_output_dir, project_name)
    tar_package_path = "%s/%s" % (project_dir, tar_package_name)
    if os.path.exists(tar_package_path):
        sh.rm("-rf", tar_package_path)

    print("Start packaging '%s' libs into %s" % (project_name,
                                                 tar_package_path))
    which_sys = platform.system()
    if which_sys == "Linux":
        sh.tar(
            "cvzf",
            "%s" % tar_package_path,
            glob.glob("%s/*" % project_dir),
            "--exclude",
            "%s/_tmp" % project_dir,
            _fg=True)
    elif which_sys == "Darwin":
        sh.tar(
            "--exclude",
            "%s/_tmp" % project_dir,
            "-cvzf",
            "%s" % tar_package_path,
            glob.glob("%s/*" % project_dir),
            _fg=True)
    print("Packaging Done!\n")
    return tar_package_path

def _archive(includes):
    tar_args = [git.archive('--format=tar','HEAD'),'czf','archive.tar.gz']
    for pttrn in includes:
        tar_args.append('--include')
        tar_args.append(pttrn)
    import debug
    tar_args.append('@-')
    tar(*tar_args)

def wget_and_unpack_package(download_path, file_name, destination, wget_no_clobber):
  print "URL {0}".format(download_path)
  print "Downloading {0} to {1}".format(file_name, destination)
  # --no-clobber avoids downloading the file if a file with the name already exists
  sh.wget(download_path, directory_prefix=destination, no_clobber=wget_no_clobber)
  print "Extracting {0}".format(file_name)
  sh.tar(z=True, x=True, f=os.path.join(destination, file_name), directory=destination)
  sh.rm(os.path.join(destination, file_name))

 def archive_repository(self, destination, *archive_args):
     try:
         (fd, tar_file) = tempfile.mkstemp()
         self.git.archive("remotes/origin/%s" % self.git_ref, *archive_args, _out=tar_file)
         sh.tar("xf", tar_file, _cwd=destination)
     finally:
         if tar_file:
             os.remove(tar_file)

def unpack(plugin):
    fname="../{pname}_{uversion}*.orig-{name}.tar*".format(pname=c.package, uversion=UPSTREAM_VERSION, name=plugin["Plugin"])
    fname = glob.glob(fname)[0]
    try:
        os.mkdir(plugin["Plugin"])
    except FileExistsError:
        shutil.rmtree(plugin["Plugin"])
        os.mkdir(plugin["Plugin"])
    sh.tar(["-C", plugin["Plugin"], "--strip-components=1", "-axf", fname])

    def test_make_tarball(self):
        """
        Can we make tarballs for all versions in the git repository.
        """

        experiment_repository = models.ExperimentRepository.objects.create(**self.setup_fields)

        for _hash, tstamp in experiment_repository.get_all_commits():
            experiment_details, tarball_path = utils.make_tarball(experiment_repository.path, _hash)

            ########
            # Test #
            ########

            # Do the experiment details look right?
            # Are the experiments mentioned in the settings file in the
            # experiment_details dir?
            experiments_settings = configobj.ConfigObj(
                os.path.join(experiment_repository.path, conf.repository_settings_filename)
            )

            for class_name in experiments_settings["experiments"]:
                self.assertTrue(experiment_details.has_key(class_name))

            ########
            # Test #
            ########

            # Are the contents of the tarball as they should be?
            tmpdir = tempfile.mkdtemp()

            if conf.tarball_compression_method == "bz2":
                tar_cmd = "-xjf"
            elif conf.tarball_compression_method == "gz":
                tar_cmd = "-xzf"

            sh.tar(tar_cmd, tarball_path, "-C", tmpdir)

            checksum_list = self.mock_repository.commit_dictionary[_hash]
            self.assertTrue(sys.check_directory_checksums(checksum_list, tmpdir))

            # Delete the exported directory.
            shutil.rmtree(tmpdir)

            ########
            # Test #
            ########

            # Check the contents of the tarball; does it import.
            tarball_model = utils.ExperimentArchiveTarball(tarball_path)
            self.assertTrue(tarball_model.integrity_check())
            self.assertTrue(tarball_model.import_check())

            # Delete the tarball
            os.unlink(tarball_path)

        models.ExperimentRepository.objects.all().delete()

def install_cmake( build_dir, prefix ):
    cmake_archive='cmake-2.8.11.2'
    sh.cd( build_dir )
    sh.wget( '-nc', 'http://www.cmake.org/files/v2.8/%s.tar.gz' % cmake_archive )
    sh.tar( 'xvzf', '%s.tar.gz' % cmake_archive )
    sh.cd( cmake_archive )
    subprocess.check_call( [ './configure', '--prefix', PREFIX ], shell = True )
    sh.make( '-j4' )
    sh.make.install()

    def setUp(self, *args, **kwargs):

        super(ChefPluginSoloTest, self).setUp(*args, **kwargs)

        self.blueprint_dir = self.copy_blueprint('chef-plugin')

        # Get resources
        with self.blueprint_dir:
            for res in 'cookbooks', 'data_bags', 'environments', 'roles':
                sh.tar('czf', res+'.tar.gz', res)

def script():
    """Run 'cfy logs get' and extract output to CONFIGURATION_DIR/logs"""
    logs_dir = cosmo.dir / 'logs'
    if logs_dir.exists():
        shutil.rmtree(logs_dir, ignore_errors=True)
    logs_dir.mkdir()
    logs_tar = logs_dir / 'logs.tar.gz'
    with logs_dir:
        cfy.logs.get(destination_path=logs_tar).wait()
        tar('xf', logs_tar, strip_components=1)
        logs_tar.remove()

def archiveDownload(url, destination, archiveType):
    logging.info('Now downloading archive file from URL %s to %s' % (url, destination))
    filename = wget.download(url)
    if archiveType == 'zip':
        logging.info('Unzipping zip file from: ', filename)
        sh.unzip(filename)
    elif archiveType == 'tar.gz':
        logging.info('Untarring tar.gz file from: ', filename)
        sh.tar('-xvzf', filename )
    logging.info('Removing archive file.')
    sh.rm(filename)
    return

    def handle(self, file_name, dest):
        dest = os.path.abspath(dest)
        from sh import unzip
        if not os.path.exists(dest):
            os.makedirs(dest)

        # Make moves
        tar(file_name, '-d', dest)

        if len(os.listdir(dest)) is 1:
            # somewhat properly packaged tarball
            dest = os.path.join(dest, os.listdir(dest).pop())
        return dest

def download_package(destination, product, version, compiler):
  label = get_release_label()
  file_name = "{0}-{1}-{2}-{3}.tar.gz".format(product, version, compiler, label)
  url_path="/{0}/{1}-{2}/{0}-{1}-{2}-{3}.tar.gz".format(product, version, compiler, label)
  download_path = HOST + url_path

  print "URL {0}".format(download_path)
  print "Downloading {0} to {1}".format(file_name, destination)
  # --no-clobber avoids downloading the file if a file with the name already exists
  sh.wget(download_path, directory_prefix=destination, no_clobber=True)
  print "Extracting {0}".format(file_name)
  sh.tar(z=True, x=True, f=os.path.join(destination, file_name), directory=destination)
  sh.rm(os.path.join(destination, file_name))

def download_package(name, destination, compiler=""):
    label = map_release_label()
    if len(compiler) > 0:
        compiler = "-" + compiler
    url = "{0}/{1}/label={2}/artifact/toolchain/build/{3}{4}.tar.gz".format(HOST, BUILD, label, name, compiler)

    # Download the file
    print "Downloading {0}".format(name)
    sh.wget(url, directory_prefix=destination, no_clobber=True)
    # Extract
    print "Extracting {0}".format(name)
    sh.tar(z=True, x=True, f="{0}/{1}{2}.tar.gz".format(destination, name, compiler), directory=destination)
    sh.rm("{0}/{1}{2}.tar.gz".format(destination, name, compiler))

def gitCheckout(version):
  directory = versionDirectory(version)
  if os.path.exists(directory):
    # already checked out
    # (since version is always a commit ID, the code can never have changed)
    return False
  else:
    os.makedirs(directory)
    # Make sure we're inside the git repository first
    os.chdir(config.PROJECT_ROOT)
    # how to checkout branch to new working directory, see:
    # http://blog.jessitron.com/2013/10/git-checkout-multiple-branches-at-same.html
    sh.tar(sh.git.archive(version), "-xC", directory)
    return True

def tgz_handle(self, file_name, dest):
    dest = os.path.abspath(dest)
    from sh import tar
    ops_flags = "zxvf"
    if not os.path.exists(dest):
        os.makedirs(dest)

    # Make moves
    tar(ops_flags, file_name, "-C", dest)

    if len(os.listdir(dest)) is 1:
        # somewhat properly packaged tarball
        dest = os.path.join(dest, os.listdir(dest).pop())
    return dest

    def setup_tomcat(self, args):
        try:
            log("Downloading Tomcat ...")
            result = wget(dict['TOMCAT_DOWNLOAD_URL'])
        except:
            log("Error getting Tomcat from : " + dict['TOMCAT_DOWNLOAD_URL'])

        try:
            log("Extracting Tomcat ...")            
            result = tar("xvzf " , dict['TOMCAT_VERSION']+ ".tar.gz")
        except:
            log("Error extracting Tomcat ..." + dict['TOMCAT_VERSION']+ ".tar.gz")
        
        setup_airavata_server(args)

        try:
            log("Copying the Airavata war files to Tomcat's webapp directory ...")
            result = cp(dict['AIRAVATA_VERSION']+ "/*.war " , dict['TOMCAT_VERSION']+ "/webapps", "-v")
        except:
            log("Error copying the Airavata war files to Tomcat's webapp directory ...")

        try :
            log("Granting executeable permissions to the script")
            result = chmod("a+x" , dict['TOMCAT_VERSION']+ "/*.sh")
        except:
            log("Error granting executable permissions to " + dict['TOMCAT_VERSION']+ "/*.sh")

def git_export(project, _hash):
    '''
    Export commit `_hash` of git project `project` into a temporary directory.
    Return the name of the temporary directory.
    The process requires tar-ing and untar-ing.
    '''
    tmpdir = tempfile.mkdtemp()
    tarball_name = os.path.join(tmpdir, 'exported_git_project.tar')
    assert os.path.exists(project) and os.path.isdir(project)
    git = sh.git.bake(_cwd=project)
    git('archive', _hash, '-o', tarball_name)
    export_tmpdir = tempfile.mkdtemp()
    sh.tar('-xf', tarball_name, '-C', export_tmpdir)
    shutil.rmtree(tmpdir) # Delete the tmpdir.

    return export_tmpdir