def update_cache(self):
        if not self.test_cache():
            rm(self.cache_dir, '-rf')
            self.cache_dir = mkdtemp()
            self.cache_uuid = uuid4()
            mkdir(os.path.join(self.cache_dir, 'repodata'))

            index_file_url = '/'.join([self.repo_url, self.index_file])
            index_file_path = os.path.join(self.cache_dir, self.index_file)

            try:
                print("Downloading index file '{0}' --> '{1}' ...".format(
                    index_file_url, index_file_path
                ))
                wget(index_file_url, '-O', index_file_path)
            except:
                self.broken = True
                return

            try:
                xmlroot = etree.parse(index_file_path).getroot()
                xmlns = xmlroot.nsmap[None]
                for item in xmlroot.findall("{{{0}}}data".format(xmlns)):
                    for subitem in item.findall("{{{0}}}location".format(xmlns)):
                        location = subitem.get('href')
                        url = '/'.join([self.repo_url, location])
                        path = '/'.join([self.cache_dir, location])
                        print("Downloading file '{0}' --> '{1}' ...".format(
                            url, path
                        ))
                        wget(url, '-O', path)
            except:
                self.broken = True

 def compile( self, source_dir, build_dir, install_dir ):
     package_source_dir = os.path.join( source_dir, self.dirname )
     assert( os.path.exists( package_source_dir ) )
     package_build_dir = os.path.join( build_dir, self.dirname )
     runpath_dir = os.path.join( package_source_dir, 'RunPath' )
     if ( not os.path.exists( os.path.join( runpath_dir, 'media.zip' ) ) ):
         sh.cd( runpath_dir )
         sh.wget( '--no-check-certificate', 'https://bitbucket.org/jacmoe/ogitor/downloads/media.zip' )
         sh.unzip( 'media.zip' )
     if ( not os.path.exists( os.path.join( runpath_dir, 'projects.zip' ) ) ):
         sh.cd( runpath_dir )
         sh.wget( '--no-check-certificate', 'https://bitbucket.org/jacmoe/ogitor/downloads/projects.zip' )
         sh.unzip( 'projects.zip' )
     sh.mkdir( '-p', package_build_dir )
     sh.cd( package_build_dir )
     if ( platform.system() == 'Darwin' ):
         sh.cmake(
             '-G', 'Xcode',
             '-D', 'CMAKE_INSTALL_PREFIX=%s' % install_dir,
             '-D', 'CMAKE_MODULE_PATH=%s' % os.path.join( install_dir, 'CMake' ),
             package_source_dir,
             _out = sys.stdout )
         sh.xcodebuild( '-configuration', 'Release', _out = sys.stdout )
     else:
         sh.cmake(
             '-D', 'CMAKE_INSTALL_PREFIX=%s' % install_dir,
             '-D', 'CMAKE_MODULE_PATH=%s' % os.path.join( install_dir, 'lib/OGRE/cmake' ),
             package_source_dir,
             _out = sys.stdout )
         sh.make( '-j4', 'VERBOSE=1', _out = sys.stdout )
         sh.make.install( _out = sys.stdout )

 def prebuild_arch(self, arch):
     if not self.is_patched(arch):
         super(ReportLabRecipe, self).prebuild_arch(arch)
         self.apply_patch('patches/fix-setup.patch', arch.arch)
         recipe_dir = self.get_build_dir(arch.arch)
         shprint(sh.touch, os.path.join(recipe_dir, '.patched'))
         ft = self.get_recipe('freetype', self.ctx)
         ft_dir = ft.get_build_dir(arch.arch)
         ft_lib_dir = os.environ.get('_FT_LIB_', os.path.join(ft_dir, 'objs', '.libs'))
         ft_inc_dir = os.environ.get('_FT_INC_', os.path.join(ft_dir, 'include'))
         tmp_dir = os.path.normpath(os.path.join(recipe_dir, "..", "..", "tmp"))
         info('reportlab recipe: recipe_dir={}'.format(recipe_dir))
         info('reportlab recipe: tmp_dir={}'.format(tmp_dir))
         info('reportlab recipe: ft_dir={}'.format(ft_dir))
         info('reportlab recipe: ft_lib_dir={}'.format(ft_lib_dir))
         info('reportlab recipe: ft_inc_dir={}'.format(ft_inc_dir))
         with current_directory(recipe_dir):
             sh.ls('-lathr')
             ensure_dir(tmp_dir)
             pfbfile = os.path.join(tmp_dir, "pfbfer-20070710.zip")
             if not os.path.isfile(pfbfile):
                 sh.wget("http://www.reportlab.com/ftp/pfbfer-20070710.zip", "-O", pfbfile)
             sh.unzip("-u", "-d", os.path.join(recipe_dir, "src", "reportlab", "fonts"), pfbfile)
             if os.path.isfile("setup.py"):
                 with open('setup.py', 'rb') as f:
                     text = f.read().replace('_FT_LIB_', ft_lib_dir).replace('_FT_INC_', ft_inc_dir)
                 with open('setup.py', 'wb') as f:
                     f.write(text)

    def download_image(self, image_id):
        """Download the image from the image_url, returning True/False if the operation
        was successful."""
        downloading_already = False
        with self._downloading_images_lock:
            downloading_already = (image_id in self._downloading_images)
            if not downloading_already:
                e = self._downloading_images[image_id] = threading.Event()
                e.clear()

        if downloading_already:
            self._log.debug("image is already being downloaded, waiting for it to finish")
            self._downloading_images[image_id].wait()
            self._log.debug("image finished downloading")
            return

        image_filename = image_id_to_volume(image_id)
        dest = os.path.join(LIBVIRT_BASE, image_filename)
        self._log.debug("downloading image {} to {}".format(image_id, dest))

        try:
            wget("-q", "-O", dest, self.image_url + "/" + image_filename)
        except:
            self._log.error("Could not download image! aborting running this VM")
            return False

        self._log.debug("downloaded {}".format(image_id))

        self._downloading_images[image_id].set()
        del self._downloading_images[image_id]

        return True

def download(target, temp_dir):
    zip_path = path.join(temp_dir, "temp.zip")
    tgt_path = path.join(temp_dir, "chunk")

    for chunk in CHUNKS:
        tif_name = TIF_FORMAT.format(chunk)
        tif_path = path.join(temp_dir, tif_name)

        wget(URL_FORMAT.format(chunk), q=True, O=zip_path)
        
        with zipfile.ZipFile(zip_path, 'r') as pack:
            contents = pack.namelist()
            if contents != [tif_name]:
                raise ValueError("Bad archive contents: {:r}".format(contents))
        
        unzip(zip_path, d=temp_dir)
        os.unlink(zip_path)

        convert(tif_path, '-quiet', 'GRAY:{}'.format(tgt_path))
        os.unlink(tif_path)

        if os.stat(tgt_path).st_size != EXPECT_SIZE:
            raise ValueError("Bad converted size: {}".format(chunk))

        with open(tgt_path, "rb") as f:
            shutil.copyfileobj(f, target)
        os.unlink(tgt_path)

def download_package(destination, product, version, compiler):
  remove_existing_package(destination, product, version)

  label = get_release_label()
  file_name = "{0}-{1}-{2}-{3}.tar.gz".format(product, version, compiler, label)
  url_path="/{0}/{1}-{2}/{0}-{1}-{2}-{3}.tar.gz".format(product, version, compiler, label)
  download_path = HOST + url_path

  print "URL {0}".format(download_path)
  print "Downloading {0} to {1}".format(file_name, destination)
  # --no-clobber avoids downloading the file if a file with the name already exists
  sh.wget(download_path, directory_prefix=destination, no_clobber=True)
  print "Extracting {0}".format(file_name)
  sh.tar(z=True, x=True, f=os.path.join(destination, file_name), directory=destination)
  sh.rm(os.path.join(destination, file_name))

  if product == "kudu":
    # The Kudu tarball is actually a renamed parcel. Rename the contents to match the
    # naming convention.
    kudu_dirs = glob.glob("{0}/KUDU*{1}*".format(destination, version))
    if not kudu_dirs:
      raise Exception("Could not find contents of Kudu tarball")
    if len(kudu_dirs) > 1:
      raise Exception("Found too many Kudu folders: %s" % (kudu_dirs, ))
    new_dir = "{0}/{1}-{2}".format(destination, product, version)
    if os.path.exists(new_dir):
      shutil.rmtree(new_dir)
    os.rename(kudu_dirs[0], new_dir)

  write_version_file(destination, product, version, compiler, label)

 def update_cache(self):
     if not self.test_cache():
         print("Downloading file ...")
         self.local_packages_gz = mkdtemp() + "/Packages.gz"
         try:
             wget(self.packages_gz_url, '-O', self.local_packages_gz)
         except:
             self.broken = True

def wget_and_unpack_package(download_path, file_name, destination, wget_no_clobber):
  print "URL {0}".format(download_path)
  print "Downloading {0} to {1}".format(file_name, destination)
  # --no-clobber avoids downloading the file if a file with the name already exists
  sh.wget(download_path, directory_prefix=destination, no_clobber=wget_no_clobber)
  print "Extracting {0}".format(file_name)
  sh.tar(z=True, x=True, f=os.path.join(destination, file_name), directory=destination)
  sh.rm(os.path.join(destination, file_name))

def install_cmake( build_dir, prefix ):
    cmake_archive='cmake-2.8.11.2'
    sh.cd( build_dir )
    sh.wget( '-nc', 'http://www.cmake.org/files/v2.8/%s.tar.gz' % cmake_archive )
    sh.tar( 'xvzf', '%s.tar.gz' % cmake_archive )
    sh.cd( cmake_archive )
    subprocess.check_call( [ './configure', '--prefix', PREFIX ], shell = True )
    sh.make( '-j4' )
    sh.make.install()

 def backups(self):
     data = self.http.load('backups', {'dato': 1})
     if data:
         dialogo = Alerta_Combo('Lista de Backup', 'backup.png', 'Escoja el backup que desea descargar:', data, liststore=(str, str))
         url = dialogo.iniciar()
         if url:
             print 'Backup'
             print url
             sh.wget(url)
             1/0
             sh.unzip()
     print data

def download_package(destination, product, version, compiler):
  label = get_release_label()
  file_name = "{0}-{1}-{2}-{3}.tar.gz".format(product, version, compiler, label)
  url_path="/{0}/{1}-{2}/{0}-{1}-{2}-{3}.tar.gz".format(product, version, compiler, label)
  download_path = HOST + url_path

  print "URL {0}".format(download_path)
  print "Downloading {0} to {1}".format(file_name, destination)
  # --no-clobber avoids downloading the file if a file with the name already exists
  sh.wget(download_path, directory_prefix=destination, no_clobber=True)
  print "Extracting {0}".format(file_name)
  sh.tar(z=True, x=True, f=os.path.join(destination, file_name), directory=destination)
  sh.rm(os.path.join(destination, file_name))

def download_package(name, destination, compiler=""):
    label = map_release_label()
    if len(compiler) > 0:
        compiler = "-" + compiler
    url = "{0}/{1}/label={2}/artifact/toolchain/build/{3}{4}.tar.gz".format(HOST, BUILD, label, name, compiler)

    # Download the file
    print "Downloading {0}".format(name)
    sh.wget(url, directory_prefix=destination, no_clobber=True)
    # Extract
    print "Extracting {0}".format(name)
    sh.tar(z=True, x=True, f="{0}/{1}{2}.tar.gz".format(destination, name, compiler), directory=destination)
    sh.rm("{0}/{1}{2}.tar.gz".format(destination, name, compiler))

def downloadPaper(paper, config):
    '''Download the paper.

    Params
    ------
    :arg paper
        A `Paper` instance result given by the ads'''

    def open_file(fname):
        sh.Command(config['adsquery']['pdf_viewer'])(fname, _bg=True)

    def process_output(line):
        print(line, end='')

    if paper.pub == 'ArXiv e-prints':
        # Get the ArXiv name
        _id = paper.bibcode.split('arXiv')[1][:-1]
        _id = _id[:4] + '.' + _id[4:]
        url = 'https://arxiv.org/pdf/{id}'.format(id=_id)
    else:
        url = ("http://adsabs.harvard.edu/cgi-bin/nph-data_query?"
               "bibcode={paper.bibcode}&link_type=ARTICLE".format(
                   paper=paper))

    print(f'Downloading {url}')

    fname = '{paper.bibcode}_{author}.pdf'.format(
        paper=paper,
        author=paper.first_author.split(',')[0])

    filesDir = os.path.join(os.path.expanduser('~'), 'ADS')
    # create the directory of not existing
    if not os.path.isdir(filesDir):
        os.path.mkdir(filesDir)

    fname = os.path.join(filesDir, fname)

    if os.path.isfile(fname):
        ans = getInput('File already exists on disk. Overwrite [Y/n]?',
                       lambda e: e.lower() if e.lower() in ['y', 'n', '']
                       else None)
        if ans == 'n':
            open_file(fname)
            return

    sh.wget(url,
            header="User-Agent: Mozilla/5.0 (Windows NT 5.1; rv:23.0) Gecko/20100101 Firefox/23.0",
            O=fname,
            _out=process_output)
    print('Downloaded into %s' % fname)
    open_file(fname)

def store():
    """
    Stores all directories in the filesystem that contains a .sync file to the configured
    ``root_sync_path``

    """
    wget('-O', 'dropbox.py', 'https://www.dropbox.com/download?dl=packages/dropbox.py')
    dropbox.start_dropbox()
    sync_paths = _get_sync_paths('/home/cada', excludes | {root_sync_path})
    sync_mappings = [(path, Path(root_sync_path) / path.relative_to('/'))
                     for path in sync_paths]
    _print_sync_mappings(sync_mappings)
    _sync(sync_mappings)
    dropbox.start_dropbox()

def download_sample(source_url: str) -> AudioSample:
    "Download an mp3 file from the internet."
    metadata = {'source_url': source_url}

    if not source_url.endswith('.mp3'):
        print('ERROR: sample doesn\'t appear to be in mp3 format',
              file=sys.stderr)
        sys.exit(1)

    t = tempfile.NamedTemporaryFile(delete=False)
    sh.wget('-O', t.name, source_url, _out=open('/dev/stdout', 'wb'),
            _err=open('/dev/stderr', 'wb'))

    return AudioSample(tempfile=t, metadata=metadata)

	def transfer(self, filePath, remote=False):
		""".. function:: transfer(files, remote)

		      Transfer files, local or remote, to host specified on command line

		      :param filePath: path/address of files to be transferred to host
		      :param remote: boolean dictating if files are remote (true) or local (false)"""

		if remote:
			sh.wget("-N", "-P", "./files", filePath) #Places files within a local directory named "files"
			scpHost = self.host + ":~"
			scp("-r", "./files", scpHost)
		else:
			scpHost = self.host + ":~"
			scp(filePath, scpHost) #May need to edit command to recursively handle directories

def retrieve_data(where, d):
    ud=where+d
    pathDeployName = ud.split('/')[-1]
    path_arg = ud + "/" + pathDeployName + ".nc3.nc"
    host_arg = SERVER+"/tabledap/" + pathDeployName + ".ncCFMA"
    print "Path Arg", path_arg
    print "Host Arg", host_arg
    args = [
        "--no-host-directories",
        "--cut-dirs=2",
        "--output-document=%s" % path_arg,
        host_arg
     ]
    print "Args:", ' '.join(args)
    sh.wget(*args)

def grade(uniqname, link):
    print("Grading {}".format(uniqname))
    with pushd("373-f15-linked-list"):
        wget(link, "-O", "{}.c".format(uniqname))
        rm("-f", "list.c", "list.o", "list")
        ln("-s", "{}.c".format(uniqname), "list.c")
        make("run")
        try:
            diff("list.out", "golden.out")
            perfect_grade(uniqname)
        except sh.ErrorReturnCode_1:
            try:
                diff("list.out", "naive.out")
                no_change(uniqname)
            except sh.ErrorReturnCode_1:
                handgrade(uniqname)

def initialize():
    # noinspection PyUnresolvedReferences
    from sh import wget, tar, rm, shasum
    if not os.path.exists(prefix):
        os.makedirs(prefix)

    if (not os.path.exists(dirs['inputs'])) or (not os.path.exists(dirs['intermediates'])):
        try:
            if not os.path.exists(prefix):
                logger.info("Creating {DIR}".format(DIR=prefix))
                os.makedirs(prefix)
            logger.info("Downloading data from {URL} to {DIR}".format(URL=data_url, DIR=prefix))
            tar(wget(data_url, "-qO-", _piped=True), "xz", _cwd=prefix)
            logger.info("Checking checksums of downloaded files")
            for line in shasum("-c", _cwd=prefix, _in=checksums, _iter=True):
                logger.info(line)
        except Exception as e:
            logger.info("Error: {}".format(e.message))
            logger.info("Deleting {DIR}".format(DIR=dirs['inputs']))
            rm(dirs['inputs'], '-rf')
            logger.info("Deleting {DIR}".format(DIR=dirs['intermediates']))
            rm(dirs['intermediates'], '-rf')
            raise

    # make sure all those directories exist
    for d in (dirs['outputs'], dirs['plots']):
        if not os.path.exists(d):
            logger.info("Creating {DIR}".format(DIR=d))
            os.makedirs(d)

    def setup_tomcat(self, args):
        try:
            log("Downloading Tomcat ...")
            result = wget(dict['TOMCAT_DOWNLOAD_URL'])
        except:
            log("Error getting Tomcat from : " + dict['TOMCAT_DOWNLOAD_URL'])

        try:
            log("Extracting Tomcat ...")            
            result = tar("xvzf " , dict['TOMCAT_VERSION']+ ".tar.gz")
        except:
            log("Error extracting Tomcat ..." + dict['TOMCAT_VERSION']+ ".tar.gz")
        
        setup_airavata_server(args)

        try:
            log("Copying the Airavata war files to Tomcat's webapp directory ...")
            result = cp(dict['AIRAVATA_VERSION']+ "/*.war " , dict['TOMCAT_VERSION']+ "/webapps", "-v")
        except:
            log("Error copying the Airavata war files to Tomcat's webapp directory ...")

        try :
            log("Granting executeable permissions to the script")
            result = chmod("a+x" , dict['TOMCAT_VERSION']+ "/*.sh")
        except:
            log("Error granting executable permissions to " + dict['TOMCAT_VERSION']+ "/*.sh")