def prebuild_arch(self, arch):
     if not self.is_patched(arch):
         super(ReportLabRecipe, self).prebuild_arch(arch)
         self.apply_patch('patches/fix-setup.patch', arch.arch)
         recipe_dir = self.get_build_dir(arch.arch)
         shprint(sh.touch, os.path.join(recipe_dir, '.patched'))
         ft = self.get_recipe('freetype', self.ctx)
         ft_dir = ft.get_build_dir(arch.arch)
         ft_lib_dir = os.environ.get('_FT_LIB_', os.path.join(ft_dir, 'objs', '.libs'))
         ft_inc_dir = os.environ.get('_FT_INC_', os.path.join(ft_dir, 'include'))
         tmp_dir = os.path.normpath(os.path.join(recipe_dir, "..", "..", "tmp"))
         info('reportlab recipe: recipe_dir={}'.format(recipe_dir))
         info('reportlab recipe: tmp_dir={}'.format(tmp_dir))
         info('reportlab recipe: ft_dir={}'.format(ft_dir))
         info('reportlab recipe: ft_lib_dir={}'.format(ft_lib_dir))
         info('reportlab recipe: ft_inc_dir={}'.format(ft_inc_dir))
         with current_directory(recipe_dir):
             sh.ls('-lathr')
             ensure_dir(tmp_dir)
             pfbfile = os.path.join(tmp_dir, "pfbfer-20070710.zip")
             if not os.path.isfile(pfbfile):
                 sh.wget("http://www.reportlab.com/ftp/pfbfer-20070710.zip", "-O", pfbfile)
             sh.unzip("-u", "-d", os.path.join(recipe_dir, "src", "reportlab", "fonts"), pfbfile)
             if os.path.isfile("setup.py"):
                 with open('setup.py', 'rb') as f:
                     text = f.read().replace('_FT_LIB_', ft_lib_dir).replace('_FT_INC_', ft_inc_dir)
                 with open('setup.py', 'wb') as f:
                     f.write(text)

def download(target, temp_dir):
    zip_path = path.join(temp_dir, "temp.zip")
    tgt_path = path.join(temp_dir, "chunk")

    for chunk in CHUNKS:
        tif_name = TIF_FORMAT.format(chunk)
        tif_path = path.join(temp_dir, tif_name)

        wget(URL_FORMAT.format(chunk), q=True, O=zip_path)
        
        with zipfile.ZipFile(zip_path, 'r') as pack:
            contents = pack.namelist()
            if contents != [tif_name]:
                raise ValueError("Bad archive contents: {:r}".format(contents))
        
        unzip(zip_path, d=temp_dir)
        os.unlink(zip_path)

        convert(tif_path, '-quiet', 'GRAY:{}'.format(tgt_path))
        os.unlink(tif_path)

        if os.stat(tgt_path).st_size != EXPECT_SIZE:
            raise ValueError("Bad converted size: {}".format(chunk))

        with open(tgt_path, "rb") as f:
            shutil.copyfileobj(f, target)
        os.unlink(tgt_path)

 def compile( self, source_dir, build_dir, install_dir ):
     package_source_dir = os.path.join( source_dir, self.dirname )
     assert( os.path.exists( package_source_dir ) )
     package_build_dir = os.path.join( build_dir, self.dirname )
     runpath_dir = os.path.join( package_source_dir, 'RunPath' )
     if ( not os.path.exists( os.path.join( runpath_dir, 'media.zip' ) ) ):
         sh.cd( runpath_dir )
         sh.wget( '--no-check-certificate', 'https://bitbucket.org/jacmoe/ogitor/downloads/media.zip' )
         sh.unzip( 'media.zip' )
     if ( not os.path.exists( os.path.join( runpath_dir, 'projects.zip' ) ) ):
         sh.cd( runpath_dir )
         sh.wget( '--no-check-certificate', 'https://bitbucket.org/jacmoe/ogitor/downloads/projects.zip' )
         sh.unzip( 'projects.zip' )
     sh.mkdir( '-p', package_build_dir )
     sh.cd( package_build_dir )
     if ( platform.system() == 'Darwin' ):
         sh.cmake(
             '-G', 'Xcode',
             '-D', 'CMAKE_INSTALL_PREFIX=%s' % install_dir,
             '-D', 'CMAKE_MODULE_PATH=%s' % os.path.join( install_dir, 'CMake' ),
             package_source_dir,
             _out = sys.stdout )
         sh.xcodebuild( '-configuration', 'Release', _out = sys.stdout )
     else:
         sh.cmake(
             '-D', 'CMAKE_INSTALL_PREFIX=%s' % install_dir,
             '-D', 'CMAKE_MODULE_PATH=%s' % os.path.join( install_dir, 'lib/OGRE/cmake' ),
             package_source_dir,
             _out = sys.stdout )
         sh.make( '-j4', 'VERBOSE=1', _out = sys.stdout )
         sh.make.install( _out = sys.stdout )

    def html_output(self):
        ext = '.html'
        today = datetime.date.today().isoformat()
        sha = self.test_file + ".html.sha"
        # cannot recover if generating html fails
        options = (['--zip'] + self.options
                   + ['-f', 'html', self.test_file,
                      self.test_out + ext + '.zip'])
        try:
            self.gdoc_to(*options,
                         _err=self.test_err + ".html.log")
            # XXX it hangs without -n, didn't have time to figure out why
            out_dir = os.path.dirname(self.test_out)
            sh.unzip('-n', '-d', out_dir, self.test_out + ext + '.zip')
            sh.sed('-i', '-e', 's/%s/TODAYS_DATE/g' % today,
                   self.test_out + ext)
            test_result = slurp('%s.html' % self.test_out)
        except ErrorReturnCode as e:
            self.say(red("gdoc-to failed: {}. See {}.html.log"),
                     e, self.test_err)
            self.say(red("Ran in {}"), os.getcwd())
            self.failed = True
            sh.rm('-f', sha)
            return
        try:
            html5check(self.test_out + ext,
                       _out=self.test_out + ".html.errors")
        except ErrorReturnCode:
            self.say(red("Test output did not validate as XHTML5!"))
            self.say(red("\tSee {}.html.errors"), self.test_out)
            self.failed = True

        if test_result != slurp(self.test_file + ext):
            # the file changed, but the change might be okay
            spit(self._canonical_body(self.test_out + ext),
                 self.test_out + ".body")
            spit(self._canonical_body(self.test_file + ext),
                 self.test_out + ".canon.body")

            if (slurp(self.test_out + '.body')
                    == slurp(self.test_out + '.canon.body')):
                self.say(yellow("File changed. Updating canonical file."))
                sh.cp(self.test_out + ext, self.test_file + ext)
            else:
                self.say(red("HTML body changed!"))
                self.say(red("\tSee {}.*"), fail_path(self.test_name))
                sh.cp(self.test_out + ext, fail_path(self.test_name + ext))
                sh.diff('-u', self.test_file + ext, self.test_out + ext,
                        _out=fail_path(self.test_name + ".html.diff"),
                        _ok_code=[0, 1])
                sh.cp(self.test_out + ".body",
                      fail_path(self.test_name + ".body"))
                sh.cp(self.test_out + ".canon.body",
                      fail_path(self.test_name + ".body.expected"))
                sh.diff('-u', self.test_out + ".canon.body",
                        self.test_out + ".body",
                        _out=fail_path(self.test_name + '.body.diff'),
                        _ok_code=[0, 1])
                self.failed = True

 def backups(self):
     data = self.http.load('backups', {'dato': 1})
     if data:
         dialogo = Alerta_Combo('Lista de Backup', 'backup.png', 'Escoja el backup que desea descargar:', data, liststore=(str, str))
         url = dialogo.iniciar()
         if url:
             print 'Backup'
             print url
             sh.wget(url)
             1/0
             sh.unzip()
     print data

def archiveDownload(url, destination, archiveType):
    logging.info('Now downloading archive file from URL %s to %s' % (url, destination))
    filename = wget.download(url)
    if archiveType == 'zip':
        logging.info('Unzipping zip file from: ', filename)
        sh.unzip(filename)
    elif archiveType == 'tar.gz':
        logging.info('Untarring tar.gz file from: ', filename)
        sh.tar('-xvzf', filename )
    logging.info('Removing archive file.')
    sh.rm(filename)
    return

def extract_data(input_file_name, output_file_names):
    sh.unzip("-o", input_file_name)

    # This also updates timestamps.  Ruffus doesn't recognize these files as complete results unles the
    # timestamp is up to date.
    sh.mv("testdata.manual.2009.06.14.csv", "sentiment140.test.csv")
    sh.mv("training.1600000.processed.noemoticon.csv", "sentiment140.train.csv")

    # Re-encode the files as utf8.  They look like utf8 already (e.g. file thinks they're utf8)
    # but they are actually encoded as latin1.  This doesn't make a difference for the test data
    # (the utf8 and latin1 encoded test data are identical files) but the train data has some
    # byte sequences that are invalid utf8 and this makes simplejson really upset.
    for output_file in output_file_names:
        sh.mv(output_file, "temp")
        sh.iconv("-f", "latin1", "-t", "utf8", "temp", _out=output_file)
        sh.rm("temp")

	def install_tool(self):
		results = sh.unzip("-o", DOWNLOAD, "-d", TOOL_HOME)
		parts = results.split('\n')
		for part in parts:
			if part.find("inflating") > -1:
				path = self._return_path_bit(part.strip().split(" ")[1])
				break
		sh.rm("-f", AMI_HOME)
		sh.ln("-s", TOOL_HOME + "/" + path, AMI_HOME)

    def fetch(self):

        cache = self.workdir('cache', True)
        key = self.key(self.path)

        if key is not None:

            outfile = '%s/%s' % (cache, os.path.basename(key.name))

            with open(outfile, 'w') as f:
                key.get_file(f)

            unpack_dir = self.workdir('unpacked', True)
            sh.unzip(outfile, _cwd=unpack_dir)
            return (key.etag.strip('"'), unpack_dir)

            raise Exception("Unable to unpack archive format '%s'" % ext)

        return (None, cache)

def _install_fancybox(stdout=None):
    """Callback to install necessary library for the Fancybox module"""
    http.dl(FANCYBOX_URI, 'fancybox.zip')
    output = unzip('fancybox.zip')

    if stdout and output:
        stdout.write(str(output).strip() + '\n')

    os.rename('fancyapps-fancyBox-18d1712', 'fancybox')
    rm('fancybox.zip')

def _install_predis(stdout=None):
    """Callback to install Predis for the Redis module"""
    http.dl(PREDIS_URI, 'predis.zip')
    output = unzip('predis.zip')

    if stdout and output:
        stdout.write(str(output).strip() + '\n')

    os.rename('nrk-predis-d02e2e1', 'predis')
    rm('predis.zip')

def _install_jquery_colorpicker(stdout=None):
    """Callback to install necessary library for the module"""
    os.makedirs('./colorpicker')
    http.dl(JQ_COLOR_PICKER_URI, './colorpicker/colorpicker.zip')
    with pushd('./colorpicker'):
        output = unzip('colorpicker.zip')

        if stdout and output:
            stdout.write(str(output).strip() + '\n')

        rm('colorpicker.zip')

    def process_layer(self, typename):
	fname_in = '%s.shp' % typename
	fname_out = '%s_rasterized.tiff' % typename
	pwd = str(sh.pwd()).strip()
        try:
            sh.cd('/tmp/layer')
            sh.rm("-rf",sh.glob('*'))
            sh.unzip('../layer.zip')
	    saga_cmd.shapes_points("Points Filter", POINTS=fname_in, FIELD="MASA_HUMEDO", FILTER="tmp1.shp", RADIUS=100, MINNUM=25, MAXNUM=200, METHOD=4, PERCENT=15)
	    saga_cmd.shapes_points("Points Filter", POINTS="tmp1.shp", FIELD="MASA_HUMEDO", FILTER="tmp2.shp", RADIUS=100, MINNUM=25, MAXNUM=200, METHOD=5, PERCENT=90)
    	    saga_cmd.grid_gridding("Shapes to Grid", INPUT="tmp2.shp", FIELD="MASA_HUMEDO", MULTIPLE=4, LINE_TYPE=0, GRID_TYPE=3, USER_SIZE=0.0001, TARGET=0, USER_GRID="tmp3.sgrd")
	    saga_cmd.grid_tools("Close Gaps", INPUT="tmp3.sgrd", RESULT="tmp4.sgrd")
	    saga_cmd.shapes_points("Convex Hull", SHAPES="tmp2.shp", HULLS="tmphull.shp", POLYPOINTS=0)
	    saga_cmd.shapes_grid("Clip Grid with Polygon", INPUT="tmp4.sgrd", OUTPUT="tmp5.sgrd", POLYGONS="tmphull.shp")
	    saga_cmd.grid_filter("Gaussian Filter", INPUT="tmp5.sgrd", RESULT="tmp6", SIGMA=3, MODE=1, RADIUS=50)

	    sh.gdal_translate("-of", "gtiff", "tmp6.sdat", fname_out)
        finally:
	    sh.cd(pwd)
	return '/tmp/layer/%s' % fname_out

    def setup_airavata_server(self, args):
        try:
            log("Downloading Airavata web-app ...")
            result = wget(dict['AIRAVATA_DOWNLOAD_URL']+ ".zip")
        except:
            log("Error Downloading Airavata web-app from: " + dict['AIRAVATA_DOWNLOAD_URL'])

        try:
            log("Extracting Airavata war")
            result = unzip(dict['AIRAVATA_VERSION']+ ".zip")
        except:
            log("Error extracting Airavata war" + dict['AIRAVATA_VERSION']+ ".zip")

    def prebuild_arch(self, arch):
        if not self.is_patched(arch):
            super(ReportLabRecipe, self).prebuild_arch(arch)
            recipe_dir = self.get_build_dir(arch.arch)

            # Some versions of reportlab ship with a GPL-licensed font.
            # Remove it, since this is problematic in .apks unless the
            # entire app is GPL:
            font_dir = os.path.join(recipe_dir,
                                    "src", "reportlab", "fonts")
            if os.path.exists(font_dir):
                for l in os.listdir(font_dir):
                    if l.lower().startswith('darkgarden'):
                        os.remove(os.path.join(font_dir, l))

            # Apply patches:
            self.apply_patch('patches/fix-setup.patch', arch.arch)
            shprint(sh.touch, os.path.join(recipe_dir, '.patched'))
            ft = self.get_recipe('freetype', self.ctx)
            ft_dir = ft.get_build_dir(arch.arch)
            ft_lib_dir = os.environ.get('_FT_LIB_', os.path.join(ft_dir, 'objs', '.libs'))
            ft_inc_dir = os.environ.get('_FT_INC_', os.path.join(ft_dir, 'include'))
            tmp_dir = os.path.normpath(os.path.join(recipe_dir, "..", "..", "tmp"))
            info('reportlab recipe: recipe_dir={}'.format(recipe_dir))
            info('reportlab recipe: tmp_dir={}'.format(tmp_dir))
            info('reportlab recipe: ft_dir={}'.format(ft_dir))
            info('reportlab recipe: ft_lib_dir={}'.format(ft_lib_dir))
            info('reportlab recipe: ft_inc_dir={}'.format(ft_inc_dir))
            with current_directory(recipe_dir):
                ensure_dir(tmp_dir)
                pfbfile = os.path.join(tmp_dir, "pfbfer-20070710.zip")
                if not os.path.isfile(pfbfile):
                    sh.wget("http://www.reportlab.com/ftp/pfbfer-20070710.zip", "-O", pfbfile)
                sh.unzip("-u", "-d", os.path.join(recipe_dir, "src", "reportlab", "fonts"), pfbfile)
                if os.path.isfile("setup.py"):
                    with open('setup.py', 'r') as f:
                        text = f.read().replace('_FT_LIB_', ft_lib_dir).replace('_FT_INC_', ft_inc_dir)
                    with open('setup.py', 'w') as f:
                        f.write(text)

 def retrieve(self):
     if self.do_retrieve:
         sh.rm("-fr", self.dirname)
         os.mkdir(self.dirname)
         self.pushd(self.dirname)
         retrieved = FileBuildConfiguration.download(self.url)
         if ".tar.gz" in retrieved:
             sh.tar("xvzf", retrieved)
         if ".tar.bz2" in retrieved:
             sh.tar("xjvf", retrieved)
         if ".zip" in retrieved:
             sh.unzip(retrieved)
     else:
         self.pushd(self.dirname)
     # Either one directory *OR* one directory + a README.
     if(len(os.listdir(".")) <= 3):
         # we can assume that we need to chdir before we can build, so set that to the local build path
         for curr in os.listdir("."):
             if(os.path.isdir(curr)):
                 self.buildpath = curr
     if not getattr(self, 'buildpath'):
         self.buildpath = "."
     self.popd()

    def download_client_config(self, cluster, service):
        """Download the client configuration zip for a particular cluster and service.

        Since cm_api does not provide a way to download the archive we build the URL
        manually and download the file. Once it downloaded the file the archive is
        extracted and its content is copied to the Hadoop configuration directories
        defined by Impala.
        """
        logger.info("Downloading client configuration for {0}".format(service.name))
        url = "http://{0}:7180/api/{1}/clusters/{2}/services/{3}/clientConfig".format(
            self.cm_host, CM_API_VERSION, urlquote(cluster.name), urlquote(service.name))
        path = mkdtemp()
        sh.curl(url, o=os.path.join(path, "clientConfig.zip"), _out=tee, _err=tee)
        current = os.getcwd()
        os.chdir(path)
        sh.unzip("clientConfig.zip")
        for root, _, file_names in os.walk("."):
            for filename in fnmatch.filter(file_names, "*.xml"):
                src = os.path.join(root, filename)
                dst = os.path.join(self.impala_home, "fe", "src", "test", "resources")
                logger.debug("Copying {0} to {1}".format(src, dst))
                shutil.copy(src, dst)
        os.chdir(current)

    def _get_code(self, nmpi_job, job_desc):
        """
        Obtain the code and place it in the working directory.

        If the experiment description is the URL of a Git repository, try to clone it.
        If it is the URL of a zip or .tar.gz archive, download and unpack it.
        Otherwise, the content of "code" is the code: write it to a file.
        """
        url_candidate = urlparse(nmpi_job['code'])
        logger.debug("Get code: %s %s", url_candidate.netloc, url_candidate.path)
        if url_candidate.scheme and url_candidate.path.endswith((".tar.gz", ".zip", ".tgz")):
            self._create_working_directory(job_desc.working_directory)
            target = os.path.join(job_desc.working_directory, os.path.basename(url_candidate.path))
            #urlretrieve(nmpi_job['code'], target) # not working via KIP https proxy
            curl(nmpi_job['code'], '-o', target)
            logger.info("Retrieved file from {} to local target {}".format(nmpi_job['code'], target))
            if url_candidate.path.endswith((".tar.gz", ".tgz")):
                tar("xzf", target, directory=job_desc.working_directory)
            elif url_candidate.path.endswith(".zip"):
                try:
                    # -o for auto-overwrite
                    unzip('-o', target, d=job_desc.working_directory)
                except:
                    logger.error("Could not unzip file {}".format(target))
        else:
            try:
                # Check the "code" field for a git url (clone it into the workdir) or a script (create a file into the workdir)
                # URL: use git clone
                git.clone('--recursive', nmpi_job['code'], job_desc.working_directory)
                logger.info("Cloned repository {}".format(nmpi_job['code']))
            except (sh.ErrorReturnCode_128, sh.ErrorReturnCode):
                # SCRIPT: create file (in the current directory)
                logger.info("The code field appears to be a script.")
                self._create_working_directory(job_desc.working_directory)
                with codecs.open(job_desc.arguments[0], 'w', encoding='utf8') as job_main_script:
                    job_main_script.write(nmpi_job['code'])

    def unpack(self, arch):
        info_main('Unpacking {} for {}'.format(self.name, arch))

        build_dir = self.get_build_container_dir(arch)

        user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower()))
        if user_dir is not None:
            info('P4A_{}_DIR exists, symlinking instead'.format(
                self.name.lower()))
            # AND: Currently there's something wrong if I use ln, fix this
            warning('Using git clone instead of symlink...fix this!')
            if exists(self.get_build_dir(arch)):
                return
            shprint(sh.rm, '-rf', build_dir)
            shprint(sh.mkdir, '-p', build_dir)
            shprint(sh.rmdir, build_dir)
            ensure_dir(build_dir)
            shprint(sh.git, 'clone', user_dir, self.get_build_dir(arch))
            return

        if self.url is None:
            info('Skipping {} unpack as no URL is set'.format(self.name))
            return

        filename = shprint(
            sh.basename, self.versioned_url).stdout[:-1].decode('utf-8')

        with current_directory(build_dir):
            directory_name = self.get_build_dir(arch)

            # AND: Could use tito's get_archive_rootdir here
            if not exists(directory_name) or not isdir(directory_name):
                extraction_filename = join(
                    self.ctx.packages_path, self.name, filename)
                if isfile(extraction_filename):
                    if extraction_filename.endswith('.tar.gz') or \
                       extraction_filename.endswith('.tgz'):
                        sh.tar('xzf', extraction_filename)
                        root_directory = shprint(
                            sh.tar, 'tzf', extraction_filename).stdout.decode(
                                'utf-8').split('\n')[0].split('/')[0]
                        if root_directory != directory_name:
                            shprint(sh.mv, root_directory, directory_name)
                    elif (extraction_filename.endswith('.tar.bz2') or
                          extraction_filename.endswith('.tbz2')):
                        info('Extracting {} at {}'
                             .format(extraction_filename, filename))
                        sh.tar('xjf', extraction_filename)
                        root_directory = sh.tar(
                            'tjf', extraction_filename).stdout.decode(
                                'utf-8').split('\n')[0].split('/')[0]
                        if root_directory != directory_name:
                            shprint(sh.mv, root_directory, directory_name)
                    elif extraction_filename.endswith('.zip'):
                        sh.unzip(extraction_filename)
                        import zipfile
                        fileh = zipfile.ZipFile(extraction_filename, 'r')
                        root_directory = fileh.filelist[0].filename.strip('/')
                        if root_directory != directory_name:
                            shprint(sh.mv, root_directory, directory_name)
                    else:
                        raise Exception(
                            'Could not extract {} download, it must be .zip, '
                            '.tar.gz or .tar.bz2')
                elif isdir(extraction_filename):
                    mkdir(directory_name)
                    for entry in listdir(extraction_filename):
                        if entry not in ('.git',):
                            shprint(sh.cp, '-Rv',
                                    join(extraction_filename, entry),
                                    directory_name)
                else:
                    raise Exception(
                        'Given path is neither a file nor a directory: {}'
                        .format(extraction_filename))

            else:
                info('{} is already unpacked, skipping'.format(self.name))

        converted_boms.append(txt)
boms += converted_boms

# Get the output file name
output_name = fab_zip.split("_to_fab")[0] + "_{}.zip".format(args.date)

# Actually make the zip

# Generate the folders we use to organize things
mkdir(FAB_FOLDER)
mkdir(ASSEM_FOLDER)
mkdir(IMAGE_FOLDER)

# Put the contents of the zip files in the folders
# This way we don't have to replicate that logic
unzip(fab_zip, "-d", FAB_FOLDER)
unzip(assem_zip, "-d", ASSEM_FOLDER)

# Put the images in the images folder
for jpg in jpgs:
    cp(jpg, IMAGE_FOLDER)

# Get the filenames for fab
fab_files = glob.glob("{}/*".format(FAB_FOLDER))
assem_files = glob.glob("{}/*".format(ASSEM_FOLDER))
image_files = glob.glob("{}/*".format(IMAGE_FOLDER))

combined = [output_name] + schs + brds + pdfs + dxfs + infos + boms + fab_files + assem_files + image_files

sh.zip(*combined)