def parse_source(source, idx, header):
    """
    Import data from a single source based on the data type.
    """
    path = '{}/{}'.format(config.workspace_dir, idx)
    if not os.path.exists(path):
        os.makedirs(path)

    cache_url = source[header.index('cache')]
    cache_filename = re.search('/[^/]*$', cache_url).group()
    fetch(cache_url, path + cache_filename)

    files = rlistdir(path)
    for f in files:
        if re.match('.*\.(zip|obj|exe)$', f):  # some files had mislabelled ext
            unzip(f, path)

    shapes = []
    files = rlistdir(path)
    for f in files:
        if re.match('.*\.({})$'.format('|'.join(config.fiona_extensions)), f):
            objs = import_with_fiona(f, source[0])
            for obj in objs:
                shapes.append(obj)
        elif re.match('.*\.csv$', f):
            objs = import_csv(f, source[0])
            for obj in objs:
                shapes.append(obj)

    shutil.rmtree(path)

    if not shapes:
        _L.warning('failed to parse source. did not find shapes. files in archive: {}'.format(files))

    return shapes

 def install_Windows(self):
     # ctags
     ctags_zip = os.path.join(DOWNLOADS, CTAGS_VER + '.zip')
     if not isfile(ctags_zip):
         wget('http://sourceforge.net/projects/ctags/files/ctags/5.8/{}.zip'.
              format(CTAGS_VER), ctags_zip)
     unzip(ctags_zip, CTAGS_VER + '/ctags.exe')

def get_sport(sport):

    global root

    utils.logmessage("Getting: " + sport, "getdata", 1)

    if sport == "baseball":
        url = "http://seanlahman.com/files/database/lahman-csv_2014-02-14.zip"
        path = root + "rawdata/lahman/"
        saveas = "lahman.zip"

    if sport == "basketball":
        url = "http://www.databasebasketball.com/databasebasketball_2009_v1.zip"
        path = root + "rawdata/basketball/"
        saveas = "basketball.zip"

    # download file
    utils.logmessage("Downloading zip file", "getdata", 1)
    utils.download_file(url, path+saveas)

    # unzip file
    utils.unzip(path+saveas, path)
    utils.logmessage("Unzipping file", "getdata", 1)

    pass

    def update(self):
        # Step 1: Get newest nightly folder
        response = urllib2.urlopen(self.nightly_dir+"/?C=N;O=D")
        html = response.read()
        folder_id =  re.findall("[0-9]{5,}", html)[0]

        # Step 2: Find the correct file
        response = urllib2.urlopen(self.nightly_dir+"/"+folder_id)
        html = response.read()
        exec_file = re.findall("jsshell-win32.zip", html)[0]
        json_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.win32.json", html)[0]

        # Step 3: Get build information
        response = urllib2.urlopen(self.nightly_dir+"/"+folder_id+"/"+json_file)
        html = response.read()
        info = json.loads(html)

        # Step 4: Fetch archive
        print "Retrieving", self.nightly_dir+"/"+folder_id+"/"+exec_file
        urllib.urlretrieve(self.nightly_dir+"/"+folder_id+"/"+exec_file, self.tmp_dir + "shell.zip")

        # Step 5: Unzip
        utils.unzip(self.tmp_dir,"shell.zip")

        # Step 6: Save info
        self.updated = True
        self.cset = info["moz_source_stamp"]

    def __init__(self):
        super(Shumway, self).__init__('shumway', '0.1', 'shumway')

        # Only update harness once a day:
        from datetime import datetime
        date = datetime.now().strftime("%Y-%m-%d")
        utils.getOrDownload("/tmp/", "shumway", date,
                            "http://mozilla.github.io/shumway/shell/shumway-shell.zip",
                            "/tmp/shumway-shell.zip")
        utils.unzip("/tmp/", "shumway-shell.zip")

def unpack(meta):
    src_path = download_to_cache(meta)

    os.makedirs(WORK_DIR)
    if src_path.endswith(('.tar.gz', '.tar.bz2', '.tgz', '.tar.xz', '.tar')):
        tar_xf(src_path, WORK_DIR)
    elif src_path.endswith('.zip'):
        unzip(src_path, WORK_DIR)
    else:
        raise Exception("not a vaild source")

 def install_webdriver(self, dest=None, channel=None, browser_binary=None):
     if dest is None:
         dest = os.pwd
     url = self._latest_chromedriver_url(browser_binary)
     self.logger.info("Downloading ChromeDriver from %s" % url)
     unzip(get(url).raw, dest)
     chromedriver_dir = os.path.join(dest, 'chromedriver_%s' % self.platform_string())
     if os.path.isfile(os.path.join(chromedriver_dir, "chromedriver")):
         shutil.move(os.path.join(chromedriver_dir, "chromedriver"), dest)
         shutil.rmtree(chromedriver_dir)
     return find_executable("chromedriver", dest)

 def install_webdriver(self, dest=None):
     """Install latest Webdriver."""
     if dest is None:
         dest = os.pwd
     latest = get("http://chromedriver.storage.googleapis.com/LATEST_RELEASE").text.strip()
     url = "http://chromedriver.storage.googleapis.com/%s/chromedriver_%s.zip" % (latest,
                                                                                  self.platform_string())
     unzip(get(url).raw, dest)
     path = find_executable(dest, "chromedriver")
     st = os.stat(path)
     os.chmod(path, st.st_mode | stat.S_IEXEC)
     return path

    def _update(self, folder_id):
        # Step 2: Find the correct file
        response = urllib2.urlopen(self.nightly_dir+"/"+folder_id)
        html = response.read()
        if self.slaveType == "android":
            exec_file = re.findall("fennec-[a-zA-Z0-9.]*.en-US.android-arm.apk", html)[0]
            json_file = re.findall("fennec-[a-zA-Z0-9.]*.en-US.android-arm.json", html)[0]
        elif self.slaveType == "mac-desktop":
            exec_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.mac.dmg", html)[0]
            json_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.mac.json", html)[0]
        elif self.slaveType == "linux-desktop":
            exec_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.linux-x86_64.tar.bz2", html)[0]
            json_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.linux-x86_64.json", html)[0]
        else:
            exec_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.win32.zip", html)[0]
            json_file = re.findall("firefox-[a-zA-Z0-9.]*.en-US.win32.json", html)[0]

        # Step 3: Get build information
        response = urllib2.urlopen(self.nightly_dir+"/"+folder_id+"/"+json_file)
        html = response.read()
        info = json.loads(html)

        # Step 4: Test if there is a new revision
        if self.slaveType == "android":
            output = self.tmp_dir + self.folder + "/fennec.apk"
        elif self.slaveType == "mac-desktop":
            output = self.tmp_dir + self.folder + "/firefox.dmg"
        elif self.slaveType == "linux-desktop":
            output = self.tmp_dir + self.folder + "/firefox.tar.bz2"
        else:
            output = self.tmp_dir + self.folder + "/firefox.zip"
        utils.getOrDownload(self.tmp_dir, "mozilla", info["moz_source_stamp"],
                            self.nightly_dir + "/" + folder_id + "/" + exec_file,
                            output)

        # Step 5: Prepare to run
        if self.slaveType == "android":
            print subprocess.check_output(["adb", "install", "-r", self.tmp_dir + self.folder + "/fennec.apk"])
        elif self.slaveType == "mac-desktop":
            if os.path.exists("/Volumes/Nightly"):
                print subprocess.check_output(["hdiutil", "detach", "/Volumes/Nightly"])
            print subprocess.check_output(["hdiutil", "attach", self.tmp_dir + self.folder + "/firefox.dmg"])
        elif self.slaveType == "linux-desktop":
            utils.unzip(self.tmp_dir + self.folder, "firefox.tar.bz2")
        else:
            utils.unzip(self.tmp_dir + self.folder, "firefox.zip")

        # Step 6: Save info
        self.updated = True
        self.cset = info["moz_source_stamp"]

def fetch_sub(location, directory=None, token=None):
    """Filename is the full path of the movie file
    dirname is the full path of the movie dir, fetch_sub will search for video file
    diretory is the directory where sub should be stored, default to the movie dir
    """
    my_open_sub = MyOpenSub(OPENSUB_URL, token)
    if not token:
        my_open_sub.login(USERNAME, PASSWORD, LANGUAGE, USER_AGENT)
    filenames = find_video(location)
    if not filenames:
        print "No video file found"
        return
    for filename in find_video(location):
        print "Fetching subtitle for %s" % filename
        file_hash, file_bytes = hashfile(filename)
        dl_links = my_open_sub.search_sub_links(file_hash, file_bytes)
        if not dl_links:
            print "Sorry, no subtitle found"
            return
        if not directory:
            directory = os.path.dirname(filename)
        for dl_link in dl_links:
            try:
                filename = wget.download(dl_link, out=directory)
                print "Download finished: %s" % filename
                filename = unzip(filename, directory)
                if filename:
                    print "Unzipped to %s" % filename
            except IOError as io_error:
                print io_error

    def install_webdriver(self, dest=None):
        if dest is None:
            dest = os.pwd
        latest = get("https://api.github.com/repos/operasoftware/operachromiumdriver/releases/latest").json()["tag_name"]
        url = "https://github.com/operasoftware/operachromiumdriver/releases/download/%s/operadriver_%s.zip" % (latest,
                                                                                                                self.platform_string())
        unzip(get(url).raw, dest)

        operadriver_dir = os.path.join(dest, "operadriver_%s" % self.platform_string())
        shutil.move(os.path.join(operadriver_dir, "operadriver"), dest)
        shutil.rmtree(operadriver_dir)

        path = find_executable("operadriver")
        st = os.stat(path)
        os.chmod(path, st.st_mode | stat.S_IEXEC)
        return path

def submit(request):
    team = get_team(request.user)
    params = dict()

    if request.method == 'POST':
        form = forms.UploadSubmissionForm(request.POST, request.FILES)
        params['form'] = form

        if form.is_valid():
            submission = Submission(
                team=team,
                package=request.FILES['file'],
                command=request.POST['command'],
            )
            submission.save()

            error = utils.unzip(submission.package.path)
            if error:
                submission.delete()
                params['error'] = error
                return render_submit(request, params)

            try:
                execute_tester(submission)
            except Exception as error:
                print u'ERROR: Blad wewnetrzny testerki:', error
                

            return my_results(
                request, message=_(u'Rozwiązanie zostało wysłane.'))

    return render_submit(request, params)

    def install_prefs(self, binary, dest=None, channel=None):
        version, channel_ = self.get_version_and_channel(binary)
        if channel is not None and channel != channel_:
            # Beta doesn't always seem to have the b in the version string, so allow the
            # manually supplied value to override the one from the binary
            self.logger.warning("Supplied channel doesn't match binary, using supplied channel")
        elif channel is None:
            channel = channel_
        if dest is None:
            dest = os.pwd

        dest = os.path.join(dest, "profiles", channel)
        if version:
            dest = os.path.join(dest, version)
        have_cache = False
        if os.path.exists(dest):
            if channel != "nightly":
                have_cache = True
            else:
                now = datetime.now()
                have_cache = (datetime.fromtimestamp(os.stat(dest).st_mtime) >
                              now - timedelta(days=1))

        # If we don't have a recent download, grab and extract the latest one
        if not have_cache:
            if os.path.exists(dest):
                shutil.rmtree(dest)
            os.makedirs(dest)

            url = self.get_profile_bundle_url(version, channel)

            self.logger.info("Installing test prefs from %s" % url)
            try:
                extract_dir = tempfile.mkdtemp()
                unzip(get(url).raw, dest=extract_dir)

                profiles = os.path.join(extract_dir, os.listdir(extract_dir)[0], 'testing', 'profiles')
                for name in os.listdir(profiles):
                    path = os.path.join(profiles, name)
                    shutil.move(path, dest)
            finally:
                shutil.rmtree(extract_dir)
        else:
            self.logger.info("Using cached test prefs from %s" % dest)

        return dest

def main(url=ZIP_CODE_URL):
    path = utils.DATASET_HOME + ADDRESS_ZIP
    utils.download(url, path)
    files = utils.unzip(path)
    if files and len(files) > 0:
        write_address(files[0])
    else:
        print("failed to download or unzip the file. please see at {0}.".format(utils.DATASET_HOME))

def get_shape_from_coords(coords):
    """ Return shape of smallest well plate that can contain provided indices."""  
    well_plate_shapes = [(2,4),(8,12)] # 8 well slide, 96 well plate
    rows,cols = unzip(coords)
    for shape in well_plate_shapes:
        if rows |are_all_in| range(shape[0]) and cols |are_all_in| range(shape[1]):
            return shape
    raise Exception("Given well coordinates do not fit in plate shapes:{}".format(well_plate_shapes))

    def install_certutil(self, dest=None):
        # TODO: this doesn't really work because it just gets the binary, and is missing the
        # libnss3 library. Getting that means either downloading the corresponding Firefox
        # and extracting the library (which is hard on mac becase DMG), or maybe downloading from
        # nss's treeherder builds?
        if dest is None:
            dest = os.pwd

        # Don't create a path like bin/bin/certutil
        split = os.path.split(dest)
        if split[1] == "bin":
            dest = split[0]

        resp = self.get_from_nightly(
            "<a[^>]*>(firefox-\d+\.\d(?:\w\d)?.en-US.%s\.common\.tests.zip)</a>" % self.platform_string())
        bin_path = path("bin/certutil", exe=True)
        unzip(resp.raw, dest=dest, limit=[bin_path])

        return os.path.join(dest, bin_path)

def validate_and_save_checkpoint(model_options, dp, params, tparams, f_pred, f_pred_prob, kf_valid, save_n):
  scores = prediction(f_pred, f_pred_prob, dp.prepare_valid_or_test_batch_image_data, 'val', kf_valid, dp.ix_to_word, dp.get_raw_sentences_from_imgid, model_options, save_n['prediction'])
  # saving a checkpoint
  save_path = os.path.join(model_options['checkpoint_save_dir'], "lstm_{0}_{1:.2f}.npz".format(save_n['checkpoint'], scores['Bleu_4'] * 100))
  params = utils.unzip(tparams)
  numpy.savez(save_path, checkpoint_save_n=save_n['checkpoint'], scores=scores, **params)
  pickle.dump(model_options, open('%s.pkl' % save_path, 'wb'), -1)
  print 'Saved checkpoint to', os.path.abspath(save_path)
  
  save_n['checkpoint'] = save_n['checkpoint'] + 1
  save_n['prediction'] = save_n['prediction'] + 1
  
  return scores

def mfold_result(task_id, zipname="now.zip", path="./results/mfold/", verbose=True):
    """Gets mfold result via task_id

    Args:
        task_id: Id of task which was given by RESTful API
        zipname: Name of zip file in which client will save results.
            After save this file is removed
        path: Path where results should be stored
        verbose: Bool which tells if function should print what she actualy does

    Returns:
        None
    """
    req = get_request("mfold_result", task_id)
    with open(zipname, "wb") as f:
        for chunk in req.iter_content():
            f.write(chunk)
    unzip(zipname, path)
    os.remove(zipname)

    if verbose:
        print("Result in: {}/".format(path + task_id))

    def install_webdriver(self, dest=None, channel=None):
        """Install latest Geckodriver."""
        if dest is None:
            dest = os.getcwd()

        if channel == "nightly":
            path = self.install_geckodriver_nightly(dest)
            if path is not None:
                return path
            else:
                self.logger.warning("Nightly webdriver not found; falling back to release")

        version = self._latest_geckodriver_version()
        format = "zip" if uname[0] == "Windows" else "tar.gz"
        self.logger.debug("Latest geckodriver release %s" % version)
        url = ("https://github.com/mozilla/geckodriver/releases/download/%s/geckodriver-%s-%s.%s" %
               (version, version, self.platform_string_geckodriver(), format))
        if format == "zip":
            unzip(get(url).raw, dest=dest)
        else:
            untar(get(url).raw, dest=dest)
        return find_executable(os.path.join(dest, "geckodriver"))

    def install_prefs(self, binary, dest=None):
        version, channel = self.get_version_and_channel(binary)
        if dest is None:
            dest = os.pwd

        dest = os.path.join(dest, "profiles", channel, version)
        have_cache = False
        if os.path.exists(dest):
            if channel != "nightly":
                have_cache = True
            else:
                now = datetime.now()
                have_cache = (datetime.fromtimestamp(os.stat(dest).st_mtime) >
                              now - timedelta(days=1))

        # If we don't have a recent download, grab and extract the latest one
        if not have_cache:
            if os.path.exists(dest):
                shutil.rmtree(dest)
            os.makedirs(dest)

            url = self.get_profile_bundle_url(version, channel)

            print("Installing test prefs from %s" % url)
            try:
                extract_dir = tempfile.mkdtemp()
                unzip(get(url).raw, dest=extract_dir)

                profiles = os.path.join(extract_dir, os.listdir(extract_dir)[0], 'testing', 'profiles')
                for name in os.listdir(profiles):
                    path = os.path.join(profiles, name)
                    shutil.move(path, dest)
            finally:
                shutil.rmtree(extract_dir)
        else:
            print("Using cached test prefs from %s" % dest)

        return dest