def test_check_output_nonzero(self):
     # check_call() function with non-zero return code
     try:
         subprocess.check_output(
                 [sys.executable, "-c", "import sys; sys.exit(5)"])
     except subprocess.CalledProcessError, e:
         self.assertEqual(e.returncode, 5)

 def do_download(self, args):
     """Download a covert file to the local file system.\n
     Use: download [covert path] [local path]"""
     a = args.split()
     if len(a) != 2:  # local path file
         print("Use: download [covert path] [local path]")
         return
     else:
         covert_path = a[0]
         local_path = a[1]
         try:
             subprocess.check_output(
                 ["ls " + local_path.rsplit('/', 1)[0]],
                 shell=True
             )
         except:
             print("Given directory does not exist")
             return
         try:
             covert_contents = self.fs.getcontents(covert_path)
         except:
             print("Given covert path does not exist")
             return
         with open(local_path, 'w') as f:
             f.write(self.san_file(covert_contents))

 def test_version(self):
     try:
         subprocess.check_output(["Trinity", "--version"])
         self.fail("Version returned 0 errorcode!")
     except subprocess.CalledProcessError as e:
         self.assertTrue('Trinity version: __TRINITY_VERSION_TAG__' in e.output)
         self.assertTrue('using Trinity devel version. Note, latest production release is: v2.0.6' in e.output)

 def find_word(self, word, file_name, directory=None):
     if directory is None:
         directory = subprocess.check_output("pwd", shell=True).rstrip()
     results = subprocess.check_output("grep '{}' {}/{}".format(word, directory, file_name), shell=True)
     results.split('\n')
     results.pop()
     return results

def testLapack(root,config):
    """Test if BLAS functions correctly and set whether G77 calling convention is needed"""
    cwd = os.getcwd()
    blas = config.get('BLAS','lib')
    lapack = config.get('LAPACK','lib')
    cxx=config.get('Main','cxx')
    cc=config.get('Main','cc')
    cflags = config.get('Main','cflags')
    cxxflags = config.get('Main','cxxflags')
    cmake_exe = config.get('CMake','exe')
    prefix = config.get('Main','prefix')
    fnull = open(os.devnull,'w')

    if sys.platform.startswith('darwin'):
        ld_path = "export DYLD_LIBRARY_PATH="+prefix+"/bempp/lib:$DYLD_LIBRARY_PATH; "
    elif sys.platform.startswith('linux'):
        ld_path = "export LD_LIBRARY_PATH="+prefix+"/bempp/lib:$LD_LIBRARY_PATH; "
    else:
        raise Exception("Wrong architecture.")

    checkCreateDir(root+"/test_lapack/build")
    os.chdir(root+"/test_lapack/build")
    config_string = "CC="+cc+" CXX="+cxx+" CFLAGS='"+cflags+"' CXXFLAGS='"+cxxflags+"' "+cmake_exe+" -D BLAS_LIBRARIES:STRING=\""+blas+"\" -D LAPACK_LIBRARIES=\""+lapack+"\" .."
    try:
        check_output(config_string,shell=True,stderr=subprocess.STDOUT)
        check_output("make",shell=True,stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError, ex:
        fnull.close()
        raise Exception("Building LAPACK tests failed with the following output:\n" +
                        ex.output +
                        "\nPlease check your compiler as well as BLAS and Lapack "
                        "library settings.\n")

def killemulator():
    try:
        subprocess.check_output(PLATDIR + "\\AppInventor\\commands-for-Appinventor\\kill-emulator", shell=True)
        print "Killed emulator\n"
    except subprocess.CalledProcessError as e:
        print "Problem stopping emulator : status %i\n" % e.returncode
        return ''

    def __execute_statement(self, statement, db_name=None):
        file = tempfile.NamedTemporaryFile(dir='/tmp',
                                           prefix='tinyAPI_rdbms_builder_',
                                           delete=False)

        file.write(statement.encode())
        file.close()

        try:
            subprocess.check_output(
                self.__get_exec_sql_command()
                + ('' if db_name is None else ' --database=' + db_name)
                + ' < ' + file.name,
                stderr=subprocess.STDOUT,
                shell=True)
        except subprocess.CalledProcessError as e:
            message = e.output.rstrip().decode()

            if len(statement) <= 2048:
                raise RDBMSBuilderException(
                        'execution of this:\n\n'
                        + statement
                        + "\n\nproduced this error:\n\n"
                        + message
                        + self.__enhance_build_error(message))
            else:
                raise RDBMSBuilderException(
                        'execution of this file:\n\n'
                        + file.name
                        + "\n\nproduced this error:\n\n"
                        + message
                        + self.__enhance_build_error(message))

        os.remove(file.name)

    def parse_args(self, args=None, namespace=None):
        result = super(ArgumentParser, self).parse_args(args, namespace)

        adb_path = result.adb_path or "adb"

        # Try to run the specified adb command
        try:
            subprocess.check_output([adb_path, "version"],
                                    stderr=subprocess.STDOUT)
        except (OSError, subprocess.CalledProcessError):
            msg = "ERROR: Unable to run adb executable (tried '{}')."
            if not result.adb_path:
                msg += "\n       Try specifying its location with --adb."
            sys.exit(msg.format(adb_path))

        try:
            if result.device == "-a":
                result.device = adb.get_device(adb_path=adb_path)
            elif result.device == "-d":
                result.device = adb.get_usb_device(adb_path=adb_path)
            elif result.device == "-e":
                result.device = adb.get_emulator_device(adb_path=adb_path)
            else:
                result.device = adb.get_device(result.serial, adb_path=adb_path)
        except (adb.DeviceNotFoundError, adb.NoUniqueDeviceError, RuntimeError):
            # Don't error out if we can't find a device.
            result.device = None

        return result

def _wait_until_port_listens(port, timeout=10, warn=True):
    """Wait for a process to start listening on the given port.

    If nothing listens on the port within the specified timeout (given
    in seconds), print a warning on stderr before returning.
    """
    try:
        subprocess.check_output(['which', 'lsof'])
    except subprocess.CalledProcessError:
        print("WARNING: No `lsof` -- cannot wait for port to listen. "+
              "Sleeping 0.5 and hoping for the best.",
              file=sys.stderr)
        time.sleep(0.5)
        return
    deadline = time.time() + timeout
    while time.time() < deadline:
        try:
            subprocess.check_output(
                ['lsof', '-t', '-i', 'tcp:'+str(port)])
        except subprocess.CalledProcessError:
            time.sleep(0.1)
            continue
        return True
    if warn:
        print(
            "WARNING: Nothing is listening on port {} (waited {} seconds).".
            format(port, timeout),
            file=sys.stderr)
    return False

def publish_release():
    tag = info['git_tag_name']
    remote = opts['git_remote']

    if not tag:
        issues.warn('Published release is not from a tag. The release you are '
                    'publishing was not retrieved from a tag. For safety '
                    'reasons, it will not get pushed upstream.')
    else:
        if not opts['dry_run']:
            log.info('Pushing tag \'{}\' to remote \'{}\''.format(
                tag, remote
            ))
            try:
                args = [opts['git_binary'],
                        'push',
                        remote,
                        tag]
                subprocess.check_output(args, cwd=opts['root'])
            except subprocess.CalledProcessError as e:
                issues.error('Failed to push tag:\n{}'.format(e.output))

        else:
            log.info('Not pushing tag \'{}\' to remote \'{}\' (dry-run)'
                     .format(tag, remote))

def data_collection_stats():
	print(check_output(["ls", "../input"]).decode("utf8"))
	train_images = check_output(["ls", "../input/train_photos"]).decode("utf8")
	print(train_images[:])
	print('time elapsed ' + str((time.time() - config.start_time)/60))

	print('Reading data...')
	train_photos = pd.read_csv('../input/train_photo_to_biz_ids.csv')
	train_photos.sort_values(['business_id'], inplace=True)
	train_photos.set_index(['business_id'])

	test_photos = pd.read_csv('../input/test_photo_to_biz.csv')
	test_photos.sort_values(['business_id'], inplace=True)
	test_photos.set_index(['business_id'])

	train = pd.read_csv('../input/train.csv')
	train.sort_values(['business_id'], inplace=True)
	train.reset_index(drop=True)

	print('Number of training samples: ', train.shape[0])
	print('Number of train samples: ', len(set(train_photos['business_id'])))
	print('Number of test samples: ', len(set(test_photos['business_id'])))
	print('Finished reading data...')
	print('Time elapsed: ' + str((time.time() - config.start_time)/60))

	print('Reading/Modifying images..')

	return (train_photos, test_photos, train)

def convert_kml_ground_overlay_to_geotiff(kml_path, other_file_path):
    """Write a geotiff file to disk from the provided kml and image

    KML files that specify GroundOverlay as their type are accompanied by a
    raster file. Since there is no direct support in geoserver for this type
    of KML, we extract the relevant information from the KML file and convert
    the raster file to a geotiff.

    """

    with open(kml_path) as kml_handler:
        kml_bytes = kml_handler.read()
    kml_doc, namespaces = get_kml_doc(kml_bytes)
    bbox = GdalBoundingBox(
        ulx=_extract_bbox_param(kml_doc, namespaces, "west"),
        uly=_extract_bbox_param(kml_doc, namespaces, "north"),
        lrx=_extract_bbox_param(kml_doc, namespaces, "east"),
        lry=_extract_bbox_param(kml_doc, namespaces, "south"),
    )
    dirname, basename = os.path.split(other_file_path)
    output_path = os.path.join(
        dirname,
        ".".join((os.path.splitext(basename)[0], "tif"))
    )
    command = [
        "gdal_translate",
        "-of", "GTiff",
        "-a_srs", "EPSG:4326",  # KML format always uses EPSG:4326
        "-a_ullr", bbox.ulx, bbox.uly, bbox.lrx, bbox.lry,
        other_file_path,
        output_path
    ]
    subprocess.check_output(command)
    return output_path

    def _uninstall_simple(self, bundle, pkginfo, check, inter=False):
        # Simple apps are handled by calling an executable "uninstall" file
        # from the package special directory

        try:
            cmd = glob.glob(os.path.join(bundle,
                                         'special',
                                         "uninstall") + ".*")[0]
        except IndexError as e:
            return "Unnstall script not found"

        if inter:
            platutils.win.run_as_current_user(cmd)
            operator = "_check_{0}".format(check[0])
            inscheck = getattr(self, operator)(check[1])
            if inscheck:
                out = "Uninstallation failed."
            else:
                out = None
        else:
            out = None
            a = os.getcwd()
            os.chdir(os.path.join(bundle, 'special'))
            try:
                subprocess.check_output(cmd, stderr=subprocess.STDOUT)
            except subprocess.CalledProcessError as e:
                out = "Error running " + e.output.decode()
            except WindowsError as e:
                out = "Error running " + cmd + ": " + e.strerror
            os.chdir(a)

        return out

def process(network, name, os_):
    tmp_network = network+'_tinczip'
    tmp_dir = os.path.join(tinc_dir,tmp_network)
    tmp_hosts_dir = os.path.join(tmp_dir,'hosts')
    os.mkdir(tmp_dir)
    os.mkdir(tmp_hosts_dir)
    assigned_ip = find_free_ip(network)
    tinc_conf = '''Name = {name}\nConnectTo = {host}'''.format(name=name,host=get_hostname(network))
    host_conf = '''Subnet = {}/32'''.format(assigned_ip)

    if os_ == 'windows':
        tinc_conf += 'Interface = VPN'
    # if _os == "linux":
    #     tinc_conf += 'Device = /dev/net/tun0'

    #write tinc.conf
    with open(os.path.join(tmp_dir, 'tinc.conf'), 'w') as tinc_conf_file:
        tinc_conf_file.write(tinc_conf)
    #write hostname file
    with open(os.path.join(tmp_hosts_dir, name), 'w') as host_conf_file:
        host_conf_file.write(host_conf)

    subprocess.check_output('tincd -n {} -K4096'.format(tmp_network).split())


    #copy client key to server folder
    local_hosts_dir = _get_hosts_dir(network)
    shutil.copy(os.path.join(tmp_hosts_dir, name),local_hosts_dir)
    #copy server key to tmp folder
    shutil.copy(os.path.join(local_hosts_dir, get_hostname(network)), tmp_hosts_dir)

    if os_ == 'linux' or os_ == 'osx':
        #make tinc-up and tinc-down
        tinc_up = 'ifconfig $INTERFACE {} netmask 255.255.255.0'.format(assigned_ip)
        tinc_down = 'ifconfig $INTERFACE down'
        tinc_up_path = os.path.join(tmp_dir,'tinc-up')
        tinc_down_path = os.path.join(tmp_dir,'tinc-down')
        with open(tinc_up_path, 'w') as tu:
            tu.write(tinc_up)
        st = os.stat(tinc_up_path)
        os.chmod(tinc_up_path, st.st_mode | stat.S_IXUSR)
        with open(tinc_down_path, 'w') as td:
            td.write(tinc_down)
        st = os.stat(tinc_down_path)
        os.chmod(tinc_down_path, st.st_mode | stat.S_IXUSR)

    zip_location = os.path.join(tinc_dir,tmp_network)

    zip_file = shutil.make_archive(zip_location, 'zip', tmp_dir)
    zip_bytes = ''
    with open(zip_file,'rb') as zip_:
        zip_bytes = zip_.read()

    #cleanup
    shutil.rmtree(tmp_dir)
    os.remove(zip_file)
    return send_file(io.BytesIO(zip_bytes),
                     attachment_filename='{}.zip'.format(tmp_network),
                     mimetype='application/zip',
                     as_attachment=True)

    def __init__(self, elf):
        ss = elf.getSections()
        now = datetime.now()

        dprint("Running \"%s\"..." % self.git_cmd)

        git_version = subprocess.check_output(
            self.git_cmd.split(" ")
        ).strip()

        dprint("Running \"%s\"..." % self.git_br_cmd)
        git_branch = subprocess.check_output(
            self.git_br_cmd.split(" ")
        ).strip()

        dprint("  %s" % git_version)

        self.image_crc    = 0x00000000  # must be calculated later
        self.image_size   = ss[-1].lma + ss[-1].sh_size - ss[0].lma
        self.git_version  = git_version
        self.git_branch  = git_branch
        self.build_user   = getpass.getuser()
        self.build_host   = platform.node()
        self.build_date   = now.strftime("%Y-%m-%d")
        self.build_time   = now.strftime("%H:%M:%S")

 def _EnumerateHostInterfaces(self):
   host_platform = platform.GetHostPlatform().GetOSName()
   if host_platform == 'linux':
     return subprocess.check_output(['ip', 'addr']).splitlines()
   if host_platform == 'mac':
     return subprocess.check_output(['ifconfig']).splitlines()
   raise NotImplementedError('Platform %s not supported!' % host_platform)

def killadb():
    try:
        subprocess.check_output(PLATDIR + "\\AppInventor\\commands-for-Appinventor\\adb kill-server", shell=True)
        print "Killed adb\n"
    except subprocess.CalledProcessError as e:
        print "Problem stopping adb : status %i\n" % e.returncode
        return ''

def determine_python_includes_ldflags_use_python_config():
    # https://docs.python.org/2/library/sys.html#sys.executable
    python_exe_path = sys.executable
    if not python_exe_path:
        # "If Python is unable to retrieve the real path to its executable,
        # `sys.executable` will be an empty string or None."
        return (None, None, None)

    python_exe_name = os.path.split(python_exe_path)[-1]
    python_config_exe_name = "%s-config" % python_exe_name
    try:
        includes = subprocess.check_output([python_config_exe_name, "--includes"])
        #libs = subprocess.check_output([python_config_exe_name, "--libs"])
        #cflags = subprocess.check_output([python_config_exe_name, "--cflags"])
        ldflags = subprocess.check_output([python_config_exe_name, "--ldflags"])
    except OSError as e:
        print("Caught OSError(%s)" % str(e), file=sys.stderr)
        return (None, None, python_config_exe_name)

    python_ver = sys.version_info
    if python_ver.major >= 3:
        # Stupid byte/string dichotomy...
        includes = includes.decode(encoding='UTF-8')
        ldflags = ldflags.decode(encoding='UTF-8')

    includes = includes.split()
    ldflags = ldflags.split()

    return (includes, ldflags, python_config_exe_name)

def _run_gemini_stats(bam_file, data, out_dir):
    """Retrieve high level variant statistics from Gemini.
    """
    out = {}
    gemini_db = (data.get("variants", [{}])[0].get("population", {}).get("db") 
                 if data.get("variants") else None)
    if gemini_db:
        gemini_stat_file = "%s-stats.yaml" % os.path.splitext(gemini_db)[0]
        if not utils.file_uptodate(gemini_stat_file, gemini_db):
            gemini = config_utils.get_program("gemini", data["config"])
            tstv = subprocess.check_output([gemini, "stats", "--tstv", gemini_db])
            gt_counts = subprocess.check_output([gemini, "stats", "--gts-by-sample", gemini_db])
            dbsnp_count = subprocess.check_output([gemini, "query", gemini_db, "-q",
                                                   "SELECT count(*) FROM variants WHERE in_dbsnp==1"])
            out["Transition/Transversion"] = tstv.split("\n")[1].split()[-1]
            for line in gt_counts.split("\n"):
                parts = line.rstrip().split()
                if len(parts) > 0 and parts[0] == data["name"][-1]:
                    _, hom_ref, het, hom_var, _, total = parts
                    out["Variations (total)"] = int(total)
                    out["Variations (heterozygous)"] = int(het)
                    out["Variations (homozygous)"] = int(hom_var)
                    break
            out["Variations (in dbSNP)"] = int(dbsnp_count.strip())
            if out.get("Variations (total)") > 0:
                out["Variations (in dbSNP) pct"] = "%.1f%%" % (out["Variations (in dbSNP)"] /
                                                               float(out["Variations (total)"]) * 100.0)
            with open(gemini_stat_file, "w") as out_handle:
                yaml.safe_dump(out, out_handle, default_flow_style=False, allow_unicode=False)
        else:
            with open(gemini_stat_file) as in_handle:
                out = yaml.safe_load(in_handle)
    return out

	def run(self):
		# Scrapy starts a Twisted reactor. You can try invoking scrapy
		# programmatically, but it does not play well with Luigi process pools.
		# So let's just start a sub process.
		tmp_output_path = self.output().path + "_tmp"
		subprocess.check_output(["scrapy", "crawl", "city", "-a", "city={}".format(self.city), "-o", tmp_output_path, "-t", "jsonlines"])
		os.rename(tmp_output_path, self.output().path)