def get_cpuinfo(platform='linux'):
    vendor_string = ''
    feature_string = ''
    if platform == "darwin":
        vendor_string = utils.run_command(['sysctl',
                                           '-n',
                                           'machdep.cpu.vendor'])
        feature_string = utils.run_command(['sysctl',
                                            '-n',
                                            'machdep.cpu.features'])
        # osx reports AVX1.0 while linux reports it as AVX
        feature_string = feature_string.replace("AVX1.0", "AVX")
    elif os.path.isfile('/proc/cpuinfo'):
        with open('/proc/cpuinfo') as f:
            cpuinfo = f.readlines()
        for line in cpuinfo:
            if 'vendor_id' in line:
                vendor_string = line.split(':')[1].strip()
            elif 'flags' in line:
                feature_string = line.split(':')[1].strip()
            if vendor_string and feature_string:
                break
    else:
        raise ValueError("Unknown platform, could not find CPU information")
    return (vendor_string.strip(), feature_string.strip())

def extract_forms(url, follow = "false", cookie_jar = None, filename = "forms.json"):
	utils.remove_file(os.path.join(os.path.dirname(__file__), filename))
	
	if cookie_jar == None:
		try:
			out = utils.run_command('{} && {}'.format(
				utils.cd(os.path.dirname(os.path.abspath(__file__))),
				'scrapy crawl form -o {} -a start_url="{}" -a follow={} -a proxy={}'.format(filename, url, follow, HTTP_PROXY)), EXTRACT_WAIT_TIME)
		except:
			out = utils.run_command('{} && {}'.format(
				utils.cd(os.path.dirname(os.path.abspath(__file__))),
				'scrapy crawl form -o {} -a start_url="{}" -a follow={}'.format(filename, url, follow)), EXTRACT_WAIT_TIME)
	else:
		cookie_jar_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), filename.replace('.json', '.txt'))
		cookie_jar.save(cookie_jar_path)
		out = utils.run_command('{} && {}'.format(
			utils.cd(os.path.dirname(os.path.abspath(__file__))),
			'scrapy crawl form_with_cookie -o {} -a start_url="{}" -a cookie_jar={}'.format(filename, url, cookie_jar_path)), EXTRACT_WAIT_TIME)

	with open(os.path.join(os.path.dirname(__file__), filename)) as json_forms:
		forms = json.load(json_forms)

	utils.remove_file(os.path.join(os.path.dirname(__file__), filename))
		
	return forms

def unf(pathin, destdir):
    fname = os.path.split(pathin)[1]
    outname = os.path.splitext(fname)[0] + '.mzML'
    outpath = os.path.join(destdir, outname)
    cmd = "%s %s > %s" % (exec_unf, pathin, outpath)
    ut.run_command(cmd)
    return outpath

def main():
    distro = get_distro()
    print()
    packages = get_packages(distro)
    dependencies = get_dependencies(distro, packages)

    if dependencies:
        install_command = install_commands[distro] + list(dependencies)

        print()
        print('Installing packages...')
        print(' '.join(install_command))
        run_command(install_command)
    else:
        print()
        print('No packages to install')

    print()
    print('Fetching submodules')
    for package in packages:
        if package.has_submodules:
            run_command(['git', 'submodule', 'update', '--init', '--recursive', package.path])

    print()
    for package in packages:
        print('Setting up {}'.format(package))
        package.setup(distro)

def get_all_wmtdata():
    thread_mono = Process(target = get_all_wmt_monolingual)
    thread_para = Process(target = get_all_wmt_parallel)
    thread_mono.start(); thread_para.start()
    thread_mono.join(); thread_para.join()
    homedir = os.path.expanduser("~")
    run_command('mv wmt-data '+homedir)

    def install_source (self, source_path):
        prefix = self.get ('mingw prefix')
        confflags="--prefix=%s --host=i586-mingw32msvc --with-gcc-arch=prescott --enable-portable-binary --with-our-malloc16 --with-windows-f77-mangling --enable-shared --disable-static --enable-threads --with-combined-threads" % (unwin(prefix))
        confflags="--prefix=%s --host=i586-mingw32msvc --with-gcc-arch=native --enable-portable-binary --with-our-malloc16 --with-windows-f77-mangling --enable-shared --disable-static" % (unwin(prefix))
        wd = os.path.join (source_path, 'double-mingw32')
        if 1:
            shutil.rmtree(wd, ignore_errors = True)
            if not os.path.isdir(wd):
                os.makedirs (wd)
        conf = unwin(os.path.join (source_path, 'configure'))
        bash = self.get('mingw bash')
        make = self.get('mingw make')
        if ' ' in conf: 
            raise RuntimeError("The path of fftw3 configure script cannot contain spaces: %r" % (conf))

        if 1:
            r = run_command('%s %s %s --enable-sse2' % (bash, conf, confflags), cwd=wd, env=self.environ,
                            verbose=True)
            if r[0]:
                return False
        r = run_command(make+' -j4', cwd=wd, env=self.environ, verbose=True)
        if r[0]:
            return False
        r = run_command(make+' install', cwd=wd, env=self.environ, verbose=True)
        return not r[0]

def has_dependencies_installed():
    try:
        import z3
        import z3.z3util
        z3_version =  z3.get_version_string()
        tested_z3_version = '4.5.1'
        if compare_versions(z3_version, tested_z3_version) > 0:
            logging.warning("You are using an untested version of z3. %s is the officially tested version" % tested_z3_version)
    except:
        logging.critical("Z3 is not available. Please install z3 from https://github.com/Z3Prover/z3.")
        return False

    if not cmd_exists("evm"):
        logging.critical("Please install evm from go-ethereum and make sure it is in the path.")
        return False
    else:
        cmd = "evm --version"
        out = run_command(cmd).strip()
        evm_version = re.findall(r"evm version (\d*.\d*.\d*)", out)[0]
        tested_evm_version = '1.7.3'
        if compare_versions(evm_version, tested_evm_version) > 0:
            logging.warning("You are using evm version %s. The supported version is %s" % (evm_version, tested_evm_version))

    if not cmd_exists("solc"):
        logging.critical("solc is missing. Please install the solidity compiler and make sure solc is in the path.")
        return False
    else:
        cmd = "solc --version"
        out = run_command(cmd).strip()
        solc_version = re.findall(r"Version: (\d*.\d*.\d*)", out)[0]
        tested_solc_version = '0.4.19'
        if compare_versions(solc_version, tested_solc_version) > 0:
            logging.warning("You are using solc version %s, The latest supported version is %s" % (solc_version, tested_solc_version))

    return True

    def test_get_device_symlinks(self):
        """Verify that getting device symlinks works as expected"""

        with self.assertRaises(GLib.GError):
            BlockDev.utils_get_device_symlinks("no_such_device")

        symlinks = BlockDev.utils_get_device_symlinks(self.loop_dev)
        # there should be at least 2 symlinks for something like "/dev/sda" (in /dev/disk/by-id/)
        self.assertGreaterEqual(len(symlinks), 2)

        symlinks = BlockDev.utils_get_device_symlinks(self.loop_dev[5:])
        self.assertGreaterEqual(len(symlinks), 2)

        # create an LV to get a device with more symlinks
        ret, _out, _err = run_command ("pvcreate %s" % self.loop_dev)
        self.assertEqual(ret, 0)
        self.addCleanup(run_command, "pvremove %s" % self.loop_dev)

        ret, _out, _err = run_command ("vgcreate utilsTestVG %s" % self.loop_dev)
        self.assertEqual(ret, 0)
        self.addCleanup(run_command, "vgremove -y utilsTestVG")

        ret, _out, _err = run_command ("lvcreate -n utilsTestLV -L 12M utilsTestVG")
        self.assertEqual(ret, 0)
        self.addCleanup(run_command, "lvremove -y utilsTestVG/utilsTestLV")

        symlinks = BlockDev.utils_get_device_symlinks("utilsTestVG/utilsTestLV")
        # there should be at least 4 symlinks for an LV
        self.assertGreaterEqual(len(symlinks), 4)

def pkgconfig_get_link_args(pkg, ucp='', system=True, static=True):
  havePcFile = pkg.endswith('.pc')
  pcArg = pkg
  if not havePcFile:
    if system:
      # check that pkg-config knows about the package in question
      run_command(['pkg-config', '--exists', pkg])
    else:
      # look for a .pc file
      if ucp == '':
        ucp = default_uniq_cfg_path()
      pcfile = pkg + '.pc' # maybe needs to be an argument later?

      pcArg = os.path.join(get_cfg_install_path(pkg, ucp), 'lib',
                           'pkgconfig', pcfile)

      if not os.access(pcArg, os.R_OK):
        error("Could not find '{0}'".format(pcArg), ValueError)

  static_arg = [ ]
  if static:
    static_arg = ['--static']

  libs_line = run_command(['pkg-config', '--libs'] + static_arg + [pcArg]);
  libs = libs_line.split()
  return libs

    def _luks_header_backup_restore(self, create_fn):
        succ = create_fn(self.loop_dev, PASSWD, None)
        self.assertTrue(succ)

        backup_file = os.path.join(self.backup_dir, "luks-header.txt")

        succ = BlockDev.crypto_luks_header_backup(self.loop_dev, backup_file)
        self.assertTrue(succ)
        self.assertTrue(os.path.isfile(backup_file))

        # now completely destroy the luks header
        ret, out, err = run_command("cryptsetup erase %s -q && wipefs -a %s" % (self.loop_dev, self.loop_dev))
        if ret != 0:
            self.fail("Failed to erase LUKS header from %s:\n%s %s" % (self.loop_dev, out, err))

        _ret, fstype, _err = run_command("blkid -p -ovalue -sTYPE %s" % self.loop_dev)
        self.assertFalse(fstype)  # false == empty

        # header is destroyed, should not be possible to open
        with self.assertRaises(GLib.GError):
            BlockDev.crypto_luks_open(self.loop_dev, "libblockdevTestLUKS", PASSWD, None)

        # and restore the header back
        succ = BlockDev.crypto_luks_header_restore(self.loop_dev, backup_file)
        self.assertTrue(succ)

        _ret, fstype, _err = run_command("blkid -p -ovalue -sTYPE %s" % self.loop_dev)
        self.assertEqual(fstype, "crypto_LUKS")

        # opening should now work
        succ = BlockDev.crypto_luks_open(self.loop_dev, "libblockdevTestLUKS", PASSWD)
        self.assertTrue(succ)

        succ = BlockDev.crypto_luks_close("libblockdevTestLUKS")
        self.assertTrue(succ)

def get_cpuinfo(platform_val='linux'):
    vendor_string = ''
    feature_string = ''
    if platform_val == "darwin":
        vendor_string = run_command(['sysctl', '-n', 'machdep.cpu.vendor'])
        feature_string = run_command(['sysctl', '-n', 'machdep.cpu.features'])
        # osx reports AVX1.0 while linux reports it as AVX
        feature_string = feature_string.replace("AVX1.0", "AVX")
        feature_string = feature_string.replace("SSE4.", "SSE4")
    elif os.path.isfile('/proc/cpuinfo'):
        with open('/proc/cpuinfo') as f:
            cpuinfo = f.readlines()
        # Compensate for missing vendor in ARM /proc/cpuinfo
        if get_native_machine() == 'aarch64':
            vendor_string = "arm"
        for line in cpuinfo:
            if 'vendor_id' in line:
                vendor_string = line.split(':')[1].strip()
            elif 'flags' in line:
                feature_string = line.split(':')[1].strip()
            elif line.startswith('Features'):
                feature_string = line.split(':')[1].strip()
            if vendor_string and feature_string:
                feature_string = feature_string.replace("sse4_", "sse4")
                break
    else:
        raise ValueError("Unknown platform, could not find CPU information")
    return (vendor_string.strip(), feature_string.strip())

 def allocation_lrc_file(self, song, lrc_path):    
     if os.path.exists(lrc_path):
         if self.vaild_lrc(lrc_path):
             save_lrc_path = self.get_lrc_filepath(song)
             if os.path.exists(save_lrc_path): os.unlink(save_lrc_path)
             utils.run_command("cp %s %s" % (lrc_path, save_lrc_path))
             Dispatcher.reload_lrc(song)

def _seed(torrent_path, seed_cache_path, torrent_seed_duration,
          torrent_listen_port_start, torrent_listen_port_end):
    plugin_path = os.path.dirname(inspect.getabsfile(inspect.currentframe()))
    seeder_path = os.path.join(plugin_path, SEEDER_PROCESS)
    seed_cmd = map(str, [seeder_path, torrent_path, seed_cache_path,
                         torrent_seed_duration, torrent_listen_port_start,
                         torrent_listen_port_end])
    utils.run_command(seed_cmd)

def make_partition(session, dev, partition_start, partition_end):
    dev_path = utils.make_dev_path(dev)

    if partition_end != "-":
        raise pluginlib.PluginError("Can only create unbounded partitions")

    utils.run_command(['sfdisk', '-uS', dev_path],
                      '%s,;\n' % (partition_start))

    def render(self):
        filename = "temp_" + self.output_file + ".gnuplot"
        f = open(filename, "w")
        f.write(self.script)
        f.close()

        utils.run_command(["gnuplot", filename])
        os.remove(filename)

    def download(self, temp_ver, store_metadata=True):
        """
        Retrieve the given template version

        Args:
            temp_ver (TemplateVersion): template version to retrieve
            store_metadata (bool): If set to ``False``, will not refresh the
                local metadata with the retrieved one

        Returns:
            None
        """
        dest = self._prefixed(temp_ver.name)
        temp_dest = '%s.tmp' % dest

        with lockfile.LockFile(dest):
            # Image was downloaded while we were waiting
            if os.path.exists(dest):
                return

            temp_ver.download(temp_dest)
            if store_metadata:
                with open('%s.metadata' % dest, 'w') as f:
                    utils.json_dump(temp_ver.get_metadata(), f)

            sha1 = hashlib.sha1()
            with open(temp_dest) as f:
                while True:
                    chunk = f.read(65536)
                    if not chunk:
                        break
                    sha1.update(chunk)
            if temp_ver.get_hash() != sha1.hexdigest():
                raise RuntimeError(
                    'Image %s does not match the expected hash %s' % (
                        temp_ver.name,
                        sha1.hexdigest(),
                    )
                )

            with open('%s.hash' % dest, 'w') as f:
                f.write(sha1.hexdigest())

            with log_utils.LogTask('Convert image', logger=LOGGER):
                utils.run_command(
                    [
                        'qemu-img',
                        'convert',
                        '-O',
                        'raw',
                        temp_dest,
                        dest,
                    ],
                )

            os.unlink(temp_dest)

            self._init_users(temp_ver)

 def sync_server(self, path):
     LOG.info('Syncing server ...')
     command = '{} && {} && unset DJANGO_SETTINGS_MODULE && python manage.py syncdb --noinput'.format(
         utils.to_env(self.base_path), utils.cd(path))
     output = utils.run_command(command)
     if 'Unknown command' in output[2]:
         command = '{} && {} && unset DJANGO_SETTINGS_MODULE && python manage.py migrate --noinput'.format(
         utils.to_env(self.base_path), utils.cd(path))
     return utils.run_command(command)

def exec_diff(old_filename, new_filename, diff_filename):
    """ Exec Latexdiff

        :param old_filename:
        :param new_filename:
        :param diff_filename:

    """
    run_command("latexdiff %s %s > %s" % (old_filename, new_filename, diff_filename))

def start_training():
  cmd = []
  cmd.append(os.path.join(CAFFE_BIN_DIR, 'caffe'))
  cmd.append('train')
  cmd.append('--solver=' + SOLVER_PROTOTXT)
  cmd.append('--weights=' + WEIGHTS_FILE)
  if is_gpu():
    cmd.append('--gpu=' + str(DEVICE_ID))
  utils.run_command(cmd)

def msconvert(fin):
    fout = fin.replace('.mzML','.mzXML')
    out_newdir = os.path.splitext(fout)[0]
    cmd = "%s %s -o %s" % (exec_msconvert, fin, out_newdir)
    ut.run_command(cmd)
    match_output = os.path.join(out_newdir, '*.mzXML')
    ut.run_command('mv %s %s' % (match_output, fout))
    os.rmdir(out_newdir)
    return fout