本文整理汇总了Python中reprounzip.common.record_usage函数的典型用法代码示例。如果您正苦于以下问题:Python record_usage函数的具体用法?Python record_usage怎么用?Python record_usage使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了record_usage函数的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Python代码示例。
示例1: should_mount_magic_dirs
def should_mount_magic_dirs(param):
"""Computes whether to mount directories inside the chroot.
"""
if os.getuid() != 0:
if param is True:
# Restoring the owner was explicitely requested
logging.critical("Not running as root, cannot mount /dev and "
"/proc")
sys.exit(1)
elif param is None:
# Nothing was requested
logging.warning("Not running as root, won't mount /dev and /proc")
ret = False
else:
# If False: skip warning
ret = False
else:
if param is None:
# Nothing was requested
logging.info("Running as root, will mount /dev and /proc")
ret = True
elif param is True:
ret = True
else:
# If False: skip warning
ret = False
record_usage(mount_magic_dirs=ret)
return ret
开发者ID:ReproNim,项目名称:reprozip,代码行数:28,代码来源:default.py
示例2: should_restore_owner
def should_restore_owner(param):
"""Computes whether to restore original files' owners.
"""
if os.getuid() != 0:
if param is True:
# Restoring the owner was explicitely requested
logging.critical("Not running as root, cannot restore files' "
"owner/group as requested")
sys.exit(1)
elif param is None:
# Nothing was requested
logging.warning("Not running as root, won't restore files' "
"owner/group")
ret = False
else:
# If False: skip warning
ret = False
else:
if param is None:
# Nothing was requested
logging.info("Running as root, we will restore files' "
"owner/group")
ret = True
elif param is True:
ret = True
else:
# If False: skip warning
ret = False
record_usage(restore_owner=ret)
return ret
开发者ID:ReproNim,项目名称:reprozip,代码行数:30,代码来源:default.py
示例3: select_box
def select_box(runs):
"""Selects a box for the experiment, with the correct distribution.
"""
distribution, version = runs[0]['distribution']
distribution = distribution.lower()
architecture = runs[0]['architecture']
record_usage(vagrant_select_box='%s;%s;%s' % (distribution, version,
architecture))
if architecture not in ('i686', 'x86_64'):
logging.critical("Error: unsupported architecture %s", architecture)
sys.exit(1)
# Ubuntu
if distribution == 'ubuntu':
if version == '12.04':
if architecture == 'i686':
return 'ubuntu', 'hashicorp/precise32'
else: # architecture == 'x86_64'
return 'ubuntu', 'hashicorp/precise64'
if version == '14.04':
if architecture == 'i686':
return 'ubuntu', 'ubuntu/trusty32'
else: # architecture == 'x86_64'
return 'ubuntu', 'ubuntu/trusty64'
if version != '15.04':
logging.warning("using Ubuntu 15.04 'Vivid' instead of '%s'",
version)
if architecture == 'i686':
return 'ubuntu', 'ubuntu/vivid32'
else: # architecture == 'x86_64':
return 'ubuntu', 'ubuntu/vivid64'
# Debian
else:
if distribution != 'debian':
logging.warning("unsupported distribution %s, using Debian",
distribution)
version = '8'
if (version == '7' or version.startswith('7.') or
version.startswith('wheezy')):
if architecture == 'i686':
return 'debian', 'remram/debian-7-i386'
else: # architecture == 'x86_64'
return 'debian', 'remram/debian-7-amd64'
if (version != '8' and not version.startswith('8.') and
not version.startswith('jessie')):
logging.warning("using Debian 8 'Jessie' instead of '%s'", version)
if architecture == 'i686':
return 'debian', 'remram/debian-8-i386'
else: # architecture == 'x86_64':
return 'debian', 'remram/debian-8-amd64'
开发者ID:aashish24,项目名称:reprozip,代码行数:55,代码来源:__init__.py
示例4: select_box
def select_box(runs, gui=False):
"""Selects a box for the experiment, with the correct distribution.
"""
distribution, version = runs[0]['distribution']
distribution = distribution.lower()
architecture = runs[0]['architecture']
record_usage(vagrant_select_box='%s;%s;%s;gui=%s' % (distribution, version,
architecture, gui))
if architecture not in ('i686', 'x86_64'):
logger.critical("Error: unsupported architecture %s", architecture)
sys.exit(1)
def find_distribution(parameter, distribution, version, architecture):
boxes = parameter['boxes']
for distrib in boxes:
if re.match(distrib['name'], distribution) is not None:
result = find_version(distrib, version, architecture)
if result is not None:
return result
default = parameter['default']
logger.warning("Unsupported distribution '%s', using %s",
distribution, default['name'])
result = default['architectures'].get(architecture)
if result:
return default['distribution'], result
def find_version(distrib, version, architecture):
if version is not None:
for box in distrib['versions']:
if re.match(box['version'], version) is not None:
result = box['architectures'].get(architecture)
if result is not None:
return box['distribution'], result
box = distrib['default']
if version is not None:
logger.warning("Using %s instead of '%s'",
box['name'], version)
result = box['architectures'].get(architecture)
if result is not None:
return box['distribution'], result
result = find_distribution(
get_parameter('vagrant_boxes_x' if gui else 'vagrant_boxes'),
distribution, version, architecture)
if result is None:
logger.critical("Error: couldn't find a base box for required "
"architecture")
sys.exit(1)
return result
开发者ID:ViDA-NYU,项目名称:reprozip,代码行数:52,代码来源:__init__.py
示例5: select_image
def select_image(runs):
"""Selects a base image for the experiment, with the correct distribution.
"""
distribution, version = runs[0]['distribution']
distribution = distribution.lower()
architecture = runs[0]['architecture']
record_usage(docker_select_box='%s;%s;%s' % (distribution, version,
architecture))
if architecture == 'i686':
logging.info("wanted architecture was i686, but we'll use x86_64 with "
"Docker")
elif architecture != 'x86_64':
logging.error("Error: unsupported architecture %s", architecture)
sys.exit(1)
# Ubuntu
if distribution == 'ubuntu':
if version == '12.04':
return 'ubuntu', 'ubuntu:12.04'
elif version == '14.04':
return 'ubuntu', 'ubuntu:14.04'
elif version == '14.10':
return 'ubuntu', 'ubuntu:14.10'
else:
if version != '15.04':
logging.warning("using Ubuntu 15.04 'Vivid' instead of '%s'",
version)
return 'ubuntu', 'ubuntu:15.04'
# Debian
else:
if distribution != 'debian':
logging.warning("unsupported distribution %s, using Debian",
distribution)
version = '8'
if (version == '6' or version.startswith('6.') or
version.startswith('squeeze')):
return 'debian', 'debian:squeeze'
elif (version == '7' or version.startswith('7.') or
version.startswith('wheezy')):
return 'debian', 'debian:wheezy'
else:
if (version != '8' and not version.startswith('8.') and
not version.startswith('jessie')):
logging.warning("using Debian 8 'Jessie' instead of '%s'",
version)
return 'debian', 'debian:jessie'
开发者ID:aashish24,项目名称:reprozip,代码行数:50,代码来源:docker.py
示例6: installpkgs
def installpkgs(args):
"""Installs the necessary packages on the current machine.
"""
if not THIS_DISTRIBUTION:
logging.critical("Not running on Linux")
sys.exit(1)
pack = args.pack[0]
missing = args.missing
# Loads config
runs, packages, other_files = load_config(pack)
try:
installer = select_installer(pack, runs)
except CantFindInstaller as e:
logging.error("Couldn't select a package installer: %s", e)
if args.summary:
# Print out a list of packages with their status
if missing:
print("Packages not present in pack:")
packages = [pkg for pkg in packages if not pkg.packfiles]
else:
print("All packages:")
pkgs = installer.get_packages_info(packages)
for pkg in packages:
print(" %s (required version: %s, status: %s)" % (
pkg.name, pkg.version, pkgs[pkg.name][1]))
else:
if missing:
# With --missing, ignore packages whose files were packed
packages = [pkg for pkg in packages if not pkg.packfiles]
# Installs packages
record_usage(installpkgs_installing=len(packages))
r, pkgs = installer.install(packages, assume_yes=args.assume_yes)
for pkg in packages:
req = pkg.version
real = pkgs[pkg.name][1]
if real == PKG_NOT_INSTALLED:
logging.warning("package %s was not installed", pkg.name)
else:
logging.warning("version %s of %s was installed, instead of "
"%s", real, pkg.name, req)
if r != 0:
logging.critical("Installer exited with %d", r)
sys.exit(r)
开发者ID:ReproNim,项目名称:reprozip,代码行数:48,代码来源:default.py
示例7: select_image
def select_image(runs):
"""Selects a base image for the experiment, with the correct distribution.
"""
distribution, version = runs[0]['distribution']
distribution = distribution.lower()
architecture = runs[0]['architecture']
record_usage(docker_select_box='%s;%s;%s' % (distribution, version,
architecture))
if architecture == 'i686':
logger.info("Wanted architecture was i686, but we'll use x86_64 with "
"Docker")
elif architecture != 'x86_64':
logger.error("Error: unsupported architecture %s", architecture)
sys.exit(1)
def find_distribution(parameter, distribution, version):
images = parameter['images']
for distrib in images:
if re.match(distrib['name'], distribution) is not None:
result = find_version(distrib, version)
if result is not None:
return result
default = parameter['default']
logger.warning("Unsupported distribution '%s', using %s",
distribution, default['name'])
return default['distribution'], default['image']
def find_version(distrib, version):
if version is not None:
for image in distrib['versions']:
if re.match(image['version'], version) is not None:
return image['distribution'], image['image']
image = distrib['default']
if version is not None:
logger.warning("Using %s instead of '%s'",
image['name'], version)
return image['distribution'], image['image']
return find_distribution(get_parameter('docker_images'),
distribution, version)
开发者ID:ViDA-NYU,项目名称:reprozip,代码行数:43,代码来源:reprounzip_docker.py
示例8: do_vistrails
def do_vistrails(target):
"""Create a VisTrails workflow that runs the experiment.
This is called from signals after an experiment has been setup by any
unpacker.
"""
record_usage(do_vistrails=True)
unpacker = signals.unpacker
dot_vistrails = Path('~/.vistrails').expand_user()
runs, packages, other_files = load_config(target / 'config.yml',
canonical=True)
for i, run in enumerate(runs):
module_name = write_cltools_module(run, dot_vistrails)
# Writes VisTrails workflow
bundle = target / 'vistrails.vt'
logging.info("Writing VisTrails workflow %s...", bundle)
vtdir = Path.tempdir(prefix='reprounzip_vistrails_')
try:
with vtdir.open('w', 'vistrail',
encoding='utf-8', newline='\n') as fp:
vistrail = VISTRAILS_TEMPLATE
cmdline = ' '.join(shell_escape(arg)
for arg in run['argv'])
vistrail = vistrail.format(
date='2014-11-12 15:31:18',
unpacker=unpacker,
directory=escape_xml(str(target.absolute())),
cmdline=escape_xml(cmdline),
module_name=module_name,
run=i)
fp.write(vistrail)
with bundle.open('wb') as fp:
z = zipfile.ZipFile(fp, 'w')
with vtdir.in_dir():
for path in Path('.').recursedir():
z.write(str(path))
z.close()
finally:
vtdir.rmtree()
开发者ID:aashish24,项目名称:reprozip,代码行数:42,代码来源:vistrails.py
示例9: main
def main():
"""Entry point when called on the command-line.
"""
# Locale
locale.setlocale(locale.LC_ALL, '')
# Parses command-line
# General options
def add_options(opts):
opts.add_argument('--version', action='version',
version="reprounzip version %s" % __version__)
# Loads plugins
for name, func, descr, descr_1 in get_plugins('reprounzip.plugins'):
func()
parser = RPUZArgumentParser(
description="reprounzip is the ReproZip component responsible for "
"unpacking and reproducing an experiment previously "
"packed with reprozip",
epilog="Please report issues to [email protected]")
add_options(parser)
parser.add_argument('-v', '--verbose', action='count', default=1,
dest='verbosity',
help="augments verbosity level")
subparsers = parser.add_subparsers(title="subcommands", metavar='')
# usage_report subcommand
parser_stats = subparsers.add_parser(
'usage_report',
help="Enables or disables anonymous usage reports")
add_options(parser_stats)
parser_stats.add_argument('--enable', action='store_true')
parser_stats.add_argument('--disable', action='store_true')
parser_stats.set_defaults(func=usage_report)
# Loads unpackers
for name, func, descr, descr_1 in get_plugins('reprounzip.unpackers'):
plugin_parser = subparsers.add_parser(
name, help=descr_1, description=descr,
formatter_class=argparse.RawDescriptionHelpFormatter)
add_options(plugin_parser)
info = func(plugin_parser)
plugin_parser.set_defaults(selected_unpacker=name)
if info is None:
info = {}
unpackers[name] = info
signals.pre_parse_args(parser=parser, subparsers=subparsers)
args = parser.parse_args()
signals.post_parse_args(args=args)
if getattr(args, 'func', None) is None:
parser.print_help(sys.stderr)
sys.exit(2)
signals.unpacker = getattr(args, 'selected_unpacker', None)
setup_logging('REPROUNZIP', args.verbosity)
setup_usage_report('reprounzip', __version__)
if hasattr(args, 'selected_unpacker'):
record_usage(unpacker=args.selected_unpacker)
signals.pre_setup.subscribe(lambda **kw: record_usage(setup=True))
signals.pre_run.subscribe(lambda **kw: record_usage(run=True))
try:
try:
args.func(args)
except UsageError:
raise
except Exception as e:
signals.application_finishing(reason=e)
submit_usage_report(result=type(e).__name__)
raise
else:
signals.application_finishing(reason=None)
except UsageError:
parser.print_help(sys.stderr)
sys.exit(2)
submit_usage_report(result='success')
sys.exit(0)
开发者ID:hugobowne,项目名称:reprozip,代码行数:80,代码来源:main.py
示例10: do_vistrails
def do_vistrails(target, pack=None, **kwargs):
"""Create a VisTrails workflow that runs the experiment.
This is called from signals after an experiment has been setup by any
unpacker.
"""
record_usage(do_vistrails=True)
config = load_config(target / 'config.yml', canonical=True)
# Writes VisTrails workflow
bundle = target / 'vistrails.vt'
logging.info("Writing VisTrails workflow %s...", bundle)
vtdir = Path.tempdir(prefix='reprounzip_vistrails_')
ids = IdScope()
try:
with vtdir.open('w', 'vistrail',
encoding='utf-8', newline='\n') as fp:
wf = Workflow(fp, ids)
# Directory module, refering to this directory
d = wf.add_module('%s:Directory' % rpz_id, rpz_version)
wf.add_function(d, 'directory',
[(directory_sig, str(target.resolve()))])
connect_from = d
for i, run in enumerate(config.runs):
inputs = sorted(n for n, f in iteritems(config.inputs_outputs)
if i in f.read_runs)
outputs = sorted(n for n, f in iteritems(config.inputs_outputs)
if i in f.write_runs)
ports = itertools.chain((('input', p) for p in inputs),
(('output', p) for p in outputs))
# Run module
r = wf.add_module('%s:Run' % rpz_id, rpz_version)
wf.add_function(r, 'cmdline', [
(string_sig,
' '.join(shell_escape(arg)
for arg in run['argv']))])
wf.add_function(r, 'run_number', [(integer_sig, i)])
# Port specs for input/output files
for type_, name in ports:
wf.add_port_spec(r, name, type_, [file_pkg_mod])
# Draw connection
wf.connect(connect_from, experiment_sig, 'experiment',
r, experiment_sig, 'experiment')
connect_from = r
wf.close()
with bundle.open('wb') as fp:
z = zipfile.ZipFile(fp, 'w')
with vtdir.in_dir():
for path in Path('.').recursedir():
z.write(str(path))
z.close()
finally:
vtdir.rmtree()
开发者ID:JosuaKrause,项目名称:reprozip,代码行数:62,代码来源:vistrails.py
示例11: vagrant_setup_create
def vagrant_setup_create(args):
"""Sets up the experiment to be run in a Vagrant-built virtual machine.
This can either build a chroot or not.
If building a chroot, we do just like without Vagrant: we copy all the
files and only get what's missing from the host. But we do install
automatically the packages whose files are required.
If not building a chroot, we install all the packages, and only unpack
files that don't come from packages.
In short: files from packages with packfiles=True will only be used if
building a chroot.
"""
if not args.pack:
logging.critical("setup/create needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
use_chroot = args.use_chroot
mount_bind = args.bind_magic_dirs
record_usage(use_chroot=use_chroot,
mount_bind=mount_bind)
signals.pre_setup(target=target, pack=pack)
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
runs, packages, other_files = config = load_config(target / 'config.yml',
True)
if not args.memory:
memory = None
else:
try:
memory = int(args.memory[-1])
except ValueError:
logging.critical("Invalid value for memory size: %r", args.memory)
sys.exit(1)
if args.base_image and args.base_image[0]:
record_usage(vagrant_explicit_image=True)
box = args.base_image[0]
if args.distribution:
target_distribution = args.distribution[0]
else:
target_distribution = None
else:
target_distribution, box = select_box(runs)
logging.info("Using box %s", box)
logging.debug("Distribution: %s", target_distribution or "unknown")
# If using chroot, we might still need to install packages to get missing
# (not packed) files
if use_chroot:
packages = [pkg for pkg in packages if not pkg.packfiles]
if packages:
record_usage(vagrant_install_pkgs=True)
logging.info("Some packages were not packed, so we'll install and "
"copy their files\n"
"Packages that are missing:\n%s",
' '.join(pkg.name for pkg in packages))
if packages:
try:
installer = select_installer(pack, runs, target_distribution)
except CantFindInstaller as e:
logging.error("Need to install %d packages but couldn't select a "
"package installer: %s",
len(packages), e)
target.mkdir(parents=True)
try:
# Writes setup script
logging.info("Writing setup script %s...", target / 'setup.sh')
with (target / 'setup.sh').open('w', encoding='utf-8',
newline='\n') as fp:
fp.write('#!/bin/sh\n\nset -e\n\n')
if packages:
# Updates package sources
fp.write(installer.update_script())
fp.write('\n')
# Installs necessary packages
fp.write(installer.install_script(packages))
fp.write('\n')
# TODO : Compare package versions (painful because of sh)
# Untar
if use_chroot:
fp.write('\n'
'mkdir /experimentroot; cd /experimentroot\n')
#.........这里部分代码省略.........
开发者ID:hugobowne,项目名称:reprozip,代码行数:101,代码来源:__init__.py
示例12: docker_setup_create
def docker_setup_create(args):
"""Sets up the experiment to be run in a Docker-built container.
"""
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# Unpacks configuration file
tar = tarfile.open(str(pack), 'r:*')
member = tar.getmember('METADATA/config.yml')
member.name = 'config.yml'
tar.extract(member, str(target))
tar.close()
# Loads config
runs, packages, other_files = load_config(target / 'config.yml', True)
if args.base_image:
record_usage(docker_explicit_base=True)
base_image = args.base_image[0]
if args.distribution:
target_distribution = args.distribution[0]
else:
target_distribution = None
else:
target_distribution, base_image = select_image(runs)
logging.info("Using base image %s", base_image)
logging.debug("Distribution: %s", target_distribution or "unknown")
target.mkdir(parents=True)
pack.copyfile(target / 'experiment.rpz')
# Writes Dockerfile
logging.info("Writing %s...", target / 'Dockerfile')
with (target / 'Dockerfile').open('w',
encoding='utf-8', newline='\n') as fp:
fp.write('FROM %s\n\n' % base_image)
# Installs busybox
download_file(busybox_url(runs[0]['architecture']),
target / 'busybox')
fp.write('COPY busybox /bin/busybox\n')
fp.write('COPY experiment.rpz /reprozip_experiment.rpz\n\n')
fp.write('RUN \\\n'
' chmod +x /bin/busybox && \\\n')
if args.install_pkgs:
# Install every package through package manager
missing_packages = []
else:
# Only install packages that were not packed
missing_packages = [pkg for pkg in packages if pkg.packfiles]
packages = [pkg for pkg in packages if not pkg.packfiles]
# FIXME : Right now, we need 'sudo' to be available (and it's not
# necessarily in the base image)
if packages:
record_usage(docker_install_pkgs=True)
else:
record_usage(docker_install_pkgs="sudo")
packages += [Package('sudo', None, packfiles=False)]
if packages:
try:
installer = select_installer(pack, runs, target_distribution)
except CantFindInstaller as e:
logging.error("Need to install %d packages but couldn't "
"select a package installer: %s",
len(packages), e)
sys.exit(1)
# Updates package sources
fp.write(' %s && \\\n' % installer.update_script())
# Installs necessary packages
fp.write(' %s && \\\n' % installer.install_script(packages))
logging.info("Dockerfile will install the %d software packages that "
"were not packed", len(packages))
# Untar
paths = set()
pathlist = []
dataroot = PosixPath('DATA')
# Adds intermediate directories, and checks for existence in the tar
tar = tarfile.open(str(pack), 'r:*')
missing_files = chain.from_iterable(pkg.files
for pkg in missing_packages)
for f in chain(other_files, missing_files):
path = PosixPath('/')
for c in f.path.components[1:]:
path = path / c
if path in paths:
continue
paths.add(path)
datapath = join_root(dataroot, path)
try:
tar.getmember(str(datapath))
except KeyError:
logging.info("Missing file %s", datapath)
#.........这里部分代码省略.........
开发者ID:aashish24,项目名称:reprozip,代码行数:101,代码来源:docker.py
示例13: directory_create
def directory_create(args):
"""Unpacks the experiment in a folder.
Only the files that are not part of a package are copied (unless they are
missing from the system and were packed).
In addition, input files are put in a tar.gz (so they can be put back after
an upload) and the configuration file is extracted.
"""
if not args.pack:
logging.critical("setup needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
if not issubclass(DefaultAbstractPath, PosixPath):
logging.critical("Not unpacking on POSIX system")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# Unpacks configuration file
tar = tarfile.open(str(pack), 'r:*')
member = tar.getmember('METADATA/config.yml')
member.name = 'config.yml'
tar.extract(member, str(target))
# Loads config
runs, packages, other_files = load_config_file(target / 'config.yml', True)
target.mkdir()
root = (target / 'root').absolute()
root.mkdir()
# Checks packages
missing_files = False
for pkg in packages:
if pkg.packfiles:
continue
for f in pkg.files:
f = Path(f.path)
if not f.exists():
logging.error(
"Missing file %s (from package %s that wasn't packed) "
"on host, experiment will probably miss it.",
f, pkg.name)
missing_files = True
if missing_files:
record_usage(directory_missing_pkgs=True)
logging.error(
"Some packages are missing, you should probably install "
"them.\nUse 'reprounzip installpkgs -h' for help")
# Unpacks files
if any('..' in m.name or m.name.startswith('/') for m in tar.getmembers()):
logging.critical("Tar archive contains invalid pathnames")
sys.exit(1)
members = [m for m in tar.getmembers() if m.name.startswith('DATA/')]
for m in members:
m.name = m.name[5:]
# Makes symlink targets relative
for m in members:
if not m.issym():
continue
linkname = PosixPath(m.linkname)
if linkname.is_absolute:
m.linkname = join_root(root, PosixPath(m.linkname)).path
logging.info("Extracting files...")
tar.extractall(str(root), members)
tar.close()
# Gets library paths
lib_dirs = []
p = subprocess.Popen(['/sbin/ldconfig', '-v', '-N'],
stdout=subprocess.PIPE)
try:
for l in p.stdout:
if len(l) < 3 or l[0] in (b' ', b'\t'):
continue
if l.endswith(b':\n'):
lib_dirs.append(Path(l[:-2]))
finally:
p.wait()
# Original input files, so upload can restore them
if any(run['input_files'] for run in runs):
logging.info("Packing up original input files...")
inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz')
for run in runs:
for ifile in itervalues(run['input_files']):
inputtar.add(str(join_root(root, PosixPath(ifile))),
str(PosixPath(ifile)))
inputtar.close()
# Meta-data for reprounzip
write_dict(target / '.reprounzip', {}, 'directory')
#.........这里部分代码省略.........
开发者ID:aashish24,项目名称:reprozip,代码行数:101,代码来源:default.py
示例14: chroot_create
def chroot_create(args):
"""Unpacks the experiment in a folder so it can be run with chroot.
All the files in the pack are unpacked; system files are copied only if
they were not packed, and busybox is installed if /bin/sh wasn't packed.
In addition, input files are put in a tar.gz (so they can be put back after
an upload) and the configuration file is extracted.
"""
if not args.pack:
logging.critical("setup/create needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
if DefaultAbstractPath is not PosixPath:
logging.critical("Not unpacking on POSIX system")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# We can only restore owner/group of files if running as root
restore_owner = should_restore_owner(args.restore_owner)
# Unpacks configuration file
tar = tarfile.open(str(pack), 'r:*')
member = tar.getmember('METADATA/config.yml')
member.name = 'config.yml'
tar.extract(member, str(target))
# Loads config
runs, packages, other_files = load_config_file(target / 'config.yml', True)
target.mkdir()
root = (target / 'root').absolute()
root.mkdir()
# Checks that everything was packed
packages_not_packed = [pkg for pkg in packages if not pkg.packfiles]
if packages_not_packed:
record_usage(chroot_missing_pkgs=True)
logging.warning("According to configuration, some files were left out "
"because they belong to the following packages:%s"
"\nWill copy files from HOST SYSTEM",
''.join('\n %s' % pkg
for pkg in packages_not_packed))
missing_files = False
for pkg in packages_not_packed:
for f in pkg.files:
f = Path(f.path)
if not f.exists():
logging.error(
"Missing file %s (from package %s) on host, "
"experiment will probably miss it",
f, pkg.name)
missing_files = True
continue
dest = join_root(root, f)
dest.parent.mkdir(parents=True)
if f.is_link():
dest.symlink(f.read_link())
else:
f.copy(dest)
if restore_owner:
stat = f.stat()
dest.chown(stat.st_uid, stat.st_gid)
if missing_files:
record_usage(chroot_mising_files=True)
# Unpacks files
if any('..' in m.name or m.name.startswith('/') for m in tar.getmembers()):
logging.critical("Tar archive contains invalid pathnames")
sys.exit(1)
members = [m for m in tar.getmembers() if m.name.startswith('DATA/')]
for m in members:
m.name = m.name[5:]
if not restore_owner:
uid = os.getuid()
gid = os.getgid()
for m in members:
m.uid = uid
m.gid = gid
logging.info("Extracting files...")
tar.extractall(str(root), members)
tar.close()
# Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary
sh_path = join_root(root, Path('/bin/sh'))
env_path = join_root(root, Path('/usr/bin/env'))
if not sh_path.lexists() or not env_path.lexists():
logging.info("Setting up busybox...")
busybox_path = join_root(root, Path('/bin/busybox'))
busybox_path.parent.mkdir(parents=True)
with make_dir_writable(join_root(root, Path('/bin'))):
download_file(busybox_url(runs[0]['architecture']),
busybox_path)
#.........这里部分代码省略.........
开发者ID:aashish24,项目名称:reprozip,代码行数:101,代码来源:default.py
示例15: directory_create
def directory_create(args):
"""Unpacks the experiment in a folder.
Only the files that are not part of a package are copied (unless they are
missing from the system and were packed).
In addition, input files are put in a tar.gz (so they can be put back after
an upload) and the configuration file is extracted.
"""
if not args.pack:
logging.critical("setup needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
if not issubclass(DefaultAbstractPath, PosixPath):
logging.critical("Not unpacking on POSIX system")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
config = load_config_file(target / 'config.yml', True)
packages = config.packages
target.mkdir()
root = (target / 'root').absolute()
# Checks packages
missing_files = False
for pkg in packages:
if pkg.packfiles:
continue
for f in pkg.files:
if not Path(f.path).exists():
logging.error(
"Missing file %s (from package %s that wasn't packed) "
"on host, experiment will probably miss it.",
f, pkg.name)
missing_files = True
if missing_files:
record_usage(directory_missing_pkgs=True)
logging.error("Some packages are missing, you should probably install "
"them.\nUse 'reprounzip installpkgs -h' for help")
root.mkdir()
try:
# Unpacks files
members = rpz_pack.list_data()
for m in members:
# Remove 'DATA/' prefix
m.name = str(rpz_pack.remove_data_prefix(m.name))
# Makes symlink targets relative
if m.issym():
linkname = PosixPath(m.linkname)
if linkname.is_absolute:
m.linkname = join_root(root, PosixPath(m.linkname)).path
logging.info("Extracting files...")
rpz_pack.extract_data(root, members)
rpz_pack.close()
# Original input files, so upload can restore them
input_files = [f.path for f in itervalues(config.inputs_outputs)
if f.read_runs]
if input_files:
logging.info("Packing up original input files...")
inputtar = tarfile.open(str(target / 'inputs.tar.gz'), 'w:gz')
for ifile in input_files:
filename = join_root(root, ifile)
if filename.exists():
inputtar.add(str(filename), str(ifile))
inputtar.close()
# Meta-data for reprounzip
metadata_write(target, metadata_initial_iofiles(config), 'directory')
signals.post_setup(target=target, pack=pack)
except Exception:
rmtree_fixed(root)
raise
开发者ID:ReproNim,项目名称:reprozip,代码行数:88,代码来源:default.py
示例16: chroot_create
def chroot_create(args):
"""Unpacks the experiment in a folder so it can be run with chroot.
All the files in the pack are unpacked; system files are copied only if
they were not packed, and busybox is installed if /bin/sh wasn't packed.
In addition, input files are put in a tar.gz (so they can be put back after
an upload) and the configuration file is extracted.
"""
if not args.pack:
logging.critical("setup/create needs the pack filename")
sys.exit(1)
pack = Path(args.pack[0])
target = Path(args.target[0])
if target.exists():
logging.critical("Target directory exists")
sys.exit(1)
if DefaultAbstractPath is not PosixPath:
logging.critical("Not unpacking on POSIX system")
sys.exit(1)
signals.pre_setup(target=target, pack=pack)
# We can only restore owner/group of files if running as root
restore_owner = should_restore_owner(args.restore_owner)
# Unpacks configuration file
rpz_pack = RPZPack(pack)
rpz_pack.extract_config(target / 'config.yml')
# Loads config
config = load_config_file(target / 'config.yml', True)
packages = config.packages
target.mkdir()
root = (target / 'root').absolute()
root.mkdir()
try:
# Checks that everything was packed
packages_not_packed = [pkg for pkg in packages if not pkg.packfiles]
if packages_not_packed:
record_usage(chroot_missing_pkgs=True)
logging.warning("According to configuration, some files were left "
"out because they belong to the following "
"packages:%s\nWill copy files from HOST SYSTEM",
''.join('\n %s' % pkg
for pkg in packages_not_packed))
missing_files = False
for pkg in packages_not_packed:
for f in pkg.files:
path = Path(f.path)
if not path.exists():
logging.error(
"Missing file %s (from package %s) on host, "
"experiment will probably miss it",
path, pkg.name)
missing_files = True
continue
dest = join_root(root, path)
dest.parent.mkdir(parents=True)
if path.is_link():
dest.symlink(path.read_link())
else:
path.copy(dest)
if restore_owner:
stat = path.stat()
dest.chown(stat.st_uid, stat.st_gid)
if missing_files:
record_usage(chroot_mising_files=True)
# Unpacks files
members = rpz_pack.list_data()
for m in members:
# Remove 'DATA/' prefix
m.name = str(rpz_pack.remove_data_prefix(m.name))
if not restore_owner:
uid = os.getuid()
gid = os.getgid()
for m in members:
m.uid = uid
m.gid = gid
logging.info("Extracting files...")
rpz_pack.extract_data(root, members)
rpz_pack.close()
# Sets up /bin/sh and /usr/bin/env, downloading busybox if necessary
sh_path = join_root(root, Path('/bin/sh'))
env_path = join_root(root, Path(
|
请发表评论