def test_item_removed(self):
        # Emit item_removed event for an item that is in a playlist
        results = self.lib.items(u'path:{0}'.format(shlex_quote(
            os.path.join(self.music_dir, 'd', 'e', 'f.mp3'))))
        item = results[0]
        beets.plugins.send('item_removed', item=item)

        # Emit item_removed event for an item that is not in a playlist
        results = self.lib.items(u'path:{0}'.format(shlex_quote(
            os.path.join(self.music_dir, 'x', 'y', 'z.mp3'))))
        item = results[0]
        beets.plugins.send('item_removed', item=item)

        # Emit cli_exit event
        beets.plugins.send('cli_exit', lib=self.lib)

        # Check playlist with absolute paths
        playlist_path = os.path.join(self.playlist_dir, 'absolute.m3u')
        with open(playlist_path, 'r') as f:
            lines = [line.strip() for line in f.readlines()]

        self.assertEqual(lines, [
            os.path.join(self.music_dir, 'a', 'b', 'c.mp3'),
            os.path.join(self.music_dir, 'nonexisting.mp3'),
        ])

        # Check playlist with relative paths
        playlist_path = os.path.join(self.playlist_dir, 'relative.m3u')
        with open(playlist_path, 'r') as f:
            lines = [line.strip() for line in f.readlines()]

        self.assertEqual(lines, [
            os.path.join('a', 'b', 'c.mp3'),
            'nonexisting.mp3',
        ])

def build_singularity_run_command(
    container_command,
    image,
    volumes=[],
    env=[],
    working_directory=DEFAULT_WORKING_DIRECTORY,
    singularity_cmd=DEFAULT_SINGULARITY_COMMAND,
    run_extra_arguments=DEFAULT_RUN_EXTRA_ARGUMENTS,
    sudo=DEFAULT_SUDO,
    sudo_cmd=DEFAULT_SUDO_COMMAND,
):
    command_parts = []
    # http://singularity.lbl.gov/docs-environment-metadata
    for (key, value) in env:
        command_parts.extend(["SINGULARITYENV_%s=%s" % (key, value)])
    command_parts += _singularity_prefix(
        singularity_cmd=singularity_cmd,
        sudo=sudo,
        sudo_cmd=sudo_cmd,
    )
    command_parts.append("exec")
    for volume in volumes:
        command_parts.extend(["-B", shlex_quote(str(volume))])
    if working_directory:
        command_parts.extend(["--pwd", shlex_quote(working_directory)])
    if run_extra_arguments:
        command_parts.append(run_extra_arguments)
    full_image = image
    command_parts.append(shlex_quote(full_image))
    command_parts.append(container_command)
    return " ".join(command_parts)

    def startup_command(self, ctx, **kwds):
        """Return a shell command used to startup this instance.

        Among other common planemo kwds, this should respect the
        ``daemon`` keyword.
        """
        daemon = kwds.get("daemon", False)
        # TODO: Allow running dockerized Galaxy here instead.
        setup_venv_command = setup_venv(ctx, kwds)
        run_script = "%s $COMMON_STARTUP_ARGS" % shlex_quote(os.path.join(self.galaxy_root, "run.sh"))
        if daemon:
            run_script += " --daemon"
            self.env["GALAXY_RUN_ALL"] = "1"
        else:
            run_script += " --server-name %s" % shlex_quote(self.server_name)
        server_ini = os.path.join(self.config_directory, "galaxy.ini")
        self.env["GALAXY_CONFIG_FILE"] = server_ini
        if parse_version(kwds.get('galaxy_python_version') or DEFAULT_PYTHON_VERSION) >= parse_version('3'):
            # We need to start under gunicorn
            self.env['APP_WEBSERVER'] = 'gunicorn'
            self.env['GUNICORN_CMD_ARGS'] = "--bind={host}:{port} --name={server_name}".format(
                host=kwds.get('host', '127.0.0.1'),
                port=kwds['port'],
                server_name=self.server_name,
            )
        cd_to_galaxy_command = ['cd', self.galaxy_root]
        return shell_join(
            cd_to_galaxy_command,
            setup_venv_command,
            setup_common_startup_args(),
            run_script,
        )

 def wrap_cmd(session, name, cmd):
     if isinstance(cmd, list):
         cmd = ' '.join(shlex_quote(str(arg)) for arg in cmd)
     if args.mode == 'tmux':
         return 'tmux send-keys -t {}:{} {} Enter'.format(session, name, shlex_quote(cmd))
     elif args.mode == 'child':
         return '{} > {}/{}.{}.out 2>&1 & echo kill $! >> {}/kill.sh'.format(
             cmd, args.logdir, session, name, args.logdir
         )

 def make_test_dockerfile(self, docker_file):
     """Used to determine if we need to rebuild the image"""
     self.setup_lines()
     docker_lines = docker_file.docker_lines + [
         "RUN echo {0}".format(shlex_quote(self.action))
       , "RUN echo {0}".format(" ".join(self.folders))
       , "RUN echo {0}".format(shlex_quote(self.default_cmd))
       ]
     return DockerFile(docker_lines=docker_lines, mtime=docker_file.mtime)

def new_cmd(session, name, cmd, mode, logdir, shell):
    if isinstance(cmd, (list, tuple)):
        cmd = " ".join(shlex_quote(str(v)) for v in cmd)
    if mode == 'tmux':
        return name, "tmux send-keys -t {}:{} {} Enter".format(session, name, shlex_quote(cmd))
    elif mode == 'child':
        return name, "{} >{}/{}.{}.out 2>&1 & echo kill $! >>{}/kill.sh".format(cmd, logdir, session, name, logdir)
    elif mode == 'nohup':
        return name, "nohup {} -c {} >{}/{}.{}.out 2>&1 & echo kill $! >>{}/kill.sh".format(shell, shlex_quote(cmd), logdir, session, name, logdir)

 def make_first_dockerfile(self, docker_file):
     """
     Makes the dockerfile for when we don't already have this image
     It will just perform the action after the normal docker lines.
     """
     self.setup_lines()
     docker_lines = docker_file.docker_lines + [
           "RUN {0} -c {1}".format(shlex_quote(self.shell), shlex_quote(self.action))
         , "CMD {0}".format(self.default_cmd)
         ]
     return DockerFile(docker_lines=docker_lines, mtime=docker_file.mtime)

def process_in_parallel(tag, total_range_size, binary, output_dir):
    """Run the specified binary cfg.NUM_GPUS times in parallel, each time as a
    subprocess that uses one GPU. The binary must accept the command line
    arguments `--range {start} {end}` that specify a data processing range.
    """
    # Snapshot the current cfg state in order to pass to the inference
    # subprocesses
    cfg_file = os.path.join(output_dir, '{}_range_config.yaml'.format(tag))
    with open(cfg_file, 'w') as f:
        yaml.dump(cfg, stream=f)
    subprocess_env = os.environ.copy()
    processes = []
    subinds = np.array_split(range(total_range_size), cfg.NUM_GPUS)
    for i in range(cfg.NUM_GPUS):
        start = subinds[i][0]
        end = subinds[i][-1] + 1
        subprocess_env['CUDA_VISIBLE_DEVICES'] = str(i)
        cmd = '{binary} --range {start} {end} --cfg {cfg_file} NUM_GPUS 1'
        cmd = cmd.format(
            binary=shlex_quote(binary),
            start=int(start),
            end=int(end),
            cfg_file=shlex_quote(cfg_file)
        )
        logger.info('{} range command {}: {}'.format(tag, i, cmd))
        if i == 0:
            subprocess_stdout = subprocess.PIPE
        else:
            filename = os.path.join(
                output_dir, '%s_range_%s_%s.stdout' % (tag, start, end)
            )
            subprocess_stdout = open(filename, 'w')  # NOQA (close below)
        p = subprocess.Popen(
            cmd,
            shell=True,
            env=subprocess_env,
            stdout=subprocess_stdout,
            stderr=subprocess.STDOUT,
            bufsize=1
        )
        processes.append((i, p, start, end, subprocess_stdout))
    # Log output from inference processes and collate their results
    outputs = []
    for i, p, start, end, subprocess_stdout in processes:
        log_subprocess_output(i, p, output_dir, tag, start, end)
        if isinstance(subprocess_stdout, file):  # NOQA (Python 2 for now)
            subprocess_stdout.close()
        range_file = os.path.join(
            output_dir, '%s_range_%s_%s.pkl' % (tag, start, end)
        )
        range_data = pickle.load(open(range_file))
        outputs.append(range_data)
    return outputs

def __add_arg(args, arg, value):
    optarg = '--%s' % arg
    if isinstance(value, bool):
        if value is True:
            args.append(optarg)
    elif isinstance(value, string_types):
        # the = in --optarg=value is usually, but not always, optional
        if value.startswith('='):
            args.append(shlex_quote(optarg + value))
        else:
            args.append(optarg)
            args.append(shlex_quote(value))
    else:
        [__add_arg(args, arg, v) for v in value]

def run_on_control_instead(args, sys_argv):
    argv = [arg for arg in sys_argv][1:]
    argv.remove('--control')
    executable = 'commcare-cloud'
    branch = getattr(args, 'branch', 'master')
    cmd_parts = [
        executable, args.env_name, 'ssh', 'control', '-t',
        'source ~/init-ansible && git fetch --prune && git checkout {branch} '
        '&& git reset --hard origin/{branch} && source ~/init-ansible && {cchq} {cchq_args}'
        .format(branch=branch, cchq=executable, cchq_args=' '.join([shlex_quote(arg) for arg in argv]))
    ]

    print_command(' '.join([shlex_quote(part) for part in cmd_parts]))
    os.execvp(executable, cmd_parts)

def publish_release():
    with in_tmpexport(commit) as td, VirtualEnv.temporary() as ve:
        if opts['dry_run']:
            log.info('Creating source distribution, no upload (dry-run)')
            subprocess.check_output(
                [ve.python, 'setup.py', 'sdist'],
                cwd=td,
            )
        else:
            ve.pip_install(td)

            args = [ve.python, 'setup.py',
                    'sdist',
                    'upload'
                    ]

            if opts['sign']:
                args.append('-s')

                if opts['identity'] is not None:
                    args.extend(['-i', shlex_quote(opts['identity'])])
                    log.info('Uploading signed source distribution to PyPI, '
                             'using key \'{}\''.format(opts['identity']))
                else:
                    log.info('Uploading signed source distribution to PyPI '
                             'using default identity')
            else:
                log.info('Uploading unsigned source distribution to PyPI')

            log.debug('Running {}'.format(args))
            ve.check_output(
                args,
                cwd=td,
            )

 def run(self, args, ssh_args):
     environment = get_environment(args.env_name)
     public_vars = environment.public_vars
     if args.server == '-':
         args.server = 'django_manage:0'
     # the default 'cchq' is redundant with ansible/group_vars/all.yml
     cchq_user = public_vars.get('cchq_user', 'cchq')
     # Name tabs like "droberts (2018-04-13)"
     window_name_expression = '"`whoami` (`date +%Y-%m-%d`)"'
     if args.remote_command:
         ssh_args = [
             '-t',
             r'sudo -iu {cchq_user} tmux attach \; new-window -n {window_name} {remote_command} '
             r'|| sudo -iu {cchq_user} tmux new -n {window_name} {remote_command}'
             .format(
                 cchq_user=cchq_user,
                 remote_command=shlex_quote('{} ; bash'.format(args.remote_command)),
                 window_name=window_name_expression,
             )
         ] + ssh_args
     else:
         ssh_args = [
             '-t',
             'sudo -iu {cchq_user} tmux attach || sudo -iu {cchq_user} tmux new -n {window_name}'
             .format(cchq_user=cchq_user, window_name=window_name_expression)
         ]
     Ssh(self.parser).run(args, ssh_args)

    def _launch_legacy(self, image, env_override, volumes):
        """Legacy launch method for use when the container interface is not enabled
        """
        raw_cmd = self.docker_cmd(image, env_override=env_override, volumes=volumes)

        log.info("Starting docker container for IE {0} with command [{1}]".format(
            self.attr.viz_id,
            ' '.join([shlex_quote(x) for x in raw_cmd])
        ))
        p = Popen(raw_cmd, stdout=PIPE, stderr=PIPE, close_fds=True)
        stdout, stderr = p.communicate()
        if p.returncode != 0:
            log.error("Container Launch error\n\n%s\n%s" % (stdout, stderr))
            return None
        else:
            container_id = stdout.strip()
            log.debug("Container id: %s" % container_id)
            inspect_data = self.inspect_container(container_id)
            port_mappings = self.get_container_port_mapping(inspect_data)
            self.attr.docker_hostname = self.get_container_host(inspect_data)
            host_port = self._find_port_mapping(port_mappings)[-1]
            log.debug("Container host/port: %s:%s", self.attr.docker_hostname, host_port)

            # Now we configure our proxy_requst object and we manually specify
            # the port to map to and ensure the proxy is available.
            self.attr.proxy_request = self.trans.app.proxy_manager.setup_proxy(
                self.trans,
                host=self.attr.docker_hostname,
                port=host_port,
                proxy_prefix=self.attr.proxy_prefix,
                route_name=self.attr.viz_id,
                container_ids=[container_id],
            )
            # These variables then become available for use in templating URLs
            self.attr.proxy_url = self.attr.proxy_request['proxy_url']

    def ansible_playbook(environment, playbook, *cmd_args):
        if os.path.isabs(playbook):
            playbook_path = playbook
        else:
            playbook_path = os.path.join(ANSIBLE_DIR, '{playbook}'.format(playbook=playbook))
        cmd_parts = (
            'ansible-playbook',
            playbook_path,
            '-i', environment.paths.inventory_source,
            '-e', '@{}'.format(environment.paths.vault_yml),
            '-e', '@{}'.format(environment.paths.public_yml),
            '-e', '@{}'.format(environment.paths.generated_yml),
            '--diff',
        ) + get_limit() + cmd_args

        public_vars = environment.public_vars
        cmd_parts += get_user_arg(public_vars, unknown_args, use_factory_auth)

        if has_arg(unknown_args, '-D', '--diff') or has_arg(unknown_args, '-C', '--check'):
            puts(colored.red("Options --diff and --check not allowed. Please remove -D, --diff, -C, --check."))
            puts("These ansible-playbook options are managed automatically by commcare-cloud and cannot be set manually.")
            return 2  # exit code

        ask_vault_pass = public_vars.get('commcare_cloud_use_vault', True)
        if ask_vault_pass:
            cmd_parts += ('--vault-password-file={}/echo_vault_password.sh'.format(ANSIBLE_DIR),)

        cmd_parts_with_common_ssh_args = get_common_ssh_args(environment, use_factory_auth=use_factory_auth)
        cmd_parts += cmd_parts_with_common_ssh_args
        cmd = ' '.join(shlex_quote(arg) for arg in cmd_parts)
        print_command(cmd)
        env_vars = ansible_context.env_vars
        if ask_vault_pass:
            env_vars['ANSIBLE_VAULT_PASSWORD'] = environment.get_ansible_vault_password()
        return subprocess.call(cmd_parts, env=env_vars)

def format_identity_file(ssh_key_file):
    """Render identity file option."""
    if ssh_key_file:
        safe_key_path = shlex_quote(str(ssh_key_file))
        return '-i {}'.format(safe_key_path)
    else:
        return ''

    def _write_load_config_sh(self, environments_dir, quiet):
        puts(colored.blue("Let's get you set up to run commcare-cloud."))

        if not environments_dir:
            environments_dir = self._determine_environments_dir(quiet=quiet)

        commcare_cloud_dir = os.path.expanduser("~/.commcare-cloud")
        if not os.path.exists(commcare_cloud_dir):
            os.makedirs(commcare_cloud_dir)
        load_config_file = os.path.expanduser("~/.commcare-cloud/load_config.sh")
        if not os.path.exists(load_config_file) or \
                ask("Overwrite your ~/.commcare-cloud/load_config.sh?", quiet=quiet):
            with open(load_config_file, 'w') as f:
                f.write(textwrap.dedent("""
                    # auto-generated with `manage-commcare-cloud configure`:
                    export COMMCARE_CLOUD_ENVIRONMENTS={COMMCARE_CLOUD_ENVIRONMENTS}
                    export PATH=$PATH:{virtualenv_path}
                    source {PACKAGE_BASE}/.bash_completion
                """.format(
                    COMMCARE_CLOUD_ENVIRONMENTS=shlex_quote(environments_dir),
                    virtualenv_path=get_virtualenv_bin_path(),
                    PACKAGE_BASE=PACKAGE_BASE,
                )).strip())
        puts(colored.blue("Add the following to your ~/.bash_profile:"))
        puts(colored.cyan("source ~/.commcare-cloud/load_config.sh"))
        puts(colored.blue(
            "and then open a new shell. "
            "You should be able to run `commcare-cloud` without entering your virtualenv."))

    def match(self, name):
        dirpath = dirname(name)

        if not any([len(i) > 0 for i in [self.ext_list, self.regex_list, self.glob_list]]):
            self.glob_list.append("*")  # no filters, return all

        ext_matches = []
        for e in self.ext_list:
            ext_matches.extend(self.glob(dirpath, "*." + e))

        glob_matches = []
        for p in self.glob_list:
            glob_matches.extend(self.glob(dirpath, p))

        regex_matches = []
        if len(self.regex_list) > 0:
            allf = self.glob(dirpath, "*")
            for r in self.regex_list:
                cr = re.compile(r)
                for p in allf:
                    if cr.match(basename(p)) is not None:
                        regex_matches.append(p)

        nodups = list(set(ext_matches + glob_matches + regex_matches))

        prefix = basename(name)
        if prefix != "":
            lf = ListFilter(nodups)
            result = lf.match(prefix)
        else:
            result = nodups

        return [shlex_quote(joinpath(dirpath, n)) for n in result]

    def setup_lines(self):
        """
        Setup convenience lines for copying and waiting for copying
        """
        if getattr(self, "_setup_lines", None):
            return
        self._setup_lines = True

        # Make the shared volume name same as this image name so it doesn't change every time
        shared_name = self["shared_name"] = self.image_name().replace('/', '__')

        # underscored names for our folders
        def without_last_slash(val):
            while val and val.endswith("/"):
                val = val[:-1]
            return val
        folders_underscored = self["folders_underscored"] = [(shlex_quote(name.replace("_", "__").replace("/", "_")), shlex_quote(without_last_slash(name))) for name in self.folders]

        self["move_from_volume"] = " ; ".join(
              "echo {0} && rm -rf {0} && mkdir -p $(dirname {0}) && mv /{1}/{2} {0}".format(name, self.shared_name, underscored)
              for underscored, name in self.folders_underscored
            )

        self["move_into_volume"] = " ; ".join(
              "echo {0} && mkdir -p {0} && mv {0} /{1}/{2}".format(name, self.shared_name, underscored)
              for underscored, name in self.folders_underscored
            )

    def attempt_port_bind(port):
        uwsgi_command = [
            "uwsgi",
            "--http",
            "%s:%s" % (host, port),
            "--pythonpath",
            os.path.join(galaxy_root, "lib"),
            "--yaml",
            yaml_config_path,
            "--module",
            "galaxy.webapps.galaxy.buildapp:uwsgi_app_factory()",
            "--enable-threads",
            "--die-on-term",
        ]

        handle_uwsgi_cli_command = getattr(
            config_object, "handle_uwsgi_cli_command", None
        )
        if handle_uwsgi_cli_command is not None:
            handle_uwsgi_cli_command(uwsgi_command)

        # we don't want to quote every argument but we don't want to print unquoted ones either, so do this
        log.info("Starting uwsgi with command line: %s", ' '.join([shlex_quote(x) for x in uwsgi_command]))
        p = subprocess.Popen(
            uwsgi_command,
            cwd=galaxy_root,
            preexec_fn=os.setsid,
        )
        return UwsgiServerWrapper(
            p, name, host, port
        )

def explore(args):
    if not os.path.exists(args.db):
        print("Unable to find database `{}`".format(args.db))
        return
    conn = sqlite3.connect(args.db)
    create_results(conn)
    call(['sqlite3', '-column', '-header', shlex_quote(args.db)])