def _syslog_to_stderr_path(path):
    """Get the path/uri of the stderr log corresponding to the given syslog.

    If the syslog is gzipped (/path/to/syslog.gz), we'll expect
    stderr to be gzipped too (/path/to/stderr.gz).
    """
    stem, filename = posixpath.split(path)
    return posixpath.join(stem, 'stderr' + file_ext(filename))

 def filter_path(path):
     filename = os.path.basename(path)
     return not(file_ext(filename).lower() in ('.pyc', '.pyo') or
         # filter out emacs backup files
         filename.endswith('~') or
         # filter out emacs lock files
         filename.startswith('.#') or
         # filter out MacFuse resource forks
         filename.startswith('._'))

    def _unarchive_file(self, path, dest):
        path = os.path.abspath(path)
        
        # figure out how to unarchive the file, based on its extension
        unarchive_args = HOW_TO_UNARCHIVE.get(file_ext(path))

        if not unarchive_args:
            raise ValueError("Don't know how to unarchive %s" % path)

        log.debug('unarchiving %s -> %s' % (path, dest))
        self.mkdir(dest)
        check_call(unarchive_args + [path], cwd=dest)

 def test_file_ext(self):
     self.assertEqual(file_ext('foo.zip'), '.zip')
     self.assertEqual(file_ext('foo.Z'), '.Z')
     self.assertEqual(file_ext('foo.tar.gz'), '.tar.gz')
     self.assertEqual(file_ext('README'), '')
     self.assertEqual(file_ext('README,v'), '')
     self.assertEqual(file_ext('README.txt,v'), '.txt,v')

 def test_file_ext(self):
     self.assertEqual(file_ext("foo.zip"), ".zip")
     self.assertEqual(file_ext("foo.Z"), ".Z")
     self.assertEqual(file_ext("foo.tar.gz"), ".tar.gz")
     self.assertEqual(file_ext("README"), "")
     self.assertEqual(file_ext("README,v"), "")
     self.assertEqual(file_ext("README.txt,v"), ".txt,v")

 def filter_path(path):
     filename = os.path.basename(path)
     return not (
         file_ext(filename).lower() in (".pyc", ".pyo")
         or
         # filter out emacs backup files
         filename.endswith("~")
         or
         # filter out emacs lock files
         filename.startswith(".#")
         or
         # filter out MacFuse resource forks
         filename.startswith("._")
     )

    def _master_bootstrap_script_content(self, bootstrap):
        """Return a list containing the lines of the master bootstrap script.
        (without trailing newlines)
        """
        out = []

        # shebang, precommands
        out.extend(self._start_of_sh_script())
        out.append('')

        # store $PWD
        out.append('# store $PWD')
        out.append('__mrjob_PWD=$PWD')
        out.append('')

        # special case for PWD being in /, which happens on Dataproc
        # (really we should cd to tmp or something)
        out.append('if [ $__mrjob_PWD = "/" ]; then')
        out.append('  __mrjob_PWD=""')
        out.append('fi')
        out.append('')

        # run commands in a block so we can redirect stdout to stderr
        # (e.g. to catch errors from compileall). See #370
        out.append('{')

        # download files
        out.append('  # download files and mark them executable')

        cp_to_local = self._cp_to_local_cmd()

        # TODO: why bother with $__mrjob_PWD here, since we're already in it?
        for name, path in sorted(
                self._bootstrap_dir_mgr.name_to_path('file').items()):
            uri = self._upload_mgr.uri(path)
            out.append('')
            out.append('  %s %s $__mrjob_PWD/%s' %
                       (cp_to_local, pipes.quote(uri), pipes.quote(name)))
            # imitate Hadoop Distributed Cache (see #1602)
            out.append('  chmod u+rx $__mrjob_PWD/%s' % pipes.quote(name))
        out.append('')

        # download and unarchive archives
        archive_names_and_paths = sorted(
            self._bootstrap_dir_mgr.name_to_path('archive').items())
        if archive_names_and_paths:
            # make tmp dir if needed
            out.append('  # download and unpack archives')
            out.append('  __mrjob_TMP=$(mktemp -d)')
            out.append('')

            for name, path in archive_names_and_paths:
                uri = self._upload_mgr.uri(path)
                ext = file_ext(basename(path))

                # copy file to tmp dir
                quoted_archive_path = '$__mrjob_TMP/%s' % pipes.quote(name)

                out.append('  %s %s %s' % (
                    cp_to_local, pipes.quote(uri), quoted_archive_path))

                # unarchive file
                if ext not in _EXT_TO_UNARCHIVE_CMD:
                    raise KeyError('unknown archive file extension: %s' % path)
                unarchive_cmd = _EXT_TO_UNARCHIVE_CMD[ext]

                out.append('  ' + unarchive_cmd % dict(
                    file=quoted_archive_path,
                    dir='$__mrjob_PWD/' + pipes.quote(name)))

                # imitate Hadoop Distributed Cache (see #1602)
                out.append(
                    '  chmod u+rx -R $__mrjob_PWD/%s' % pipes.quote(name))

                out.append('')

        # run bootstrap commands
        out.append('  # bootstrap commands')
        for cmd in bootstrap:
            # reconstruct the command line, substituting $__mrjob_PWD/<name>
            # for path dicts
            line = '  '
            for token in cmd:
                if isinstance(token, dict):
                    # it's a path dictionary
                    line += '$__mrjob_PWD/'
                    line += pipes.quote(self._bootstrap_dir_mgr.name(**token))
                else:
                    # it's raw script
                    line += token
            out.append(line)

        out.append('} 1>&2')  # stdout -> stderr for ease of error log parsing

        return out