def run_job(self, args=()):
     args = [sys.executable, MRTwoStepJob.mr_job_script()] + list(args) + ["--no-conf"]
     # add . to PYTHONPATH (in case mrjob isn't actually installed)
     env = combine_envs(os.environ, {"PYTHONPATH": os.path.abspath(".")})
     proc = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env)
     stdout, stderr = proc.communicate(input="foo\nbar\nbar\n")
     return stdout, stderr, proc.returncode

 def test_should_exit_when_invoked_as_script(self):
     args = [sys.executable, MRJob.mr_job_script(), '--quux', 'baz']
     # add . to PYTHONPATH (in case mrjob isn't actually installed)
     env = combine_envs(os.environ, {'PYTHONPATH': os.path.abspath('.')})
     proc = Popen(args, stderr=PIPE, stdout=PIPE, env=env)
     proc.communicate()
     self.assertEqual(proc.returncode, 2)

 def test_should_exit_when_invoked_as_script(self):
     args = [sys.executable, inspect.getsourcefile(MRJobLauncher), "--quux", "baz"]
     # add . to PYTHONPATH (in case mrjob isn't actually installed)
     env = combine_envs(os.environ, {"PYTHONPATH": os.path.abspath(".")})
     proc = Popen(args, stderr=PIPE, stdout=PIPE, env=env)
     proc.communicate()
     self.assertEqual(proc.returncode, 2)

    def test_should_exit_when_invoked_as_script(self):
        args = [sys.executable, inspect.getsourcefile(MRJobLauncher),
                '--quux', 'baz']

        # add . to PYTHONPATH (in case mrjob isn't actually installed)
        env = combine_envs(os.environ,
                           {'PYTHONPATH': mrjob_pythonpath()})
        proc = Popen(args, stderr=PIPE, stdout=PIPE, env=env)
        _, err = proc.communicate()
        self.assertEqual(proc.returncode, 2, err)

 def test_paths(self):
     assert_equal(combine_envs(
         {'PATH': '/bin:/usr/bin',
          'PYTHONPATH': '/usr/lib/python/site-packages',
          'PS1': '> '},
         {'PATH': '/home/dave/bin',
          'PYTHONPATH': '/home/dave/python',
          'CLASSPATH': '/home/dave/java',
          'PS1': '\w> '}),
         {'PATH': '/home/dave/bin:/bin:/usr/bin',
          'PYTHONPATH': '/home/dave/python:/usr/lib/python/site-packages',
          'CLASSPATH': '/home/dave/java',
          'PS1': '\w> '})

 def test_clear_paths(self):
     self.assertEqual(
         combine_envs(
             {'PATH': '/bin:/usr/bin',
              'PYTHONPATH': '/usr/lib/python/site-packages',
              'PS1': '> '},
             {'PATH': ClearedValue('/home/dave/bin'),
              'PYTHONPATH': ClearedValue(None),
              'CLASSPATH': '/home/dave/java',
              'PS1': '\w> '}),
         {'PATH': '/home/dave/bin',
          'CLASSPATH': '/home/dave/java',
          'PS1': '\w> '})

 def test_clear_paths(self):
     self.assertEqual(
         combine_envs(
             {"PATH": "/bin:/usr/bin", "PYTHONPATH": "/usr/lib/python/site-packages", "PS1": "> "},
             {
                 "PATH": ClearedValue("/home/dave/bin"),
                 "PYTHONPATH": ClearedValue(None),
                 "CLASSPATH": "/home/dave/java",
                 "PS1": "\w> ",
             },
         ),
         {"PATH": "/home/dave/bin", "CLASSPATH": "/home/dave/java", "PS1": "\w> "},
     )

 def test_skip_None(self):
     assert_equal(combine_envs(None, {'USER': 'dave'}, None,
                               {'TERM': 'xterm'}, None),
                  {'USER': 'dave', 'TERM': 'xterm'})

 def test_later_values_take_precedence(self):
     assert_equal(
         combine_envs({'TMPDIR': '/tmp', 'HOME': '/home/dave'},
                      {'TMPDIR': '/var/tmp'}),
         {'TMPDIR': '/var/tmp', 'HOME': '/home/dave'})

 def test_empty(self):
     assert_equal(combine_envs(), {})

 def test_empty(self):
     self.assertEqual(combine_envs(), {})

 def test_skip_None(self):
     self.assertEqual(
         combine_envs(None, {"USER": "dave"}, None, {"TERM": "xterm"}, None), {"USER": "dave", "TERM": "xterm"}
     )

 def test_later_values_take_precedence(self):
     self.assertEqual(
         combine_envs({"TMPDIR": "/tmp", "HOME": "/home/dave"}, {"TMPDIR": "/var/tmp"}),
         {"TMPDIR": "/var/tmp", "HOME": "/home/dave"},
     )

    def _invoke_step(self, args, outfile_name, env=None):
        """Run the given command, outputting into outfile, and reading
        from the previous outfile (or, for the first step, from our
        original output files).
        
        outfile is a path relative to our local tmp dir. commands are run
        inside self._working_dir

        We'll intelligently handle stderr from the process.
        """
        # keep the current environment because we need PATH to find binaries
        # and make PYTHONPATH work
        env = combine_envs(
            {'PYTHONPATH': os.getcwd()},
            os.environ,
            self._cmdenv,
            env or {})
        
        # decide where to get input
        if self._prev_outfile is not None:
            input_paths = [self._prev_outfile]
        else:
            input_paths = []
            for path in self._input_paths:
                if path == '-':
                    input_paths.append(self._dump_stdin_to_local_file())
                else:
                    input_paths.append(path)

        # add input to the command line
        for path in input_paths:
            args.append(os.path.abspath(path))

        log.info('> %s' % cmd_line(args))
        
        # set up outfile
        outfile = os.path.join(self._get_local_tmp_dir(), outfile_name)
        log.info('writing to %s' % outfile)
        log.debug('')

        self._prev_outfile = outfile
        write_to = open(outfile, 'w')

        # run the process
        proc = Popen(args, stdout=write_to, stderr=PIPE,
                     cwd=self._working_dir, env=env)

        # handle counters, status msgs, and other stuff on stderr
        stderr_lines = self._process_stderr_from_script(proc.stderr)
        tb_lines = find_python_traceback(stderr_lines)

        self._print_counters()

        returncode = proc.wait()
        if returncode != 0:
            # try to throw a useful exception
            if tb_lines:
                raise Exception(
                    'Command %r returned non-zero exit status %d:\n%s' %
                    (args, returncode, ''.join(tb_lines)))
            else:
                raise Exception(
                    'Command %r returned non-zero exit status %d: %s' %
                    (args, returncode))

        # flush file descriptors
        write_to.flush()

 def _add_python_archive(self, path):
     file_dict = self._add_archive_for_upload(path)
     log.debug('adding %s to PYTHONPATH' % file_dict['name'])
     self._cmdenv = combine_envs(
         self._cmdenv, {'PYTHONPATH': file_dict['name']})
     self._python_archives.append(file_dict)