def test_find_python_traceback(self):
        def run(*args):
            return Popen(args, stdout=PIPE, stderr=PIPE).communicate()

        # sanity-check normal operations
        ok_stdout, ok_stderr = run('python', '-c', "print sorted('321')")
        self.assertEqual(ok_stdout.rstrip(), "['1', '2', '3']")
        self.assertEqual(find_python_traceback(StringIO(ok_stderr)), None)

        # Oops, can't sort a number.
        stdout, stderr = run('python', '-c', "print sorted(321)")

        # We expect something like this:
        #
         # Traceback (most recent call last):
        #   File "<string>", line 1, in <module>
        # TypeError: 'int' object is not iterable
        self.assertEqual(stdout, '')
        # save the traceback for the next step
        tb = find_python_traceback(StringIO(stderr))
        self.assertNotEqual(tb, None)
        assert isinstance(tb, list)
        # The first line ("Traceback...") is not skipped
        self.assertEqual(len(tb), 3)

        # make sure we can find the same traceback in noise
        verbose_stdout, verbose_stderr = run(
            'python', '-v', '-c', "print sorted(321)")
        self.assertEqual(verbose_stdout, '')
        self.assertNotEqual(verbose_stderr, stderr)
        verbose_tb = find_python_traceback(StringIO(verbose_stderr))
        self.assertEqual(verbose_tb, tb)

    def _wait_for_process(self, proc_dict, step_num):
        # handle counters, status msgs, and other stuff on stderr
        proc = proc_dict['proc']

        stderr_lines = self._process_stderr_from_script(
            proc.stderr, step_num=step_num)
        tb_lines = find_python_traceback(stderr_lines)

        # proc.stdout isn't always defined
        if proc.stdout:
            proc.stdout.close()
        proc.stderr.close()

        returncode = proc.wait()

        if returncode != 0:
            # show counters before raising exception
            counters = self._counters[step_num]
            if counters:
                log.info(_format_counters(counters))

            # try to throw a useful exception
            if tb_lines:
                for line in tb_lines:
                    log.error(line.rstrip('\r\n'))

            reason = str(
                CalledProcessError(returncode, proc_dict['args']))
            raise StepFailedException(
                reason=reason, step_num=step_num,
                num_steps=len(self._get_steps()))

    def _wait_for_process(self, proc_dict, step_num):
        # handle counters, status msgs, and other stuff on stderr
        proc = proc_dict['proc']

        stderr_lines = self._process_stderr_from_script(
            proc.stderr, step_num=step_num)
        tb_lines = find_python_traceback(stderr_lines)

        # proc.stdout isn't always defined
        if proc.stdout:
            proc.stdout.close()
        proc.stderr.close()

        returncode = proc.wait()

        if returncode != 0:
            self.print_counters([step_num + 1])
            # try to throw a useful exception
            if tb_lines:
                raise Exception(
                    'Command %r returned non-zero exit status %d:\n%s' %
                    (proc_dict['args'], returncode, ''.join(tb_lines)))
            else:
                raise Exception(
                    'Command %r returned non-zero exit status %d' %
                    (proc_dict['args'], returncode))

    def _invoke_step(self, args, outfile_name, env=None):
        """Run the given command, outputting into outfile, and reading
        from the previous outfile (or, for the first step, from our
        original output files).

        outfile is a path relative to our local tmp dir. commands are run
        inside self._working_dir

        We'll intelligently handle stderr from the process.
        """
        # keep the current environment because we need PATH to find binaries
        # and make PYTHONPATH work
        env = combine_local_envs(
            {'PYTHONPATH': os.getcwd()},
            os.environ,
            self._get_cmdenv(),
            env or {})

        # decide where to get input
        if self._prev_outfile is not None:
            input_paths = [self._prev_outfile]
        else:
            input_paths = []
            for path in self._input_paths:
                if path == '-':
                    input_paths.append(self._dump_stdin_to_local_file())
                else:
                    input_paths.append(path)

        # add input to the command line
        for path in input_paths:
            args.append(os.path.abspath(path))

        log.info('> %s' % cmd_line(args))

        # set up outfile
        outfile = os.path.join(self._get_local_tmp_dir(), outfile_name)
        log.info('writing to %s' % outfile)
        log.debug('')

        self._prev_outfile = outfile
        write_to = open(outfile, 'w')

        # run the process
        proc = Popen(args, stdout=write_to, stderr=PIPE,
                     cwd=self._working_dir, env=env)

        # handle counters, status msgs, and other stuff on stderr
        stderr_lines = self._process_stderr_from_script(proc.stderr)
        tb_lines = find_python_traceback(stderr_lines)

        self._print_counters()

        returncode = proc.wait()
        if returncode != 0:
            # try to throw a useful exception
            if tb_lines:
                raise Exception(
                    'Command %r returned non-zero exit status %d:\n%s' %
                    (args, returncode, ''.join(tb_lines)))
            else:
                raise Exception(
                    'Command %r returned non-zero exit status %d: %s' %
                    (args, returncode))

        # flush file descriptors
        write_to.flush()

 def test_find_python_traceback_with_more_stderr_2(self):
     total_traceback = self.EXAMPLE_STDERR_TRACEBACK_2 + 'junk\n'
     tb = find_python_traceback(StringIO(total_traceback))
     self.assertEqual(''.join(tb), self.EXAMPLE_STDERR_TRACEBACK_2)

 def test_find_multiple_python_tracebacks(self):
     total_traceback = self.EXAMPLE_TRACEBACK + 'junk\n'
     tb = find_python_traceback(StringIO(total_traceback))
     self.assertEqual(''.join(tb), self.EXAMPLE_TRACEBACK)

 def test_find_python_traceback_with_more_stderr_2(self):
     total_traceback = self.EXAMPLE_STDERR_TRACEBACK_2 + b'junk\n'
     tb = find_python_traceback(BytesIO(total_traceback))
     self.assertEqual(''.join(tb),
                      self.EXAMPLE_STDERR_TRACEBACK_2.decode('ascii'))

 def test_find_multiple_python_tracebacks(self):
     total_traceback = self.EXAMPLE_TRACEBACK + b'junk\n'
     tb = find_python_traceback(BytesIO(total_traceback))
     self.assertEqual(''.join(tb),
                      self.EXAMPLE_TRACEBACK.decode('ascii'))

 def parse(self, lines):
     return find_python_traceback(lines)