def test_stubgen(testcase):
    if 'stubgen-test-path' not in sys.path:
        sys.path.insert(0, 'stubgen-test-path')
    os.mkdir('stubgen-test-path')
    source = '\n'.join(testcase.input)
    handle = tempfile.NamedTemporaryFile(prefix='prog_', suffix='.py', dir='stubgen-test-path')
    assert os.path.isabs(handle.name)
    path = os.path.basename(handle.name)
    name = path[:-3]
    path = os.path.join('stubgen-test-path', path)
    out_dir = '_out'
    os.mkdir(out_dir)
    try:
        with open(path, 'w') as file:
            file.write(source)
            file.close()
            # Without this we may sometimes be unable to import the module below, as importlib
            # caches os.listdir() results in Python 3.3+ (Guido explained this to me).
            reset_importlib_caches()
            try:
                if testcase.name.endswith('_import'):
                    generate_stub_for_module(name, out_dir, quiet=True)
                else:
                    generate_stub(path, out_dir)
                a = load_output(out_dir)
            except CompileError as e:
                a = e.messages
            assert_string_arrays_equal(testcase.output, a,
                                       'Invalid output ({}, line {})'.format(
                                           testcase.file, testcase.line))
    finally:
        shutil.rmtree(out_dir)
        handle.close()

    def run_case(self, testcase: DataDrivenTestCase) -> None:
        """Perform a test case."""
        try:
            # Build test case input.
            src = '\n'.join(testcase.input)
            result = build.build(sources=[BuildSource('main', None, src)],
                                 options=get_semanal_options(),
                                 alt_lib_path=test_temp_dir)
            a = result.errors
            if a:
                raise CompileError(a)

            # Collect all TypeInfos in top-level modules.
            typeinfos = TypeInfoMap()
            for f in result.files.values():
                for n in f.names.values():
                    if isinstance(n.node, TypeInfo):
                        assert n.fullname is not None
                        typeinfos[n.fullname] = n.node

            # The output is the symbol table converted into a string.
            a = str(typeinfos).split('\n')
        except CompileError as e:
            a = e.messages
        assert_string_arrays_equal(
            testcase.output, a,
            'Invalid semantic analyzer output ({}, line {})'.format(
                testcase.file, testcase.line))

    def run_case(self, testcase: DataDrivenTestCase) -> None:
        extra = []
        mods = []
        source = '\n'.join(testcase.input)
        for file, content in testcase.files + [('./main.py', source)]:
            mod = os.path.basename(file)[:-3]
            mods.append(mod)
            extra.extend(['-m', mod])
            with open(file, 'w') as f:
                f.write(content)

        options = self.parse_flags(source, extra)
        out_dir = 'out'
        try:
            try:
                if not testcase.name.endswith('_import'):
                    options.no_import = True
                if not testcase.name.endswith('_semanal'):
                    options.parse_only = True
                generate_stubs(options, quiet=True, add_header=False)
                a = []  # type: List[str]
                self.add_file(os.path.join(out_dir, 'main.pyi'), a)
            except CompileError as e:
                a = e.messages
            assert_string_arrays_equal(testcase.output, a,
                                       'Invalid output ({}, line {})'.format(
                                           testcase.file, testcase.line))
        finally:
            for mod in mods:
                if mod in sys.modules:
                    del sys.modules[mod]
            shutil.rmtree(out_dir)

 def check_module_equivalence(self, name: str,
                              expected: Optional[Set[str]], actual: Set[str]) -> None:
     if expected is not None:
         assert_string_arrays_equal(
             list(sorted(expected)),
             list(sorted(actual.difference({"__main__"}))),
             'Set of {} modules does not match expected set'.format(name))

    def run_test(self, testcase):
        """Perform a test case."""
        try:
            # Build test case input.
            src = '\n'.join(testcase.input)
            result = build.build('main',
                                 target=build.SEMANTIC_ANALYSIS,
                                 program_text=src,
                                 flags=[build.TEST_BUILTINS],
                                 alt_lib_path=test_temp_dir)

            # Collect all TypeInfos in top-level modules.
            typeinfos = TypeInfoMap()
            for f in result.files.values():
                for n in f.names.values():
                    if isinstance(n.node, TypeInfo):
                        typeinfos[n.fullname] = n.node

            # The output is the symbol table converted into a string.
            a = str(typeinfos).split('\n')
        except CompileError as e:
            a = e.messages
        assert_string_arrays_equal(
            testcase.output, a,
            'Invalid semantic analyzer output ({}, line {})'.format(
                testcase.file, testcase.line))

def test_error_stream(testcase: DataDrivenTestCase) -> None:
    """Perform a single error streaming test case.

    The argument contains the description of the test case.
    """
    options = Options()
    options.show_traceback = True

    logged_messages = []  # type: List[str]

    def flush_errors(msgs: List[str], serious: bool) -> None:
        if msgs:
            logged_messages.append('==== Errors flushed ====')
            logged_messages.extend(msgs)

    sources = [BuildSource('main', '__main__', '\n'.join(testcase.input))]
    try:
        build.build(sources=sources,
                    options=options,
                    alt_lib_path=test_temp_dir,
                    flush_errors=flush_errors)
    except CompileError as e:
        assert e.messages == []

    assert_string_arrays_equal(testcase.output, logged_messages,
                               'Invalid output ({}, line {})'.format(
                                   testcase.file, testcase.line))

def test_python_evaluation(testcase: DataDrivenTestCase) -> None:
    # Write the program to a file.
    program = '_program.py'
    program_path = os.path.join(test_temp_dir, program)
    with open(program_path, 'w') as file:
        for s in testcase.input:
            file.write('{}\n'.format(s))
    args = parse_args(testcase.input[0])
    args.append('--tb')  # Show traceback on crash.
    # Type check the program.
    fixed = [python3_path,
             os.path.join(testcase.old_cwd, 'scripts', 'mypy')]
    process = subprocess.Popen(fixed + args,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.STDOUT,
                               cwd=test_temp_dir)
    outb = process.stdout.read()
    # Split output into lines.
    out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
    # Remove temp file.
    os.remove(program_path)
    # Compare actual output to expected.
    assert_string_arrays_equal(testcase.output, out,
                               'Invalid output ({}, line {})'.format(
                                   testcase.file, testcase.line))

def test_stubgen(testcase):
    source = '\n'.join(testcase.input)
    name = 'prog%d' % random.randrange(1000 * 1000 * 1000)
    path = '%s.py' % name
    out_dir = '_out'
    os.mkdir(out_dir)
    try:
        with open(path, 'w') as file:
            file.write(source)
            file.close()
            # Without this we may sometimes be unable to import the module below, as importlib
            # caches os.listdir() results in Python 3.3+ (Guido explained this to me).
            reset_importlib_caches()
            try:
                if testcase.name.endswith('_import'):
                    generate_stub_for_module(name, out_dir, quiet=True)
                else:
                    generate_stub(path, out_dir)
                a = load_output(out_dir)
            except CompileError as e:
                a = e.messages
            assert_string_arrays_equal(testcase.output, a,
                                       'Invalid output ({}, line {})'.format(
                                           testcase.file, testcase.line))
    finally:
        shutil.rmtree(out_dir)
        os.remove(path)

    def run_case(self, testcase: DataDrivenTestCase) -> None:
        first_src = '\n'.join(testcase.input)
        files_dict = dict(testcase.files)
        second_src = files_dict['tmp/next.py']
        options = parse_options(first_src, testcase, 1)

        messages1, files1 = self.build(first_src, options)
        messages2, files2 = self.build(second_src, options)

        a = []
        if messages1:
            a.extend(messages1)
        if messages2:
            a.append('== next ==')
            a.extend(messages2)

        assert files1 is not None and files2 is not None, ('cases where CompileError'
                                                           ' occurred should not be run')
        prefix = '__main__'
        snapshot1 = snapshot_symbol_table(prefix, files1['__main__'].names)
        snapshot2 = snapshot_symbol_table(prefix, files2['__main__'].names)
        diff = compare_symbol_table_snapshots(prefix, snapshot1, snapshot2)
        for trigger in sorted(diff):
            a.append(trigger)

        assert_string_arrays_equal(
            testcase.output, a,
            'Invalid output ({}, line {})'.format(testcase.file,
                                                  testcase.line))

def test_cgen(testcase):
    # Build the program.
    text = '\n'.join(testcase.input)
    program = '_program.py'
    try:
        build.build(program,
                    target=build.C,
                    program_text=text,
                    flags=[build.TEST_BUILTINS],
                    alt_lib_path='lib')
        # Run the program.
        outfile = './_program'
        outb = subprocess.check_output([outfile], stderr=subprocess.STDOUT)
        # Split output into lines.
        out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
        # Remove temp file.
        os.remove(outfile)
    except errors.CompileError as e:
        out = e.messages
    # Include line-end comments in the expected output.
    # Note: # characters in string literals can confuse this.
    for s in testcase.input:
        m = re.search(' #(?! type:)(.*)', s)
        if m:
            testcase.output.append(m.group(1).strip())
    # Verify output.
    assert_string_arrays_equal(testcase.output, out,
                               'Invalid output ({}, line {})'.format(
                                   testcase.file, testcase.line))

def test_transform(testcase):
    """Perform an identity transform test case."""
    
    try:
        src = '\n'.join(testcase.input)
        result = build.build('main',
                             target=build.SEMANTIC_ANALYSIS,
                             program_text=src,
                             pyversion=testfile_pyversion(testcase.file),
                             flags=[build.TEST_BUILTINS],
                             alt_lib_path=test_temp_dir)
        a = []
        # Include string representations of the source files in the actual
        # output.
        for fnam in sorted(result.files.keys()):
            f = result.files[fnam]

            # Omit the builtins module and files with a special marker in the
            # path.
            # TODO the test is not reliable
            if (not f.path.endswith((os.sep + 'builtins.py',
                                     'typing.py',
                                     'abc.py'))
                    and not os.path.basename(f.path).startswith('_')
                    and not os.path.splitext(
                        os.path.basename(f.path))[0].endswith('_')):
                t = TestTransformVisitor()
                f = t.node(f)
                a += str(f).split('\n')
    except CompileError as e:
        a = e.messages
    assert_string_arrays_equal(
        testcase.output, a,
        'Invalid semantic analyzer output ({}, line {})'.format(testcase.file,
                                                                testcase.line))

def test_semanal(testcase):
    """Perform a semantic analysis test case.

    The testcase argument contains a description of the test case
    (inputs and output).
    """

    try:
        src = "\n".join(testcase.input)
        options = get_semanal_options()
        options.python_version = testfile_pyversion(testcase.file)
        result = build.build(sources=[BuildSource("main", None, src)], options=options, alt_lib_path=test_temp_dir)
        a = result.errors
        if a:
            raise CompileError(a)
        # Include string representations of the source files in the actual
        # output.
        for fnam in sorted(result.files.keys()):
            f = result.files[fnam]
            # Omit the builtins module and files with a special marker in the
            # path.
            # TODO the test is not reliable
            if (
                not f.path.endswith(
                    (os.sep + "builtins.pyi", "typing.pyi", "mypy_extensions.pyi", "abc.pyi", "collections.pyi")
                )
                and not os.path.basename(f.path).startswith("_")
                and not os.path.splitext(os.path.basename(f.path))[0].endswith("_")
            ):
                a += str(f).split("\n")
    except CompileError as e:
        a = e.messages
    assert_string_arrays_equal(
        testcase.output, a, "Invalid semantic analyzer output ({}, line {})".format(testcase.file, testcase.line)
    )

def test_python_evaluation(testcase):
    # Write the program to a file.
    program = "_program.py"
    outfile = "_program.out"
    f = open(program, "w")
    for s in testcase.input:
        f.write("{}\n".format(s))
    f.close()
    # Use Python 2 interpreter if running a Python 2 test case.
    if testcase.name.lower().endswith("python2"):
        args = ["--py2", python2_path]
    else:
        args = []
    # Set up module path.
    typing_path = os.path.join(os.getcwd(), "lib-typing", "3.2")
    assert os.path.isdir(typing_path)
    os.environ["PYTHONPATH"] = os.pathsep.join([typing_path, "."])
    os.environ["MYPYPATH"] = "."
    # Run the program.
    outb = subprocess.check_output([python3_path, os.path.join("scripts", "mypy")] + args + [program])
    # Split output into lines.
    out = [s.rstrip("\n\r") for s in str(outb, "utf8").splitlines()]
    # Remove temp file.
    os.remove(program)
    assert_string_arrays_equal(
        testcase.output, out, "Invalid output ({}, line {})".format(testcase.file, testcase.line)
    )

def test_python_evaluation(testcase: DataDrivenTestCase, cache_dir: str) -> None:
    """Runs Mypy in a subprocess.

    If this passes without errors, executes the script again with a given Python
    version.
    """
    assert testcase.old_cwd is not None, "test was not properly set up"
    # TODO: Enable strict optional for these tests
    mypy_cmdline = [
        '--show-traceback',
        '--no-site-packages',
        '--no-strict-optional',
        '--no-silence-site-packages',
    ]
    if testcase.name.lower().endswith('_newsemanal'):
        mypy_cmdline.append('--new-semantic-analyzer')
    py2 = testcase.name.lower().endswith('python2')
    if py2:
        mypy_cmdline.append('--py2')
        interpreter = try_find_python2_interpreter()
        if interpreter is None:
            # Skip, can't find a Python 2 interpreter.
            pytest.skip()
            # placate the type checker
            return
    else:
        interpreter = python3_path
        mypy_cmdline.append('--python-version={}'.format('.'.join(map(str, PYTHON3_VERSION))))

    # Write the program to a file.
    program = '_' + testcase.name + '.py'
    program_path = os.path.join(test_temp_dir, program)
    mypy_cmdline.append(program_path)
    with open(program_path, 'w', encoding='utf8') as file:
        for s in testcase.input:
            file.write('{}\n'.format(s))
    mypy_cmdline.append('--cache-dir={}'.format(cache_dir))
    output = []
    # Type check the program.
    out, err, returncode = api.run(mypy_cmdline)
    # split lines, remove newlines, and remove directory of test case
    for line in (out + err).splitlines():
        if line.startswith(test_temp_dir + os.sep):
            output.append(line[len(test_temp_dir + os.sep):].rstrip("\r\n"))
        else:
            output.append(line.rstrip("\r\n"))
    if returncode == 0:
        # Execute the program.
        returncode, interp_out = run_command([interpreter, program])
        output.extend(interp_out)
    # Remove temp file.
    os.remove(program_path)
    for i, line in enumerate(output):
        if os.path.sep + 'typeshed' + os.path.sep in line:
            output[i] = line.split(os.path.sep)[-1]
    assert_string_arrays_equal(adapt_output(testcase), output,
                               'Invalid output ({}, line {})'.format(
                                   testcase.file, testcase.line))

def test_parse_error(testcase):
    try:
        # Compile temporary file.
        parse(bytes("\n".join(testcase.input), "ascii"), INPUT_FILE_NAME)
        raise AssertionFailure("No errors reported")
    except CompileError as e:
        # Verify that there was a compile error and that the error messages
        # are equivalent.
        assert_string_arrays_equal(
            testcase.output, e.messages, "Invalid compiler output ({}, line {})".format(testcase.file, testcase.line)
        )

def test_python_evaluation(testcase):
    python2_interpreter = try_find_python2_interpreter()
    # Use Python 2 interpreter if running a Python 2 test case.
    if testcase.name.lower().endswith('python2'):
        if not python2_interpreter:
            # Skip, can't find a Python 2 interpreter.
            raise SkipTestCaseException()
        interpreter = python2_interpreter
        args = ['--py2']
        py2 = True
    else:
        interpreter = python3_path
        args = []
        py2 = False
    args.append('--show-traceback')
    # Write the program to a file.
    program = '_program.py'
    program_path = os.path.join(test_temp_dir, program)
    with open(program_path, 'w') as file:
        for s in testcase.input:
            file.write('{}\n'.format(s))
    # Type check the program.
    # This uses the same PYTHONPATH as the current process.
    process = subprocess.Popen([python3_path,
                                os.path.join(testcase.old_cwd, 'scripts', 'mypy')]
                            + args + [program],
                               stdout=subprocess.PIPE,
                               stderr=subprocess.STDOUT,
                               cwd=test_temp_dir)
    outb = process.stdout.read()
    # Split output into lines.
    out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
    if not process.wait():
        # Set up module path for the execution.
        # This needs the typing module but *not* the mypy module.
        vers_dir = '2.7' if py2 else '3.2'
        typing_path = os.path.join(testcase.old_cwd, 'lib-typing', vers_dir)
        assert os.path.isdir(typing_path)
        env = os.environ.copy()
        env['PYTHONPATH'] = typing_path
        process = subprocess.Popen([interpreter, program],
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.STDOUT,
                                   cwd=test_temp_dir,
                                   env=env)
        outb = process.stdout.read()
        # Split output into lines.
        out += [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
    # Remove temp file.
    os.remove(program_path)
    assert_string_arrays_equal(testcase.output, out,
                               'Invalid output ({}, line {})'.format(
                                   testcase.file, testcase.line))

def test_parse_error(testcase):
    try:
        # Compile temporary file. The test file contains non-ASCII characters.
        parse(bytes('\n'.join(testcase.input), 'utf-8'), INPUT_FILE_NAME, None, Options())
        raise AssertionFailure('No errors reported')
    except CompileError as e:
        # Verify that there was a compile error and that the error messages
        # are equivalent.
        assert_string_arrays_equal(
            testcase.output, e.messages,
            'Invalid compiler output ({}, line {})'.format(testcase.file,
                                                           testcase.line))

    def run_case_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
        find_module_clear_caches()
        program_text = '\n'.join(testcase.input)
        module_name, program_name, program_text = self.parse_module(program_text)

        options = self.parse_options(program_text)
        options.use_builtins_fixtures = True
        options.python_version = testcase_pyversion(testcase.file, testcase.name)

        output = testcase.output
        if incremental:
            options.incremental = True
            if incremental == 1:
                # In run 1, copy program text to program file.
                output = []
                with open(program_name, 'w') as f:
                    f.write(program_text)
            elif incremental == 2:
                # In run 2, copy *.py.next files to *.py files.
                for dn, dirs, files in os.walk(os.curdir):
                    for file in files:
                        if file.endswith('.py.next'):
                            full = os.path.join(dn, file)
                            target = full[:-5]
                            shutil.copy(full, target)
            # Always set to none so we're forced to reread program_name
            program_text = None
        source = BuildSource(program_name, module_name, program_text)
        try:
            res = build.build(sources=[source],
                              options=options,
                              alt_lib_path=test_temp_dir)
            a = res.errors
        except CompileError as e:
            res = None
            a = e.messages
        a = normalize_error_messages(a)

        if output != a and self.update_data:
            update_testcase_output(testcase, a)

        assert_string_arrays_equal(
            output, a,
            'Invalid type checker output ({}, line {})'.format(
                testcase.file, testcase.line))

        if incremental and res:
            self.verify_cache(module_name, program_name, a, res.manager)
            if testcase.expected_stale_modules is not None and incremental == 2:
                assert_string_arrays_equal(
                    list(sorted(testcase.expected_stale_modules)),
                    list(sorted(res.manager.stale_modules.difference({"__main__"}))),
                    'Set of stale modules does not match expected set')

 def check_module_equivalence(self, name: str,
                              expected: Optional[Set[str]], actual: Set[str]) -> None:
     if expected is not None:
         expected_normalized = sorted(expected)
         actual_normalized = sorted(actual.difference({"__main__"}))
         assert_string_arrays_equal(
             expected_normalized,
             actual_normalized,
             ('Actual modules ({}) do not match expected modules ({}) '
              'for "[{} ...]"').format(
                 ', '.join(actual_normalized),
                 ', '.join(expected_normalized),
                 name))

    def run_case(self, testcase: DataDrivenTestCase) -> None:
        try:
            line = testcase.input[0]
            mask = ''
            if line.startswith('##'):
                mask = '(' + line[2:].strip() + ')$'

            src = '\n'.join(testcase.input)
            options = Options()
            options.strict_optional = False  # TODO: Enable strict optional checking
            options.use_builtins_fixtures = True
            options.show_traceback = True
            options.export_types = True
            result = build.build(sources=[BuildSource('main', None, src)],
                                 options=options,
                                 alt_lib_path=test_temp_dir)
            a = result.errors
            map = result.types
            nodes = map.keys()

            # Ignore NameExpr nodes of variables with explicit (trivial) types
            # to simplify output.
            searcher = SkippedNodeSearcher()
            for file in result.files.values():
                file.accept(searcher)
            ignored = searcher.nodes

            # Filter nodes that should be included in the output.
            keys = []
            for node in nodes:
                if node.line is not None and node.line != -1 and map[node]:
                    if ignore_node(node) or node in ignored:
                        continue
                    if (re.match(mask, short_type(node))
                            or (isinstance(node, NameExpr)
                                and re.match(mask, node.name))):
                        # Include node in output.
                        keys.append(node)

            for key in sorted(keys,
                              key=lambda n: (n.line, short_type(n),
                                             str(n) + str(map[n]))):
                ts = str(map[key]).replace('*', '')  # Remove erased tags
                ts = ts.replace('__main__.', '')
                a.append('{}({}) : {}'.format(short_type(key), key.line, ts))
        except CompileError as e:
            a = e.messages
        assert_string_arrays_equal(
            testcase.output, a,
            'Invalid type checker output ({}, line {})'.format(testcase.file,
                                                               testcase.line))