for fnam in sorted(result.files.keys()):
            f = result.files[fnam]
            # Omit the builtins module and files marked for omission.
            if not f.path.endswith(os.sep +
                                   'builtins.py') and '-skip.' not in f.path:
                # Add file name + colon for files other than the first.
                if not first:
                    a.append('{}:'.format(
                        fix_path(remove_prefix(f.path, test_temp_dir))))

                ver = 3
                # Generate Python 2 instead of 3?
                if '-2' in testcase.name:
                    ver = 2
                v = PythonGenerator(ver)
                f.accept(v)
                s = v.output()
                if s != '':
                    a += s.split('\n')
            first = False
    except CompileError as e:
        a = e.messages
    assert_string_arrays_equal(
        expected, a, 'Invalid source code output ({}, line {})'.format(
            testcase.file, testcase.line))


if __name__ == '__main__':
    import sys
    run_test(PythonGenerationSuite(), sys.argv[1:])

            for lvalue in s.lvalues:
                if isinstance(lvalue, NameExpr):
                    self.nodes.add(lvalue)
            if (isinstance(s.rvalue, NameExpr)
                    and s.rvalue.fullname == 'typing.Undefined'):
                self.nodes.add(s.rvalue)


def ignore_node(node):
    """Return True if node is to be omitted from test case output."""

    # We want to get rid of object() expressions in the typing module stub
    # and also TypeVar(...) expressions. Since detecting whether a node comes
    # from the typing module is not easy, we just to strip them all away.
    if isinstance(node, TypeVarExpr):
        return True
    if isinstance(node, NameExpr) and node.fullname == 'builtins.object':
        return True
    if isinstance(node, NameExpr) and node.fullname == 'builtins.None':
        return True
    if isinstance(node, CallExpr) and (ignore_node(node.callee) or
                                       node.analyzed):
        return True

    return False


if __name__ == '__main__':
    import sys
    run_test(TypeExportSuite(), sys.argv[1:])

        process = subprocess.Popen([interpreter, program],
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.STDOUT,
                                   cwd=test_temp_dir,
                                   env=env)
        outb = process.stdout.read()
        # Split output into lines.
        out += [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
    # Remove temp file.
    os.remove(program_path)
    assert_string_arrays_equal(testcase.output, out,
                               'Invalid output ({}, line {})'.format(
                                   testcase.file, testcase.line))


def try_find_python2_interpreter():
    for interpreter in default_python2_interpreter:
        try:
            process = subprocess.Popen([interpreter, '-V'], stdout=subprocess.PIPE,
                                       stderr=subprocess.STDOUT)
            stdout, stderr = process.communicate()
            if b'Python 2.7' in stdout:
                return interpreter
        except OSError:
            pass
    return None


if __name__ == '__main__':
    run_test(PythonEvaluationSuite(), sys.argv[1:])

            testcase.output, a,
            'Invalid semantic analyzer output ({}, line {})'.format(
                testcase.file, testcase.line))


class TypeInfoMap(Dict[str, TypeInfo]):
    def __str__(self) -> str:
        a = ['TypeInfoMap(']  # type: List[str]
        for x, y in sorted(self.items()):
            if isinstance(x, str) and (not x.startswith('builtins.') and
                                       not x.startswith('typing.') and
                                       not x.startswith('abc.')):
                ti = ('\n' + '  ').join(str(y).split('\n'))
                a.append('  {} : {}'.format(x, ti))
        a[-1] += ')'
        return '\n'.join(a)


class CombinedSemAnalSuite(Suite):
    def __init__(self):
        self.test_semanal = SemAnalSuite()
        self.test_semanal_errors = SemAnalErrorSuite()
        self.test_semanal_symtable = SemAnalSymtableSuite()
        self.test_semanal_typeinfos = SemAnalTypeInfoSuite()
        super().__init__()


if __name__ == '__main__':
    import sys
    run_test(CombinedSemAnalSuite(), sys.argv[1:])

from mypy.test import teststubgen
from mypy.test import testdocstring


class AllSuite(Suite):
    def __init__(self):
        self.test_types = testtypes.TypesSuite()
        self.test_typeops = testtypes.TypeOpsSuite()
        self.test_join = testtypes.JoinSuite()
        self.test_meet = testtypes.MeetSuite()
        self.test_subtypes = testsubtypes.SubtypingSuite()
        self.test_solve = testsolve.SolveSuite()
        self.test_infer = testinfer.MapActualsToFormalsSuite()
        self.test_lex = testlex.LexerSuite()
        self.test_parse = testparse.ParserSuite()
        self.test_parse_errors = testparse.ParseErrorSuite()
        self.test_semanal = testsemanal.SemAnalSuite()
        self.test_semanal_errors = testsemanal.SemAnalErrorSuite()
        self.test_semanal_symtable = testsemanal.SemAnalSymtableSuite()
        self.test_semanal_typeinfos = testsemanal.SemAnalTypeInfoSuite()
        self.test_transform = testtransform.TransformSuite()
        self.test_check = testcheck.TypeCheckSuite()
        self.test_typegen = testtypegen.TypeExportSuite()
        self.test_stubgen = teststubgen.StubgenSuite()
        self.test_docstring = testdocstring.DocstringSuite()
        super().__init__()


if __name__ == '__main__':
    run_test(AllSuite(), sys.argv[1:])

            # Omit the builtins module and files with a special marker in the
            # path.
            # TODO the test is not reliable
            if (not f.path.endswith((os.sep + 'builtins.py',
                                     'typing.py',
                                     'abc.py'))
                    and not os.path.basename(f.path).startswith('_')
                    and not os.path.splitext(
                        os.path.basename(f.path))[0].endswith('_')):
                t = TestTransformVisitor()
                f = t.node(f)
                a += str(f).split('\n')
    except CompileError as e:
        a = e.messages
    assert_string_arrays_equal(
        testcase.output, a,
        'Invalid semantic analyzer output ({}, line {})'.format(testcase.file,
                                                                testcase.line))


class TestTransformVisitor(TransformVisitor):
    def type(self, type):
        assert type is not None
        return type


if __name__ == '__main__':
    import sys
    run_test(TransformSuite(), sys.argv[1:])

    def cases(self):
        # Test case descriptions are in an external file.
        return parse_test_cases(os.path.join(testconfig.test_data_prefix,
                                             'parse-errors.test'),
                                test_parse_error)


def test_parse_error(testcase):
    try:
        # Compile temporary file.
        parse('\n'.join(testcase.input), INPUT_FILE_NAME)
        raise AssertionFailure('No errors reported')
    except CompileError as e:
        # Verify that there was a compile error and that the error messages
        # are equivalent.
        assert_string_arrays_equal(
            testcase.output, e.messages,
            'Invalid compiler output ({}, line {})'.format(testcase.file,
                                                           testcase.line))


class CombinedParserSuite(Suite):
    def __init__(self):
        self.test_parse = ParserSuite()
        self.test_parse_errors = ParseErrorSuite()
        super().__init__()


if __name__ == '__main__':
    run_test(CombinedParserSuite(), sys.argv[1:])

    #  * more generic interface subtyping test cases
    #  * type variables
    #  * tuple types
    #  * void type
    #  * None type
    #  * any type
    #  * generic function types

    def assert_subtype(self, s, t):
        assert_true(is_subtype(s, t), '{} not subtype of {}'.format(s, t))

    def assert_not_subtype(self, s, t):
        assert_true(not is_subtype(s, t), '{} subtype of {}'.format(s, t))

    def assert_strict_subtype(self, s, t):
        self.assert_subtype(s, t)
        self.assert_not_subtype(t, s)

    def assert_equivalent(self, s, t):
        self.assert_subtype(s, t)
        self.assert_subtype(t, s)

    def assert_unrelated(self, s, t):
        self.assert_not_subtype(s, t)
        self.assert_not_subtype(t, s)


if __name__ == '__main__':
    import sys
    run_test(SubtypingSuite(), sys.argv[1:])

    def test_infer_getitem_sig(self):
        assert_equal(infer_method_sig('__getitem__'), '(index)')

    def test_infer_setitem_sig(self):
        assert_equal(infer_method_sig('__setitem__'), '(index, object)')

    def test_infer_binary_op_sig(self):
        for op in ('eq', 'ne', 'lt', 'le', 'gt', 'ge',
                   'add', 'radd', 'sub', 'rsub', 'mul', 'rmul'):
            assert_equal(infer_method_sig('__%s__' % op), '(other)')

    def test_infer_unary_op_sig(self):
        for op in ('neg', 'pos'):
            assert_equal(infer_method_sig('__%s__' % op), '()')


class StubgenSuite(Suite):
    """Collect all the test classes defined in this file."""

    def __init__(self):
        self.test_python = StubgenPythonSuite()
        self.test_c = StubgencSuite()
        self.test_util = StubgenUtilSuite()
        super().__init__()


if __name__ == '__main__':
    import sys
    run_test(StubgenSuite(), sys.argv[1:])

                        f.path, test_temp_dir))))
                
                v = OutputVisitor()
                f.accept(v)
                s = v.output()
                if s != '':
                    a += s.split('\n')
            first = False
    except CompileError as e:
        a = e.messages
    assert_string_arrays_equal(
        expected, a, 'Invalid source code output ({}, line {})'.format(
            testcase.file, testcase.line))


def remove_prefix(path, prefix):
    regexp = '^' + prefix.replace('\\', '\\\\')
    np = re.sub(regexp, '', path)
    if np.startswith(os.sep):
        np = np[1:]
    return np


def fix_path(path):
    return path.replace('\\', '/')


if __name__ == '__main__':
    import sys
    run_test(OutputSuite(), sys.argv[1:])

        # Run the program.
        outfile = './_program'
        outb = subprocess.check_output([outfile], stderr=subprocess.STDOUT)
        # Split output into lines.
        out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
        # Remove temp file.
        os.remove(outfile)
    except errors.CompileError as e:
        out = e.messages
    # Include line-end comments in the expected output.
    # Note: # characters in string literals can confuse this.
    for s in testcase.input:
        m = re.search(' #(?! type:)(.*)', s)
        if m:
            testcase.output.append(m.group(1).strip())
    # Verify output.
    assert_string_arrays_equal(testcase.output, out,
                               'Invalid output ({}, line {})'.format(
                                   testcase.file, testcase.line))


class CGenSuite(Suite):
    def __init__(self):
        self.test_compile = CGenCompileSuite()
        self.test_run = CGenRunSuite()
        super().__init__()


if __name__ == '__main__':
    run_test(CGenSuite(), sys.argv[1:])

def expand_caller_kinds(kinds_or_names):
    kinds = []
    names = []
    for k in kinds_or_names:
        if isinstance(k, str):
            kinds.append(ARG_NAMED)
            names.append(k)
        else:
            kinds.append(k)
            names.append(None)
    return kinds, names


def expand_callee_kinds(kinds_and_names):
    kinds = []
    names = []
    for v in kinds_and_names:
        if isinstance(v, tuple):
            kinds.append(v[0])
            names.append(v[1])
        else:
            kinds.append(v)
            names.append(None)
    return kinds, names


if __name__ == '__main__':
    import sys
    run_test(MapActualsToFormalsSuite(), sys.argv[1:])

def builtins_wrapper(func, path):
    """Decorate a function that implements a data-driven test case to copy an
    alternative builtins module implementation in place before performing the
    test case. Clean up after executing the test case.
    """
    return lambda testcase: perform_test(func, path, testcase)


def perform_test(func, path, testcase):
    for path, _ in testcase.files:
        if os.path.basename(path) == 'builtins.py':
            default_builtins = False
            break
    else:
        # Use default builtins.
        builtins = os.path.join(test_temp_dir, 'builtins.py')
        shutil.copyfile(path, builtins)
        default_builtins = True

    # Actually peform the test case.
    func(testcase)
    
    if default_builtins:
        # Clean up.
        os.remove(builtins)


if __name__ == '__main__':
    import sys
    run_test(DyncheckTransformSuite(), sys.argv[1:])

        self.assert_solve(['T'],
                          [self.supc(self.fx.t, self.fx.a),
                           self.subc(self.fx.t, self.fx.anyt)],
                          [(self.fx.anyt, self.fx.anyt)])

        self.assert_solve(['T'],
                          [self.supc(self.fx.t, self.fx.anyt),
                           self.subc(self.fx.t, self.fx.a)],
                          [(self.fx.anyt, self.fx.anyt)])

    def assert_solve(self, vars, constraints, results):
        res = []
        for r in results:
            if isinstance(r, tuple):
                res.append(r[0])
            else:
                res.append(r)
        actual = solve_constraints(vars, constraints)
        assert_equal(str(actual), str(res))

    def supc(self, type_var, bound):
        return Constraint(type_var.name, SUPERTYPE_OF, bound)

    def subc(self, type_var, bound):
        return Constraint(type_var.name, SUBTYPE_OF, bound)


if __name__ == '__main__':
    import sys
    run_test(SolveSuite(), sys.argv[1:])

            a.append('def {}:'.format(fn))
            try:
                funccode = result.icode[fn]
            except KeyError:
                raise RuntimeError('no icode for %s (%s)' % (
                    fn, list(result.icode.keys())))
            code = icode.render(funccode)
            a.extend(code)
    except CompileError as e:
        a = e.messages
    assert_string_arrays_equal_wildcards(
        expected, a,
        'Invalid source code output ({}, line {})'.format(testcase.file,
                                                          testcase.line))


def get_func_names(expected):
    res = []
    for s in expected:
        m = re.match(r'def ([_a-zA-Z0-9.*$]+):', s)
        if m:
            res.append(m.group(1))
    if not res:
        raise RuntimeError('No function name in test case output')
    return res


if __name__ == '__main__':
    import sys
    run_test(IcodeGenerationSuite(), sys.argv[1:])

            c += parse_test_cases(os.path.join(test_data_prefix, f),
                                  self.run_test, test_temp_dir, True)
        return c

    def run_test(self, testcase):
        a = []
        pyversion = testcase_pyversion(testcase.file, testcase.name)
        try:
            src = '\n'.join(testcase.input)
            build.build('main',
                        target=build.TYPE_CHECK,
                        program_text=src,
                        pyversion=pyversion,
                        flags=[build.TEST_BUILTINS],
                        alt_lib_path=test_temp_dir)
        except CompileError as e:
            a = normalize_error_messages(e.messages)

        if testcase.output != a and UPDATE_TESTCASES:
            update_testcase_output(testcase, a, APPEND_TESTCASES)

        assert_string_arrays_equal(
            testcase.output, a,
            'Invalid type checker output ({}, line {})'.format(
                testcase.file, testcase.line))


if __name__ == '__main__':
    import sys
    run_test(TypeCheckSuite(), sys.argv[1:])

                        '{} not subtype of {}'.format(result, s))
        if not isinstance(t, ErrorType) and not isinstance(result, ErrorType):
            assert_true(is_subtype(result, t),
                        '{} not subtype of {}'.format(result, t))
    
    def tuple(self, *a):
        return TupleType(a)
    
    def callable(self, *a):
        """callable(a1, ..., an, r) constructs a callable with argument types
        a1, ... an and return type r.
        """
        n = len(a) - 1
        return Callable(a[:-1],
                        [ARG_POS] * n, [None] * n,
                        a[-1], False)


class CombinedTypesSuite(Suite):
    def __init__(self):
        self.test_types = TypesSuite()
        self.test_type_ops = TypeOpsSuite()
        self.test_join = JoinSuite()
        self.test_meet = MeetSuite()
        super().__init__()


if __name__ == '__main__':
    import sys
    run_test(CombinedTypesSuite(), sys.argv[1:])