def test_cgen(testcase):
    # Build the program.
    text = '\n'.join(testcase.input)
    program = '_program.py'
    try:
        build.build(program,
                    target=build.C,
                    program_text=text,
                    flags=[build.TEST_BUILTINS],
                    alt_lib_path='lib')
        # Run the program.
        outfile = './_program'
        outb = subprocess.check_output([outfile], stderr=subprocess.STDOUT)
        # Split output into lines.
        out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()]
        # Remove temp file.
        os.remove(outfile)
    except errors.CompileError as e:
        out = e.messages
    # Include line-end comments in the expected output.
    # Note: # characters in string literals can confuse this.
    for s in testcase.input:
        m = re.search(' #(?! type:)(.*)', s)
        if m:
            testcase.output.append(m.group(1).strip())
    # Verify output.
    assert_string_arrays_equal(testcase.output, out,
                               'Invalid output ({}, line {})'.format(
                                   testcase.file, testcase.line))

def test_error_stream(testcase: DataDrivenTestCase) -> None:
    """Perform a single error streaming test case.

    The argument contains the description of the test case.
    """
    options = Options()
    options.show_traceback = True

    logged_messages = []  # type: List[str]

    def flush_errors(msgs: List[str], serious: bool) -> None:
        if msgs:
            logged_messages.append('==== Errors flushed ====')
            logged_messages.extend(msgs)

    sources = [BuildSource('main', '__main__', '\n'.join(testcase.input))]
    try:
        build.build(sources=sources,
                    options=options,
                    alt_lib_path=test_temp_dir,
                    flush_errors=flush_errors)
    except CompileError as e:
        assert e.messages == []

    assert_string_arrays_equal(testcase.output, logged_messages,
                               'Invalid output ({}, line {})'.format(
                                   testcase.file, testcase.line))

def type_check_only(sources: List[BuildSource],
        options: Options) -> None:
    # Type check the program and dependencies and translate to Python.
    build.build(sources=sources,
                target=build.TYPE_CHECK,
                implementation=options.implementation,
                custom_typing_module=options.custom_typing_module,
                report_dirs=options.report_dirs,
                flags=options.build_flags,
                python_path=options.python_path)

def type_check_only(path: str, module: str, program_text: str,
        bin_dir: str, options: Options) -> None:
    # Type check the program and dependencies and translate to Python.
    build.build(path,
                module=module,
                program_text=program_text,
                bin_dir=bin_dir,
                target=build.TYPE_CHECK,
                pyversion=options.pyversion,
                custom_typing_module=options.custom_typing_module,
                report_dirs=options.report_dirs,
                flags=options.build_flags,
                python_path=options.python_path)

 def run_test(self, testcase):
     a = []
     try:
         src = '\n'.join(testcase.input)
         build.build('main',
                     target=build.TYPE_CHECK,
                     program_text=src,
                     flags=[build.TEST_BUILTINS],
                     alt_lib_path=test_temp_dir)
     except CompileError as e:
         a = normalize_error_messages(e.messages)
     assert_string_arrays_equal(
         testcase.output, a,
         'Invalid type checker output ({}, line {})'.format(
             testcase.file, testcase.line))

 def run_test(self, testcase):
     a = []
     try:
         line = testcase.input[0]
         mask = ''
         if line.startswith('##'):
             mask = '(' + line[2:].strip() + ')$'
         
         src = '\n'.join(testcase.input)
         result = build.build(program_path='main',
                              target=build.TYPE_CHECK,
                              program_text=src,
                              flags=[build.TEST_BUILTINS],
                              alt_lib_path=testconfig.test_temp_dir)
         map = result.types
         kk = map.keys()
         keys = []
         for k in kk:
             if k.line is not None and k.line != -1 and map[k]:
                 if (re.match(mask, short_type(k))
                         or (isinstance(k, NameExpr)
                             and re.match(mask, k.name))):
                     keys.append(k)
         for key in sorted(keys,
                           key=lambda n: (n.line, short_type(n),
                                          str(n) + str(map[n]))):
             ts = str(map[key]).replace('*', '') # Remove erased tags
             ts = ts.replace('__main__.', '')
             a.append('{}({}) : {}'.format(short_type(key), key.line, ts))
     except CompileError as e:
         a = e.messages
     assert_string_arrays_equal(
         testcase.output, a,
         'Invalid type checker output ({}, line {})'.format(testcase.file,
                                                            testcase.line))

    def run_test(self, testcase):
        """Perform a test case."""
        try:
            # Build test case input.
            src = '\n'.join(testcase.input)
            result = build.build('main',
                                 target=build.SEMANTIC_ANALYSIS,
                                 program_text=src,
                                 flags=[build.TEST_BUILTINS],
                                 alt_lib_path=test_temp_dir)

            # Collect all TypeInfos in top-level modules.
            typeinfos = TypeInfoMap()
            for f in result.files.values():
                for n in f.names.values():
                    if isinstance(n.node, TypeInfo):
                        typeinfos[n.fullname] = n.node

            # The output is the symbol table converted into a string.
            a = str(typeinfos).split('\n')
        except CompileError as e:
            a = e.messages
        assert_string_arrays_equal(
            testcase.output, a,
            'Invalid semantic analyzer output ({}, line {})'.format(
                testcase.file, testcase.line))

    def run_case(self, testcase: DataDrivenTestCase) -> None:
        """Perform a test case."""
        try:
            # Build test case input.
            src = '\n'.join(testcase.input)
            result = build.build(sources=[BuildSource('main', None, src)],
                                 options=get_semanal_options(),
                                 alt_lib_path=test_temp_dir)
            a = result.errors
            if a:
                raise CompileError(a)

            # Collect all TypeInfos in top-level modules.
            typeinfos = TypeInfoMap()
            for f in result.files.values():
                for n in f.names.values():
                    if isinstance(n.node, TypeInfo):
                        assert n.fullname is not None
                        typeinfos[n.fullname] = n.node

            # The output is the symbol table converted into a string.
            a = str(typeinfos).split('\n')
        except CompileError as e:
            a = e.messages
        assert_string_arrays_equal(
            testcase.output, a,
            'Invalid semantic analyzer output ({}, line {})'.format(
                testcase.file, testcase.line))

def test_semanal(testcase):
    """Perform a semantic analysis test case.

    The testcase argument contains a description of the test case
    (inputs and output).
    """

    try:
        src = "\n".join(testcase.input)
        options = get_semanal_options()
        options.python_version = testfile_pyversion(testcase.file)
        result = build.build(sources=[BuildSource("main", None, src)], options=options, alt_lib_path=test_temp_dir)
        a = result.errors
        if a:
            raise CompileError(a)
        # Include string representations of the source files in the actual
        # output.
        for fnam in sorted(result.files.keys()):
            f = result.files[fnam]
            # Omit the builtins module and files with a special marker in the
            # path.
            # TODO the test is not reliable
            if (
                not f.path.endswith(
                    (os.sep + "builtins.pyi", "typing.pyi", "mypy_extensions.pyi", "abc.pyi", "collections.pyi")
                )
                and not os.path.basename(f.path).startswith("_")
                and not os.path.splitext(os.path.basename(f.path))[0].endswith("_")
            ):
                a += str(f).split("\n")
    except CompileError as e:
        a = e.messages
    assert_string_arrays_equal(
        testcase.output, a, "Invalid semantic analyzer output ({}, line {})".format(testcase.file, testcase.line)
    )

def test_transform(testcase):
    """Perform an identity transform test case."""
    
    try:
        src = '\n'.join(testcase.input)
        result = build.build('main',
                             target=build.SEMANTIC_ANALYSIS,
                             program_text=src,
                             pyversion=testfile_pyversion(testcase.file),
                             flags=[build.TEST_BUILTINS],
                             alt_lib_path=test_temp_dir)
        a = []
        # Include string representations of the source files in the actual
        # output.
        for fnam in sorted(result.files.keys()):
            f = result.files[fnam]

            # Omit the builtins module and files with a special marker in the
            # path.
            # TODO the test is not reliable
            if (not f.path.endswith((os.sep + 'builtins.py',
                                     'typing.py',
                                     'abc.py'))
                    and not os.path.basename(f.path).startswith('_')
                    and not os.path.splitext(
                        os.path.basename(f.path))[0].endswith('_')):
                t = TestTransformVisitor()
                f = t.node(f)
                a += str(f).split('\n')
    except CompileError as e:
        a = e.messages
    assert_string_arrays_equal(
        testcase.output, a,
        'Invalid semantic analyzer output ({}, line {})'.format(testcase.file,
                                                                testcase.line))

def test_transform(testcase):
    """Perform a runtime checking transformation test case."""
    expected = remove_comment_lines(testcase.output)

    func_names = get_func_names(expected)

    try:
        # Construct input as a single single.
        src = '\n'.join(testcase.input)
        # Parse and type check the input program.
        result = build.build(program_path='main',
                             target=build.ICODE,
                             program_text=src,
                             alt_lib_path=test_temp_dir)
        a = []
        for fn in func_names:
            a.append('def {}:'.format(fn))
            try:
                funccode = result.icode[fn]
            except KeyError:
                raise RuntimeError('no icode for %s (%s)' % (
                    fn, list(result.icode.keys())))
            code = icode.render(funccode)
            a.extend(code)
    except CompileError as e:
        a = e.messages
    assert_string_arrays_equal_wildcards(
        expected, a,
        'Invalid source code output ({}, line {})'.format(testcase.file,
                                                          testcase.line))

def type_check_only(sources: List[BuildSource], bin_dir: Optional[str],
                    options: Options,
                    flush_errors: Optional[Callable[[List[str], bool], None]]) -> BuildResult:
    # Type-check the program and dependencies.
    return build.build(sources=sources,
                       bin_dir=bin_dir,
                       options=options,
                       flush_errors=flush_errors)

def test_semanal_error(testcase):
    """Perform a test case."""

    try:
        src = '\n'.join(testcase.input)
        build.build(target=build.SEMANTIC_ANALYSIS,
                    sources=[BuildSource('main', None, src)],
                    flags=[build.TEST_BUILTINS],
                    alt_lib_path=test_temp_dir)
        raise AssertionError('No errors reported in {}, line {}'.format(
            testcase.file, testcase.line))
    except CompileError as e:
        # Verify that there was a compile error and that the error messages
        # are equivalent.
        assert_string_arrays_equal(
            testcase.output, normalize_error_messages(e.messages),
            'Invalid compiler output ({}, line {})'.format(testcase.file,
                                                           testcase.line))

 def build(self,
           options: Options,
           sources: List[BuildSource]) -> List[str]:
     try:
         result = build.build(sources=sources,
                              options=options)
     except CompileError as e:
         return e.messages
     return result.errors

def compile_to_python(path, module, args):
    if path:
        basedir = os.path.dirname(path)
    else:
        basedir = os.getcwd()
    
    outputdir = os.path.join(basedir, '__mycache__')
    tempdir = False
    if not os.path.isdir(outputdir):
        try:
            os.mkdir(outputdir)
        except OSError:
            # Could not create a directory under program directory; must
            # fall back to a temp directory. It will be removed later.
            outputdir = tempfile.mkdtemp()
            tempdir = True

    try:
        # Type check the program and dependencies and translate to Python.
        build.build(path,
                    module=module,
                    target=build.PYTHON,
                    output_dir=outputdir,
                    flags=build_flags)

        if build.COMPILE_ONLY not in build_flags:
            # Run the translated program.
            if module:
                # Run the module using runpy. We can't use -m since Python
                # would try to run the mypy code instead of the translated
                # code.
                p = os.path.join(outputdir, '__main__.py')
                f = open(p, 'w')
                f.write('import runpy\n'
                        "runpy.run_module('%s', run_name='__main__')" % module)
                f.close()
                opts = [p]
            else:
                opts = [os.path.join(outputdir, os.path.basename(path))]
            status = subprocess.call([interpreter] + opts + args)
            sys.exit(status)
    finally:
        if tempdir:
            shutil.rmtree(outputdir)

def type_check_only(sources: List[BuildSource],
        bin_dir: str, options: Options) -> BuildResult:
    # Type-check the program and dependencies and translate to Python.
    return build.build(sources=sources,
                       target=build.TYPE_CHECK,
                       bin_dir=bin_dir,
                       pyversion=options.pyversion,
                       custom_typing_module=options.custom_typing_module,
                       report_dirs=options.report_dirs,
                       flags=options.build_flags,
                       python_path=options.python_path)

def test_cgen_compile(testcase):
    # Build the program.
    text = '\n'.join(testcase.input)
    try:
        build.build('_program.py',
                    target=build.C,
                    program_text=text, 
                    alt_lib_path='lib',
                    flags=[build.COMPILE_ONLY, build.TEST_BUILTINS])
        outfile = '_program.c'
        f = open(outfile)
        out = [s.rstrip('\n\r') for s in f.readlines()]
        f.close()
        os.remove(outfile)
    except errors.CompileError as e:
        out = e.messages
    # Verify output.
    assert_string_arrays_equal_wildcards(testcase.output, out,
                               'Invalid output ({}, line {})'.format(
                                   testcase.file, testcase.line))

def compile_to_c(path, module, args):
    assert not module # Not supported yet
    assert not args   # Not supported yet
    
    # Compile the program to C (also generate binary by default).
    result = build.build(path, target=build.C, flags=build_flags)

    if build.COMPILE_ONLY not in build_flags:
        # Run the compiled program.
        # TODO command line arguments
        status = subprocess.call([result.binary_path])
        sys.exit(status)

    def run_case_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
        find_module_clear_caches()
        program_text = '\n'.join(testcase.input)
        module_name, program_name, program_text = self.parse_module(program_text)

        options = self.parse_options(program_text)
        options.use_builtins_fixtures = True
        options.python_version = testcase_pyversion(testcase.file, testcase.name)

        output = testcase.output
        if incremental:
            options.incremental = True
            if incremental == 1:
                # In run 1, copy program text to program file.
                output = []
                with open(program_name, 'w') as f:
                    f.write(program_text)
            elif incremental == 2:
                # In run 2, copy *.py.next files to *.py files.
                for dn, dirs, files in os.walk(os.curdir):
                    for file in files:
                        if file.endswith('.py.next'):
                            full = os.path.join(dn, file)
                            target = full[:-5]
                            shutil.copy(full, target)
            # Always set to none so we're forced to reread program_name
            program_text = None
        source = BuildSource(program_name, module_name, program_text)
        try:
            res = build.build(sources=[source],
                              options=options,
                              alt_lib_path=test_temp_dir)
            a = res.errors
        except CompileError as e:
            res = None
            a = e.messages
        a = normalize_error_messages(a)

        if output != a and self.update_data:
            update_testcase_output(testcase, a)

        assert_string_arrays_equal(
            output, a,
            'Invalid type checker output ({}, line {})'.format(
                testcase.file, testcase.line))

        if incremental and res:
            self.verify_cache(module_name, program_name, a, res.manager)
            if testcase.expected_stale_modules is not None and incremental == 2:
                assert_string_arrays_equal(
                    list(sorted(testcase.expected_stale_modules)),
                    list(sorted(res.manager.stale_modules.difference({"__main__"}))),
                    'Set of stale modules does not match expected set')

    def run_test(self, testcase):
        a = []
        pyversion = testcase_pyversion(testcase.file, testcase.name)
        try:
            src = '\n'.join(testcase.input)
            build.build('main',
                        target=build.TYPE_CHECK,
                        program_text=src,
                        pyversion=pyversion,
                        flags=[build.TEST_BUILTINS],
                        alt_lib_path=test_temp_dir)
        except CompileError as e:
            a = normalize_error_messages(e.messages)

        if testcase.output != a and UPDATE_TESTCASES:
            update_testcase_output(testcase, a, APPEND_TESTCASES)

        assert_string_arrays_equal(
            testcase.output, a,
            'Invalid type checker output ({}, line {})'.format(
                testcase.file, testcase.line))