def pullInstallDragonEgg(args=None):
    """downloads and installs dragonegg (assumes that compatible GCC and G++ versions are installed)"""
    toolDir = join(_toolDir, "tools/dragonegg")
    mx.ensure_dir_exists(toolDir)
    url = "https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/dragonegg-3.2.src.tar.gz"
    localPath = pullsuite(toolDir, [url])
    tar(localPath, toolDir)
    os.remove(localPath)
    if mx.get_os() == "darwin":
        gccToolDir = join(_toolDir, "tools/gcc")
        url = "https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/gcc-4.6.4.tar.gz"
        localPath = pullsuite(gccToolDir, [url])
        tar(localPath, gccToolDir)
        os.remove(localPath)
        mx.run(
            ["patch", "-p1", _toolDir + "tools/dragonegg/dragonegg-3.2.src/Makefile", "mx.sulong/dragonegg-mac.patch"]
        )
    os.environ["GCC"] = getGCC()
    os.environ["CXX"] = getGPP()
    os.environ["CC"] = getGCC()
    pullLLVMBinaries()
    os.environ["LLVM_CONFIG"] = findLLVMProgram("llvm-config")
    print os.environ["LLVM_CONFIG"]
    compileCommand = ["make"]
    return mx.run(compileCommand, cwd=_toolDir + "tools/dragonegg/dragonegg-3.2.src")

def get_java_module_info(dist, fatalIfNotModule=False):
    """
    Gets the metadata for the module derived from `dist`.

    :param JARDistribution dist: a distribution possibly defining a module
    :param bool fatalIfNotModule: specifies whether to abort if `dist` does not define a module
    :return: None if `dist` does not define a module otherwise a tuple containing
             the name of the module, the directory in which the class files
             (including module-info.class) for the module are staged and finally
             the path to the jar file containing the built module
    """
    if dist.suite.getMxCompatibility().moduleDepsEqualDistDeps():
        moduleName = getattr(dist, 'moduleName', None)
        if not moduleName:
            if fatalIfNotModule:
                mx.abort('Distribution ' + dist.name + ' does not define a module')
            return None
        assert len(moduleName) > 0, '"moduleName" attribute of distribution ' + dist.name + ' cannot be empty'
    else:
        if not get_module_deps(dist):
            if fatalIfNotModule:
                mx.abort('Module for distribution ' + dist.name + ' would be empty')
            return None
        moduleName = dist.name.replace('_', '.').lower()

    modulesDir = mx.ensure_dir_exists(join(dist.suite.get_output_root(), 'modules'))
    moduleDir = mx.ensure_dir_exists(join(modulesDir, moduleName))
    moduleJar = join(modulesDir, moduleName + '.jar')
    return moduleName, moduleDir, moduleJar

def testgraal(args):
    cloneFrom = mx.get_env("GRAAL_URL")
    if not cloneFrom:
        cloneFrom = "http://github.com/graalvm/graal-core"
    graalSuiteSubDir = mx.get_env("GRAAL_SUITE_SUBDIR")

    suite = mx.suite('truffle')
    suiteDir = suite.dir
    workDir = join(suite.get_output_root(), 'sanitycheck')
    mx.ensure_dir_exists(join(workDir, suite.name))
    for f in os.listdir(suiteDir):
        subDir = os.path.join(suiteDir, f)
        if subDir == suite.get_output_root():
            continue
        src = join(suiteDir, f)
        tgt = join(workDir, suite.name, f)
        if isdir(src):
            if exists(tgt):
                shutil.rmtree(tgt)
            shutil.copytree(src, tgt)
        else:
            shutil.copy(src, tgt)

    sanityDir = join(workDir, 'sanity')
    git = mx.GitConfig()
    if exists(sanityDir):
        git.pull(sanityDir)
    else:
        git.clone(cloneFrom, sanityDir)

    sanitySuiteDir = sanityDir if graalSuiteSubDir is None else join(sanityDir, graalSuiteSubDir)
    return mx.run_mx(['--java-home=' + mx.get_jdk().home, 'gate', '-B--force-deprecation-as-warning', '--tags', 'build,test'], sanitySuiteDir)

def pullLLVMSuite(args=None):
    """downloads the official (non Truffle) LLVM test suite"""
    mx.ensure_dir_exists(_llvmSuiteDir)
    urls = ["https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/test-suite-3.2.src.tar.gz"]
    localPath = pullsuite(_llvmSuiteDir, urls)
    tar(localPath, _llvmSuiteDir)
    os.remove(localPath)

def pullLifetime(args=None):
    """downloads the lifetime reference outputs"""
    mx.ensure_dir_exists(_lifetimeReferenceDir)
    urls = ["https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/lifetime-analysis-ref.tar.gz"]
    localPath = pullsuite(_lifetimeReferenceDir, urls)
    tar(localPath, _lifetimeReferenceDir)
    os.remove(localPath)

def pullArgon2(args=None):
    """downloads Argon2"""
    mx.ensure_dir_exists(_argon2Dir)
    urls = ["https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/20160406.tar.gz"]
    localPath = pullsuite(_argon2Dir, urls)
    tar(localPath, _argon2Dir, ['phc-winner-argon2-20160406/'], stripLevels=1)
    os.remove(localPath)

def pullNWCCSuite(args=None):
    """downloads the NWCC test suite"""
    mx.ensure_dir_exists(_nwccSuiteDir)
    urls = ["https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/nwcc_0.8.3.tar.gz"]
    localPath = pullsuite(_nwccSuiteDir, urls)
    tar(localPath, _nwccSuiteDir, ['nwcc_0.8.3/tests/', 'nwcc_0.8.3/test2/'], stripLevels=1)
    os.remove(localPath)

def pullLLVMBinaries(args=None):
    """downloads the LLVM binaries"""
    toolDir = join(_toolDir, "tools/llvm")
    mx.ensure_dir_exists(toolDir)
    osStr = mx.get_os()
    arch = mx.get_arch()
    if osStr == 'windows':
        print 'windows currently only supported with cygwin!'
        return
    elif osStr == 'linux':
        if arch == 'amd64':
            urls = ['http://lafo.ssw.uni-linz.ac.at/sulong-deps/clang+llvm-3.2-x86_64-linux-ubuntu-12.04.tar.gz',
                    'http://llvm.org/releases/3.2/clang+llvm-3.2-x86_64-linux-ubuntu-12.04.tar.gz']
        else:
            urls = ['http://lafo.ssw.uni-linz.ac.at/sulong-deps/clang+llvm-3.2-x86-linux-ubuntu-12.04.tar.gz',
                    'http://llvm.org/releases/3.2/clang+llvm-3.2-x86-linux-ubuntu-12.04.tar.gz']
    elif osStr == 'darwin':
        urls = ['http://lafo.ssw.uni-linz.ac.at/sulong-deps/clang+llvm-3.2-x86_64-apple-darwin11.tar.gz',
                'http://llvm.org/releases/3.2/clang+llvm-3.2-x86_64-apple-darwin11.tar.gz']
    elif osStr == 'cygwin':
        urls = ['http://lafo.ssw.uni-linz.ac.at/sulong-deps/clang+llvm-3.2-x86-mingw32-EXPERIMENTAL.tar.gz',
                'http://llvm.org/releases/3.2/clang+llvm-3.2-x86-mingw32-EXPERIMENTAL.tar.gz']
    else:
        print osStr, arch, "not supported!"
    localPath = pullsuite(toolDir, urls)
    tar(localPath, toolDir, stripLevels=1)
    os.remove(localPath)

def _update_JDK9_STUBS_library():
    """
    Sets the "path" and "sha1" attributes of the "JDK9_STUBS" library.
    """
    jdk9InternalLib = _suite.suiteDict['libraries']['JDK9_STUBS']
    jarInputDir = join(_suite.get_output_root(), 'jdk9-stubs')
    jarPath = join(_suite.get_output_root(), 'jdk9-stubs.jar')

    stubs = [
        ('jdk.internal.misc', 'VM', """package jdk.internal.misc;
public class VM {
    public static String getSavedProperty(String key) {
        throw new InternalError("should not reach here");
    }
}
""")
    ]

    if not exists(jarPath):
        sourceFiles = []
        for (package, className, source) in stubs:
            sourceFile = join(jarInputDir, package.replace('.', os.sep), className + '.java')
            mx.ensure_dir_exists(os.path.dirname(sourceFile))
            with open(sourceFile, 'w') as fp:
                fp.write(source)
            sourceFiles.append(sourceFile)
        jdk = mx.get_jdk(tag='default')
        mx.run([jdk.javac, '-d', jarInputDir] + sourceFiles)
        mx.run([jdk.jar, 'cf', jarPath, '.'], cwd=jarInputDir)

    jdk9InternalLib['path'] = jarPath
    jdk9InternalLib['sha1'] = mx.sha1OfFile(jarPath)

def _get_java_module_info(dist):
    assert len(get_module_deps(dist)) != 0
    modulesDir = mx.ensure_dir_exists(join(dist.suite.get_output_root(), 'modules'))
    moduleName = dist.name.replace('_', '.').lower()
    moduleDir = mx.ensure_dir_exists(join(modulesDir, moduleName))
    moduleJar = join(modulesDir, moduleName + '.jar')
    return moduleName, moduleDir, moduleJar

def jacocoreport(args):
    """create a JaCoCo coverage report

    Creates the report from the 'jacoco.exec' file in the current directory.
    Default output directory is 'coverage', but an alternative can be provided as an argument."""
    jacocoreport = mx.library("JACOCOREPORT", True)
    out = 'coverage'
    if len(args) == 1:
        out = args[0]
    elif len(args) > 1:
        mx.abort('jacocoreport takes only one argument : an output directory')

    includes = list(_jacoco_includes)
    for p in mx.projects():
        projsetting = getattr(p, 'jacoco', '')
        if projsetting == 'include' or projsetting == '':
            includes.append(p.name)

    includedirs = set()

    for p in mx.projects():
        projsetting = getattr(p, 'jacoco', '')
        if projsetting == 'exclude':
            continue
        for include in includes:
            if include in p.dir:
                includedirs.add(p.dir)

    for i in includedirs:
        bindir = i + '/bin'
        mx.ensure_dir_exists(bindir)

    mx.run_java(['-jar', jacocoreport.get_path(True), '--in', 'jacoco.exec', '--out', out] + sorted(includedirs))

def pullBenchmarkGame(args=None):
    """downloads the benchmarks"""
    mx.ensure_dir_exists(_benchGameSuiteDir)
    urls = ["https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/benchmarksgame-scm-latest.tar.gz"]
    localPath = pullsuite(_benchGameSuiteDir, urls)
    tar(localPath, _benchGameSuiteDir, ['benchmarksgame-2014-08-31/benchmarksgame/bench/'], stripLevels=3)
    os.remove(localPath)
    renameBenchmarkFiles()

def pullGCCSuite(args=None):
    """downloads the GCC test suite"""
    suiteDir = _gccSuiteDir
    mx.ensure_dir_exists(suiteDir)
    urls = ["https://lafo.ssw.uni-linz.ac.at/pub/sulong-deps/gcc-5.2.0.tar.gz"]
    localPath = pullsuite(suiteDir, urls)
    tar(localPath, suiteDir, ['gcc-5.2.0/gcc/testsuite/'])
    os.remove(localPath)

def pullNWCCSuite(args=None):
    """downloads the NWCC test suite"""
    mx.ensure_dir_exists(_nwccSuiteDir)
    urls = ["http://lafo.ssw.uni-linz.ac.at/sulong-deps/nwcc_0.8.3.tar.gz",
            "http://sourceforge.net/projects/nwcc/files/nwcc/nwcc%200.8.3/nwcc_0.8.3.tar.gz/download"]
    localPath = pullsuite(_nwccSuiteDir, urls)
    tar(localPath, _nwccSuiteDir, ['nwcc_0.8.3/tests/', 'nwcc_0.8.3/test2/'], stripLevels=1)
    os.remove(localPath)

def pullTestSuite(library, destDir, **kwargs):
    """downloads and unpacks a test suite"""
    mx.ensure_dir_exists(destDir)
    localPath = mx.library(library).get_path(True)
    mx_sulong.tar(localPath, destDir, **kwargs)
    os.remove(localPath)
    sha1Path = localPath + '.sha1'
    if os.path.exists(sha1Path):
        os.remove(sha1Path)

def pullGCCSuite(args=None):
    """downloads the GCC test suite"""
    suiteDir = _gccSuiteDir
    mx.ensure_dir_exists(suiteDir)
    urls = ["http://lafo.ssw.uni-linz.ac.at/sulong-deps/gcc-5.2.0.tar.gz",
            "ftp://gd.tuwien.ac.at/gnu/gcc/releases/gcc-5.2.0/gcc-5.2.0.tar.gz",
            "ftp://ftp.fu-berlin.de/unix/languages/gcc/releases/gcc-5.2.0/gcc-5.2.0.tar.gz",
            "http://mirrors-usa.go-parts.com/gcc/releases/gcc-5.2.0/gcc-5.2.0.tar.gz"]
    localPath = pullsuite(suiteDir, urls)
    tar(localPath, suiteDir, ['gcc-5.2.0/gcc/testsuite/'])
    os.remove(localPath)

def _sigtest_check(checktype, args, suite=None, projects=None):
    """run sigtest against Java projects with API"""
    sigtestlib = mx.library('SIGTEST').get_path(resolve=True)
    nonTestProjects = [p for p in mx.projects() if _should_test_project(p)]
    if not nonTestProjects:
        return 1
    javaCompliance = max([p.javaCompliance for p in nonTestProjects])

    class OutputCapture:
        def __init__(self):
            self.data = ""
        def __call__(self, data):
            self.data += data
    failed = None
    for p in nonTestProjects:
        sigtestResults = p.dir + os.sep + 'snapshot.sigtest'
        if not os.path.exists(sigtestResults):
            continue
        jdk = mx.get_jdk(javaCompliance)
        cmd = ['-cp', mx._cygpathU2W(sigtestlib), 'com.sun.tdk.signaturetest.SignatureTest',
            '-Static', '-Mode', 'bin', '-FileName', sigtestResults,
            '-ClassPath', mx.classpath(p, jdk=jdk) + os.pathsep + jdk.bootclasspath(),
        ]
        if checktype != 'all':
            cmd.append('-b')
        for pkg in mx._find_packages(p):
            cmd = cmd + ['-PackageWithoutSubpackages', pkg]
        out = OutputCapture()
        print 'Checking ' + checktype + ' signature changes against ' + sigtestResults
        exitcode = mx.run_java(cmd, nonZeroIsFatal=False, jdk=mx.get_jdk(javaCompliance), out=out, err=out)
        mx.ensure_dir_exists(p.get_output_root())
        with open(p.get_output_root() + os.path.sep + 'sigtest-junit.xml', 'w') as f:
            f.write('<?xml version="1.0" encoding="UTF-8" ?>\n')
            f.write('<testsuite tests="1" name="' + p.name + '.sigtest.' + checktype + '">\n')
            f.write('<testcase classname="' + p.name + '" name="sigtest.' + checktype + '">\n')
            if exitcode != 95:
                print out.data
                failed = sigtestResults
                f.write('<failure type="SignatureCheck"><![CDATA[\n')
                f.write(out.data)
                f.write(']]></failure>')
            else:
                f.write('<system-err><![CDATA[\n')
                f.write(out.data)
                f.write(']]></system-err>')
            f.write('</testcase>\n')
            f.write('</testsuite>\n')
    if failed:
        mx.abort('Signature error in ' + failed)
    else:
        print 'OK.'
    return 0

def pullInstallDragonEgg(args=None):
    """downloads and installs dragonegg (assumes that compatible GCC and G++ versions are installed)"""
    toolDir = join(_toolDir, "tools/dragonegg")
    mx.ensure_dir_exists(toolDir)
    url = 'http://llvm.org/releases/3.2/dragonegg-3.2.src.tar.gz'
    localPath = pullsuite(toolDir, [url])
    tar(localPath, toolDir)
    os.remove(localPath)
    os.environ['GCC'] = getGCC()
    os.environ['CXX'] = getGPP()
    os.environ['CC'] = getGCC()
    os.environ['LLVM_CONFIG'] = _toolDir + 'tools/llvm/bin/llvm-config'
    compileCommand = ['make']
    return mx.run(compileCommand, cwd=_toolDir + 'tools/dragonegg/dragonegg-3.2.src')

def pullInstallDragonEgg(args=None):
    """downloads and installs dragonegg (assumes that GCC 4.6 is on the path)"""
    if hasDragoneggGCCInstalled():
        toolDir = join(_toolDir, "tools/dragonegg")
        mx.ensure_dir_exists(toolDir)
        url = 'http://llvm.org/releases/3.2/dragonegg-3.2.src.tar.gz'
        localPath = pullsuite(toolDir, [url])
        tar(localPath, toolDir)
        os.remove(localPath)
        os.environ['GCC'] = 'gcc-4.6'
        os.environ['LLVM_CONFIG'] = _toolDir + 'tools/llvm/bin/llvm-config'
        compileCommand = ['make']
        mx.run(compileCommand, cwd=_toolDir + 'tools/dragonegg/dragonegg-3.2.src')
    else:
        print 'could not find gcc-4.6, skip installing dragonegg!'

def testdownstream(args):
    """test downstream users of GraalCore"""
    parser = ArgumentParser(prog='mx testdownstream')
    parser.add_argument('--target', action='store', help='URL of client repo to clone', required=True, metavar='<url>')
    parser.add_argument('--suitedir', action='store', help='directory of target suite in client repo', default='.', metavar='<path>')
    parser.add_argument('-C', dest='clientMxCmd', action='append', help='arg to mx command run on client (e.g., -C-v -C--strict-compliance -Cgate)', default=[], metavar='<arg>')

    args = parser.parse_args(args)

    workDir = join(_suite.get_output_root(), 'testdownstream')
    mirror = join(workDir, _suite.name)
    if exists(mirror):
        shutil.rmtree(mirror)
    mx.ensure_dir_exists(mirror)
    for f in os.listdir(_suite.dir):
        subDir = join(_suite.dir, f)
        if subDir == _suite.get_output_root():
            continue
        src = join(_suite.dir, f)
        dst = join(mirror, f)
        mx.logv('[Creating symlink from {} to {}]'.format(dst, src))
        os.symlink(src, dst)

    # Deduce a target name from the target URL
    url = urlparse(args.target)
    targetName = url.path
    if targetName.rfind('/') != -1:
        targetName = targetName[targetName.rfind('/') + 1:]
    if targetName.endswith('.git'):
        targetName = targetName[0:-len('.git')]

    targetDir = join(workDir, targetName)
    git = mx.GitConfig()
    if exists(targetDir):
        git.pull(targetDir)
    else:
        git.clone(args.target, targetDir)

    # See if there's a matching (non-master) branch downstream and use it if there is
    branch = git.git_command(_suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
    if branch != 'master':
        git.git_command(targetDir, ['checkout', branch], abortOnError=False)

    targetSuiteDir = join(targetDir, args.suitedir)
    cmd = ['--java-home=' + mx.get_jdk().home] + args.clientMxCmd
    mx.logv('[running "mx ' + ' '.join(cmd) + '" in ' + targetSuiteDir + ']')
    return mx.run_mx(cmd, targetSuiteDir)