def cfgJsCompile(shell):
    '''Configures, compiles and copies a js shell according to required parameters.'''
    print "Compiling..."  # Print *with* a trailing newline to avoid breaking other stuff
    os.mkdir(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), 'objdir-js')))
    shell.setJsObjdir(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), 'objdir-js')))

    autoconfRun(shell.getRepoDirJsSrc())
    configureTryCount = 0
    while True:
        try:
            cfgBin(shell)
            break
        except Exception as e:
            configureTryCount += 1
            if configureTryCount > 3:
                print 'Configuration of the js binary failed 3 times.'
                raise
            # This exception message is returned from sps.captureStdout via cfgBin.
            # No idea why this is sps.isLinux as well..
            if sps.isLinux or (sps.isWin and 'Windows conftest.exe configuration permission' in repr(e)):
                print 'Trying once more...'
                continue
    compileJs(shell)
    inspectShell.verifyBinary(shell)

    compileLog = sps.normExpUserPath(os.path.join(shell.getShellCacheDir(),
                                                  shell.getShellNameWithoutExt() + '.fuzzmanagerconf'))
    if not os.path.isfile(compileLog):
        envDump(shell, compileLog)

def getOneBuild(isJsShell, url, buildType):
    '''
    Try to get a complete working build.
    '''
    idNum = getIdFromTboxUrl(url)
    tboxCacheFolder = sps.normExpUserPath(os.path.join(compileShell.ensureCacheDir(),
                                                       'tboxjs-' + buildType + '-' + idNum))
    createTboxCacheFolder(tboxCacheFolder)

    incompleteBuildTxtFile = sps.normExpUserPath(os.path.join(tboxCacheFolder, INCOMPLETE_NOTE))

    if os.path.isfile(getTboxJsBinPath(tboxCacheFolder)):
        return True, idNum, tboxCacheFolder  # Cached, complete

    if os.path.isfile(incompleteBuildTxtFile):
        assert os.listdir(tboxCacheFolder) == [INCOMPLETE_NOTE], 'Only ' + \
            'incompleteBuild.txt should be present in ' + tboxCacheFolder
        readIncompleteBuildTxtFile(incompleteBuildTxtFile, idNum)
        return False, None, None  # Cached, incomplete

    if downloadBuild.downloadBuild(url, tboxCacheFolder, jsShell=isJsShell):
        assert os.listdir(tboxCacheFolder) == ['build'], 'Only ' + \
            'the build subdirectory should be present in ' + tboxCacheFolder
        return True, idNum, tboxCacheFolder  # Downloaded, complete
    else:
        writeIncompleteBuildTxtFile(url, tboxCacheFolder, incompleteBuildTxtFile, idNum)
        return False, None, None  # Downloaded, incomplete

def parseShellOptions(inputArgs):
    """Returns a 'buildOptions' object, which is intended to be immutable."""

    parser, randomizer = addParserOptions()
    buildOptions = parser.parse_args(inputArgs.split())

    # Ensures releng machines do not enter the if block and assumes mozilla-central always exists
    if os.path.isdir(DEFAULT_TREES_LOCATION):
        # Repositories do not get randomized if a repository is specified.
        if buildOptions.repoDir is None:
            # For patch fuzzing without a specified repo, do not randomize repos, assume m-c instead
            if buildOptions.enableRandom and not buildOptions.patchFile:
                buildOptions.repoDir = getRandomValidRepo(DEFAULT_TREES_LOCATION)
            else:
                buildOptions.repoDir = os.path.realpath(sps.normExpUserPath(
                    os.path.join(DEFAULT_TREES_LOCATION, 'mozilla-central')))

        assert hgCmds.isRepoValid(buildOptions.repoDir)

        if buildOptions.patchFile:
            hgCmds.ensureMqEnabled()
            buildOptions.patchFile = sps.normExpUserPath(buildOptions.patchFile)
            assert os.path.isfile(buildOptions.patchFile)

    if buildOptions.enableRandom:
        buildOptions = generateRandomConfigurations(parser, randomizer)
    else:
        buildOptions.buildOptionsStr = inputArgs
        valid = areArgsValid(buildOptions)
        if not valid[0]:
            print 'WARNING: This set of build options is not tested well because: ' + valid[1]

    return buildOptions

def writeIncompleteBuildTxtFile(url, cacheFolder, txtFile, num):
    '''
    Writes a text file indicating that this particular build is incomplete.
    '''
    if os.path.isdir(sps.normExpUserPath(os.path.join(cacheFolder, 'build', 'dist'))) or \
            os.path.isdir(sps.normExpUserPath(os.path.join(cacheFolder, 'build', 'download'))):
        sps.rmTreeIncludingReadOnly(sps.normExpUserPath(os.path.join(cacheFolder, 'build')))
    assert not os.path.isfile(txtFile), 'incompleteBuild.txt should not be present.'
    with open(txtFile, 'wb') as f:
        f.write('This build with numeric ID ' + num + ' is incomplete.')
    assert num == getIdFromTboxUrl(url), 'The numeric ID ' + num + \
        ' has to be the one we downloaded from ' + url
    print 'Wrote a text file that indicates numeric ID ' + num + ' has an incomplete build.'
    return False  # False indicates that this text file has not yet been looked at.

def makeRegressionTestPrologue(repo, regressionTestListFile):
    """Generate a JS string to tell jsfunfuzz where to find SpiderMonkey's regression tests"""

    # We use json.dumps to escape strings (Windows paths have backslashes).
    return """
        const regressionTestsRoot = %s;
        const libdir = regressionTestsRoot + %s; // needed by jit-tests
        var regressionTestList;
        try { regressionTestList = read(%s).match(/.+/g); } catch(e) { }
    """ % (
        json.dumps(sps.normExpUserPath(repo) + os.sep),
        json.dumps(os.path.join('js', 'src', 'jit-test', 'lib') + os.sep),
        json.dumps(os.path.abspath(sps.normExpUserPath(regressionTestListFile))),
    )

def getRandomValidRepo(treeLocation):
    validRepos = []
    for repo in ['mozilla-central', 'mozilla-aurora', 'mozilla-esr45']:
        if os.path.isfile(sps.normExpUserPath(os.path.join(
                treeLocation, repo, '.hg', 'hgrc'))):
            validRepos.append(repo)

    # After checking if repos are valid, reduce chances that non-mozilla-central repos are chosen
    if 'mozilla-aurora' in validRepos and chance(0.5):
        validRepos.remove('mozilla-aurora')
    if 'mozilla-esr45' in validRepos and chance(0.9):
        validRepos.remove('mozilla-esr45')

    return os.path.realpath(sps.normExpUserPath(
        os.path.join(treeLocation, random.choice(validRepos))))

def ensureCacheDir():
    '''Returns a cache directory for compiled shells to live in, creating one if needed'''
    cacheDir = os.path.join(sps.normExpUserPath('~'), 'shell-cache')
    ensureDir(cacheDir)

    # Expand long Windows paths (overcome legacy MS-DOS 8.3 stuff)
    # This has to occur after the shell-cache directory is created
    if sps.isWin:  # adapted from http://stackoverflow.com/a/3931799
        winTmpDir = unicode(cacheDir)
        GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
        unicodeBuffer = ctypes.create_unicode_buffer(GetLongPathName(winTmpDir, 0, 0))
        GetLongPathName(winTmpDir, unicodeBuffer, len(unicodeBuffer))
        cacheDir = sps.normExpUserPath(str(unicodeBuffer.value))  # convert back to a str

    return cacheDir

 def getShellCompiledRunLibsPath(self):
     lDir = self.getJsObjdir()
     libsList = [
         sps.normExpUserPath(os.path.join(lDir, 'dist', 'bin', runLib))
         for runLib in inspectShell.ALL_RUN_LIBS
     ]
     return libsList

def patchHgRepoUsingMq(patchFile, workingDir=os.getcwdu()):
    # We may have passed in the patch with or without the full directory.
    patchAbsPath = os.path.abspath(sps.normExpUserPath(patchFile))
    pname = os.path.basename(patchAbsPath)
    assert pname != ''
    qimportOutput, qimportRetCode = sps.captureStdout(['hg', '-R', workingDir, 'qimport', patchAbsPath],
                                                   combineStderr=True, ignoreStderr=True,
                                                   ignoreExitCode=True)
    if qimportRetCode != 0:
        if 'already exists' in qimportOutput:
            print "A patch with the same name has already been qpush'ed. Please qremove it first."
        raise Exception('Return code from `hg qimport` is: ' + str(qimportRetCode))

    print("Patch qimport'ed..."),

    qpushOutput, qpushRetCode = sps.captureStdout(['hg', '-R', workingDir, 'qpush', pname],
        combineStderr=True, ignoreStderr=True)
    assert ' is empty' not in qpushOutput, "Patch to be qpush'ed should not be empty."

    if qpushRetCode != 0:
        hgQpopQrmAppliedPatch(patchFile, workingDir)
        print 'You may have untracked .rej or .orig files in the repository.'
        print '`hg status` output of the repository of interesting files in ' + workingDir + ' :'
        subprocess.check_call(['hg', '-R', workingDir, 'status', '--modified', '--added',
                               '--removed', '--deleted'])
        raise Exception('Return code from `hg qpush` is: ' + str(qpushRetCode))

    print("Patch qpush'ed. Continuing..."),
    return pname

def getRepoNameFromHgrc(repoDir):
    '''Looks in the hgrc file in the .hg directory of the repository and returns the name.'''
    assert isRepoValid(repoDir)
    hgCfg = ConfigParser.SafeConfigParser()
    hgCfg.read(sps.normExpUserPath(os.path.join(repoDir, '.hg', 'hgrc')))
    # Not all default entries in [paths] end with "/".
    return [i for i in hgCfg.get('paths', 'default').split('/') if i][-1]

def getRepoNameFromHgrc(repoDir):
    """Look in the hgrc file in the .hg directory of the repository and return the name."""
    assert isRepoValid(repoDir)
    hgCfg = ConfigParser.SafeConfigParser()
    hgCfg.read(sps.normExpUserPath(os.path.join(repoDir, ".hg", "hgrc")))
    # Not all default entries in [paths] end with "/".
    return [i for i in hgCfg.get("paths", "default").split("/") if i][-1]

 def getShellCompiledRunLibsPath(self):
     lDir = self.getJsObjdir() if self.getJsBuildSystemConsidersNspr() else self.getNsprObjdir()
     libsList = [
         sps.normExpUserPath(os.path.join(lDir, 'dist', 'bin', runLib))
         for runLib in inspectShell.ALL_RUN_LIBS
     ]
     return libsList

def jsFilesIn(repoPathLength, root):
    return [
        os.path.join(path, filename)[repoPathLength:]
        for path, _dirs, files in os.walk(sps.normExpUserPath(root))
        for filename in files
        if filename.endswith(".js")
    ]

def compileJs(shell):
    '''This function compiles and copies a binary.'''
    try:
        cmdList = [MAKE_BINARY, '-C', shell.getJsObjdir(), '-j' + str(COMPILATION_JOBS), '-s']
        out = sps.captureStdout(cmdList, combineStderr=True, ignoreExitCode=True,
                                currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0]
    except Exception as e:
        # This exception message is returned from sps.captureStdout via cmdList.
        if (sps.isLinux or sps.isMac) and \
                ('GCC running out of memory' in repr(e) or 'Clang running out of memory' in repr(e)):
            # FIXME: Absolute hack to retry after hitting OOM.
            print 'Trying once more due to the compiler running out of memory...'
            out = sps.captureStdout(cmdList, combineStderr=True, ignoreExitCode=True,
                                    currWorkingDir=shell.getJsObjdir(), env=shell.getEnvFull())[0]
        # A non-zero error can be returned during make, but eventually a shell still gets compiled.
        if os.path.exists(shell.getShellCompiledPath()):
            print 'A shell was compiled even though there was a non-zero exit code. Continuing...'
        else:
            print MAKE_BINARY + " did not result in a js shell:"
            raise

    if os.path.exists(shell.getShellCompiledPath()):
        shutil.copy2(shell.getShellCompiledPath(), shell.getShellCacheFullPath())
        for runLib in shell.getShellCompiledRunLibsPath():
            if os.path.isfile(runLib):
                shutil.copy2(runLib, shell.getShellCacheDir())
        if sps.isLinux:
            # Restrict this to only Linux for now. At least Mac OS X needs some (possibly *.a)
            # files in the objdir or else the stacks from failing testcases will lack symbols.
            shutil.rmtree(sps.normExpUserPath(os.path.join(shell.getShellCacheDir(), 'objdir-js')))
    else:
        print out
        raise Exception(MAKE_BINARY + " did not result in a js shell, no exception thrown.")

def compileNspr(shell):
    '''Compile a NSPR binary.'''
    cfgBin(shell, 'nspr')
    # Continue to use -j1 because NSPR does not yet seem to support parallel compilation very well.
    # Even if we move to parallel compile NSPR in the future, we must beware of breaking old
    # build during bisection. Maybe find the changeset that fixes this, and if before that, use -j1,
    # and after that, use -jX ?
    nsprCmdList = [MAKE_BINARY, '-C', shell.getNsprObjdir(), '-j1', '-s']
    out = sps.captureStdout(nsprCmdList, combineStderr=True, ignoreExitCode=True,
                            currWorkingDir=shell.getNsprObjdir(), env=shell.getEnvFull())[0]
    for compileLib in inspectShell.ALL_COMPILE_LIBS:
        if not sps.normExpUserPath(os.path.join(shell.getNsprObjdir(), 'dist', 'lib', compileLib)):
            print out
            raise Exception(MAKE_BINARY + " did not result in a NSPR binary.")

    assert os.path.isdir(sps.normExpUserPath(
        os.path.join(shell.getNsprObjdir(), 'dist', 'include', 'nspr')))

def parseOptions():
    parser = OptionParser()
    parser.add_option('-R', '--repo', dest='rDir',
                      help='Sets the repository to analyze..')
    options, _args = parser.parse_args()
    assert options.rDir is not None
    assert os.path.isdir(sps.normExpUserPath(options.rDir))
    return options

def assertSaneJsBinary(cacheF):
    '''
    If the cache folder is present, check that the js binary is working properly.
    '''
    if os.path.isdir(cacheF):
        fList = os.listdir(cacheF)
        if 'build' in fList:
            if INCOMPLETE_NOTE in fList:
                print cacheF + ' has subdirectories: ' + str(fList)
                raise Exception('Downloaded binaries and incompleteBuild.txt should not both be ' +
                                'present together in this directory.')
            assert os.path.isdir(sps.normExpUserPath(os.path.join(cacheF, 'build', 'download')))
            assert os.path.isdir(sps.normExpUserPath(os.path.join(cacheF, 'build', 'dist')))
            assert os.path.isfile(sps.normExpUserPath(os.path.join(cacheF, 'build', 'dist',
                                                                   'js' + ('.exe' if sps.isWin else ''))))
            try:
                shellPath = getTboxJsBinPath(cacheF)
                # Ensure we don't fail because the shell lacks u+x
                if not os.access(shellPath, os.X_OK):
                    os.chmod(shellPath, stat.S_IXUSR)

                # tbpl binaries are always:
                # * run without Valgrind (they are not compiled with --enable-valgrind)
                retCode = inspectShell.testBinary(shellPath, ['-e', '42'], False)[1]
                # Exit code -1073741515 on Windows shows up when a required DLL is not present.
                # This was testable at the time of writing, see bug 953314.
                isDllNotPresentWinStartupError = (sps.isWin and retCode == -1073741515)
                # We should have another condition here for non-Windows platforms but we do not yet
                # have a situation where we can test broken treeherder js shells on those platforms.
                if isDllNotPresentWinStartupError:
                    raise Exception('Shell startup error - a .dll file is probably not present.')
                elif retCode != 0:
                    raise Exception('Non-zero return code: ' + str(retCode))
                return True  # Binary is working correctly
            except (OSError, IOError):
                raise Exception('Cache folder ' + cacheF + ' is corrupt, please delete it ' +
                                'and try again.')
        elif INCOMPLETE_NOTE in fList:
            return True
        else:
            raise Exception('Neither build/ nor INCOMPLETE_NOTE were found in the cache folder.')
    else:
        raise Exception('Cache folder ' + cacheF + ' is not found.')

def getTimestampAndHashFromTboxFiles(folder):
    '''
    Returns timestamp and changeset information from the .txt file downloaded from treeherder.
    '''
    downloadDir = sps.normExpUserPath(os.path.join(folder, 'build', 'download'))
    for fn in os.listdir(downloadDir):
        if fn.startswith('firefox-') and fn.endswith('.txt') and '_info' not in fn:
            with open(os.path.join(downloadDir, fn), 'rb') as f:
                fContents = f.read().splitlines()
            break
    assert len(fContents) == 2, 'Contents of the .txt file should only have 2 lines'
    return fContents[0], fContents[1].split('/')[-1]

def extractVersions(objdir):
    """Extract the version from <objdir>/js/src/js.pc and put it into *.fuzzmanagerconf."""
    jspcFilename = sps.normExpUserPath(os.path.join(objdir, 'js', 'src', 'js.pc'))
    if os.path.isfile(jspcFilename):
        with open(jspcFilename, 'rb') as f:
            for line in f:
                if line.startswith('Version: '):
                    # Sample line: 'Version: 47.0a2'
                    version = line.split(': ')[1].rstrip()
                    majorVersion = version.split('.')[0]
                    return majorVersion, version
    return '', ''

def makeRegressionTestPrologue(repo):
    """Generate a JS string to tell jsfunfuzz where to find SpiderMonkey's regression tests"""

    return """
        const regressionTestsRoot = %s;
        const libdir = regressionTestsRoot + %s; // needed by jit-tests
        const regressionTestList = %s;
    """ % (
        json.dumps(sps.normExpUserPath(repo) + os.sep),
        json.dumps(os.path.join('js', 'src', 'jit-test', 'lib') + os.sep),
        json.dumps(inTreeRegressionTests(repo))
    )