def test_tests_affected_idlharness(capsys, manifest_dir):
    commit = "47cea8c38b88c0ddd3854e4edec0c5b6f2697e62"
    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["tests-affected", "--metadata", manifest_dir, "%s~..%s" % (commit, commit)])
    assert excinfo.value.code == 0
    out, err = capsys.readouterr()
    assert "webrtc-stats/idlharness.window.js\nwebrtc/idlharness.https.window.js\n" == out

def test_list_tests_invalid_manifest(manifest_dir):
    """The `--list-tests` option should not produce an error in the presence of
    a malformed test manifest file."""

    manifest_filename = os.path.join(manifest_dir, "MANIFEST.json")

    assert os.path.isfile(manifest_filename)

    with open(manifest_filename, "a+") as handle:
        handle.write("extra text which invalidates the file")

    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["run",
                       # This test triggers the creation of a new manifest
                       # file which is not necessary to ensure successful
                       # process completion. Specifying the current directory
                       # as the tests source via the --tests` option
                       # drastically reduces the time to execute the test.
                       "--tests", here,
                       "--metadata", manifest_dir,
                       "--list-tests",
                       "--yes",
                       "firefox", "/dom/nodes/Element-tagName.html"])

    assert excinfo.value.code == 0

def test_run_verify_unstable(temp_test):
    """Unstable tests should be reported with a non-zero exit status. Stable
    tests should be reported with a zero exit status."""
    if is_port_8000_in_use():
        pytest.skip("port 8000 already in use")
    unstable_test = temp_test("""
        test(function() {
            if (localStorage.getItem('wpt-unstable-test-flag')) {
              throw new Error();
            }

            localStorage.setItem('wpt-unstable-test-flag', 'x');
        }, 'my test');
    """)

    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["run", "--yes", "--verify", "--binary-arg", "headless",
                       "chrome", unstable_test])
    assert excinfo.value.code != 0

    stable_test = temp_test("test(function() {}, 'my test');")

    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["run", "--yes", "--verify", "--binary-arg", "headless",
                       "chrome", stable_test])
    assert excinfo.value.code == 0

def test_run_firefox(manifest_dir):
    # TODO: It seems like there's a bug in argparse that makes this argument order required
    # should try to work around that
    if is_port_8000_in_use():
        pytest.skip("port 8000 already in use")

    if sys.platform == "darwin":
        fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "browsers", "nightly", "Firefox Nightly.app")
    else:
        fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "browsers", "nightly", "firefox")
    if os.path.exists(fx_path):
        shutil.rmtree(fx_path)
    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["run", "--no-pause", "--install-browser", "--yes",
                       # The use of `--binary-args` is intentional: it
                       # demonstrates that internally-managed command-line
                       # arguments are properly merged with those specified by
                       # the user. See
                       # https://github.com/web-platform-tests/wpt/pull/13154
                       "--binary-arg=-headless",
                       "--metadata", manifest_dir,
                       "firefox", "/dom/nodes/Element-tagName.html"])
    assert os.path.exists(fx_path)
    shutil.rmtree(fx_path)
    assert excinfo.value.code == 0

def test_list_tests(manifest_dir):
    """The `--list-tests` option should not produce an error under normal
    conditions."""

    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["run", "--metadata", manifest_dir, "--list-tests",
                       "--yes", "chrome", "/dom/nodes/Element-tagName.html"])
    assert excinfo.value.code == 0

def test_install_chromedriver():
    chromedriver_path = os.path.join(wpt.localpaths.repo_root, "_venv", "bin", "chromedriver")
    if os.path.exists(chromedriver_path):
        os.unlink(chromedriver_path)
    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["install", "chrome", "webdriver"])
    assert excinfo.value.code == 0
    assert os.path.exists(chromedriver_path)
    os.unlink(chromedriver_path)

def test_run_chrome(manifest_dir):
    if is_port_8000_in_use():
        pytest.skip("port 8000 already in use")

    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["run", "--yes", "--no-pause", "--binary-arg", "headless",
                       "--metadata", manifest_dir,
                       "chrome", "/dom/nodes/Element-tagName.html"])
    assert excinfo.value.code == 0

def test_install_firefox():
    fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "firefox")
    if os.path.exists(fx_path):
        shutil.rmtree(fx_path)
    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["install", "firefox", "browser"])
    assert excinfo.value.code == 0
    assert os.path.exists(fx_path)
    shutil.rmtree(fx_path)

def test_tests_affected(capsys, manifest_dir):
    # This doesn't really work properly for random commits because we test the files in
    # the current working directory for references to the changed files, not the ones at
    # that specific commit. But we can at least test it returns something sensible
    commit = "9047ac1d9f51b1e9faa4f9fad9c47d109609ab09"
    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["tests-affected", "--metadata", manifest_dir, "%s~..%s" % (commit, commit)])
    assert excinfo.value.code == 0
    out, err = capsys.readouterr()
    assert "html/browsers/offline/appcache/workers/appcache-worker.html" in out

def test_install_firefox():
    if sys.platform == "darwin":
        fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "browsers", "nightly", "Firefox Nightly.app")
    else:
        fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "browsers", "nightly", "firefox")
    if os.path.exists(fx_path):
        shutil.rmtree(fx_path)
    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["install", "firefox", "browser", "--channel=nightly"])
    assert excinfo.value.code == 0
    assert os.path.exists(fx_path)
    shutil.rmtree(fx_path)

def test_tests_affected_null(capsys, manifest_dir):
    # This doesn't really work properly for random commits because we test the files in
    # the current working directory for references to the changed files, not the ones at
    # that specific commit. But we can at least test it returns something sensible.
    # The test will fail if the file we assert is renamed, so we choose a stable one.
    commit = "9bf1daa3d8b4425f2354c3ca92c4cf0398d329dd"
    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["tests-affected", "--null", "--metadata", manifest_dir, "%s~..%s" % (commit, commit)])
    assert excinfo.value.code == 0
    out, err = capsys.readouterr()

    tests = out.split("\0")
    assert "dom/interfaces.html" in tests
    assert "html/dom/interfaces.https.html" in tests

def test_files_changed(capsys):
    commit = "9047ac1d9f51b1e9faa4f9fad9c47d109609ab09"
    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["files-changed", "%s~..%s" % (commit, commit)])
    assert excinfo.value.code == 0
    out, err = capsys.readouterr()
    assert out == """html/browsers/offline/appcache/workers/appcache-worker.html
html/browsers/offline/appcache/workers/resources/appcache-dedicated-worker-not-in-cache.js
html/browsers/offline/appcache/workers/resources/appcache-shared-worker-not-in-cache.js
html/browsers/offline/appcache/workers/resources/appcache-worker-data.py
html/browsers/offline/appcache/workers/resources/appcache-worker-import.py
html/browsers/offline/appcache/workers/resources/appcache-worker.manifest
html/browsers/offline/appcache/workers/resources/appcache-worker.py
"""
    assert err == ""

def test_run_zero_tests():
    """A test execution describing zero tests should be reported as an error
    even in the presence of the `--no-fail-on-unexpected` option."""
    if is_port_8000_in_use():
        pytest.skip("port 8000 already in use")

    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["run", "--yes", "--no-pause", "--binary-arg", "headless",
                       "chrome", "/non-existent-dir/non-existent-file.html"])
    assert excinfo.value.code != 0

    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["run", "--yes", "--no-pause", "--binary-arg", "headless",
                       "--no-fail-on-unexpected",
                       "chrome", "/non-existent-dir/non-existent-file.html"])
    assert excinfo.value.code != 0

def test_run_firefox():
    # TODO: It seems like there's a bug in argparse that makes this argument order required
    # should try to work around that
    os.environ["MOZ_HEADLESS"] = "1"
    try:
        fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "firefox")
        if os.path.exists(fx_path):
            shutil.rmtree(fx_path)
        with pytest.raises(SystemExit) as excinfo:
            wpt.main(argv=["run", "--no-pause", "--install-browser", "--yes",
                           "--metadata", "~/meta/",
                           "firefox", "/dom/nodes/Element-tagName.html"])
        assert os.path.exists(fx_path)
        shutil.rmtree(fx_path)
        assert excinfo.value.code == 0
    finally:
        del os.environ["MOZ_HEADLESS"]

def test_run_failing_test():
    """Failing tests should be reported with a non-zero exit status unless the
    `--no-fail-on-unexpected` option has been specified."""
    if is_port_8000_in_use():
        pytest.skip("port 8000 already in use")
    failing_test = "/infrastructure/expected-fail/failing-test.html"

    assert os.path.isfile("../../%s" % failing_test)

    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["run", "--yes", "--no-pause", "--binary-arg", "headless",
                       "chrome", failing_test])
    assert excinfo.value.code != 0

    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["run", "--yes", "--no-pause", "--binary-arg", "headless",
                       "--no-fail-on-unexpected",
                       "chrome", failing_test])
    assert excinfo.value.code == 0

def test_list_tests_missing_manifest(manifest_dir):
    """The `--list-tests` option should not produce an error in the absence of
    a test manifest file."""

    os.remove(os.path.join(manifest_dir, "MANIFEST.json"))

    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["run",
                       # This test triggers the creation of a new manifest
                       # file which is not necessary to ensure successful
                       # process completion. Specifying the current directory
                       # as the tests source via the --tests` option
                       # drastically reduces the time to execute the test.
                       "--tests", here,
                       "--metadata", manifest_dir,
                       "--list-tests",
                       "--yes",
                       "firefox", "/dom/nodes/Element-tagName.html"])

    assert excinfo.value.code == 0

def test_run_firefox(manifest_dir):
    # TODO: It seems like there's a bug in argparse that makes this argument order required
    # should try to work around that
    if is_port_8000_in_use():
        pytest.skip("port 8000 already in use")

    os.environ["MOZ_HEADLESS"] = "1"
    try:
        if sys.platform == "darwin":
            fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "browsers", "Firefox Nightly.app")
        else:
            fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "browsers", "firefox")
        if os.path.exists(fx_path):
            shutil.rmtree(fx_path)
        with pytest.raises(SystemExit) as excinfo:
            wpt.main(argv=["run", "--no-pause", "--install-browser", "--yes",
                           "--metadata", manifest_dir,
                           "firefox", "/dom/nodes/Element-tagName.html"])
        assert os.path.exists(fx_path)
        shutil.rmtree(fx_path)
        assert excinfo.value.code == 0
    finally:
        del os.environ["MOZ_HEADLESS"]

def init_manifest():
    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["manifest", "--no-download",
                       "--path", get_persistent_manifest_path()])
    assert excinfo.value.code == 0

def test_help():
    # TODO: It seems like there's a bug in argparse that makes this argument order required
    # should try to work around that
    with pytest.raises(SystemExit) as excinfo:
        wpt.main(argv=["--help"])
    assert excinfo.value.code == 0

def test_missing():
    with pytest.raises(SystemExit):
        wpt.main(argv=["#missing-command"])