def uninstall(service_name, package_name=None):
    start = time.time()

    if package_name is None:
        package_name = service_name
    print('Uninstalling/janitoring {}'.format(service_name))
    try:
        shakedown.uninstall_package_and_wait(package_name, service_name=service_name)
    except (dcos.errors.DCOSException, ValueError) as e:
        print('Got exception when uninstalling package, ' +
              'continuing with janitor anyway: {}'.format(e))

    janitor_start = time.time()

    janitor_cmd = (
        'docker run mesosphere/janitor /janitor.py '
        '-r {svc}-role -p {svc}-principal -z dcos-service-{svc} --auth_token={auth}')
    shakedown.run_command_on_master(janitor_cmd.format(
        svc=service_name,
        auth=shakedown.run_dcos_command('config show core.dcos_acs_token')[0].strip()))

    finish = time.time()

    print('Uninstall done after pkg({}) + janitor({}) = total({})'.format(
        sdk_spin.pretty_time(janitor_start - start),
        sdk_spin.pretty_time(finish - janitor_start),
        sdk_spin.pretty_time(finish - start)))

def test_launch_container_with_persistent_volume():
    """ Tests launching a task with PV.  It will write to a file in the PV.
        The app is killed and restarted and we can still read from the PV.
    """
    app_def = persistent_volume_app()
    app_id = app_def['id']
    client = marathon.create_client()
    client.add_app(app_def)
    shakedown.deployment_wait()

    tasks = client.get_tasks(app_id)
    assert len(tasks) == 1

    port = tasks[0]['ports'][0]
    host = tasks[0]['host']
    cmd = "curl {}:{}/data/foo".format(host, port)
    run, data = shakedown.run_command_on_master(cmd)

    assert run, "{} did not succeed".format(cmd)
    assert data == 'hello\n', "'{}' was not equal to hello\\n".format(data)

    client.restart_app(app_id)
    shakedown.deployment_wait()

    tasks = client.get_tasks(app_id)
    assert len(tasks) == 1

    port = tasks[0]['ports'][0]
    host = tasks[0]['host']
    cmd = "curl {}:{}/data/foo".format(host, port)
    run, data = shakedown.run_command_on_master(cmd)

    assert run, "{} did not succeed".format(cmd)
    assert data == 'hello\nhello\n', "'{}' was not equal to hello\\nhello\\n".format(data)

def uninstall():
    print("Uninstalling/janitoring {}".format(PACKAGE_NAME))
    try:
        shakedown.uninstall_package_and_wait(PACKAGE_NAME, service_name=PACKAGE_NAME)
    except (dcos.errors.DCOSException, ValueError) as e:
        print("Got exception when uninstalling package, continuing with janitor anyway: {}".format(e))

    shakedown.run_command_on_master(
        "docker run mesosphere/janitor /janitor.py "
        "-r cassandra-role -p {} -z dcos-service-cassandra "
        "--auth_token={}".format(PRINCIPAL, shakedown.run_dcos_command("config show core.dcos_acs_token")[0].strip())
    )

def uninstall():
    print('Uninstalling/janitoring {}'.format(PACKAGE_NAME))
    try:
        shakedown.uninstall_package_and_wait(PACKAGE_NAME, app_id=PACKAGE_NAME)
    except (dcos.errors.DCOSException, ValueError) as e:
        print('Got exception when uninstalling package, continuing with janitor anyway: {}'.format(e))

    shakedown.run_command_on_master(
        'docker run mesosphere/janitor /janitor.py '
        '-r cassandra-role -p cassandra-principal -z dcos-service-cassandra '
        '--auth_token={}'.format(
            shakedown.run_dcos_command(
                'config show core.dcos_acs_token'
            )[0].strip()
        )
    )

def test_pod_file_based_secret(secret_fixture):
    secret_name, secret_value = secret_fixture
    secret_normalized_name = secret_name.replace('/', '')

    pod_id = '/{}'.format(uuid.uuid4().hex)

    pod_def = {
        "id": pod_id,
        "containers": [{
            "name": "container-1",
            "resources": {
                "cpus": 0.1,
                "mem": 64
            },
            "endpoints": [{
                "name": "http",
                "hostPort": 0,
                "protocol": [
                    "tcp"
                ]}
            ],
            "exec": {
                "command": {
                    "shell": "cat {} >> {}_file && /opt/mesosphere/bin/python -m http.server $ENDPOINT_HTTP".format(
                        secret_normalized_name, secret_normalized_name),
                }
            },
            "volumeMounts": [{
                "name": "vol",
                "mountPath": secret_name
            }],
        }],
        "networks": [{
            "mode": "host"
        }],
        "volumes": [{
            "name": "vol",
            "secret": "secret1"
        }],
        "secrets": {
            "secret1": {
                "source": secret_name
            }
        }
    }

    client = marathon.create_client()
    client.add_pod(pod_def)
    shakedown.deployment_wait()

    instances = client.show_pod(pod_id)['instances']
    assert len(instances) == 1, 'Failed to start the file based secret pod'

    port = instances[0]['containers'][0]['endpoints'][0]['allocatedHostPort']
    host = instances[0]['networks'][0]['addresses'][0]
    cmd = "curl {}:{}/{}_file".format(host, port, secret_normalized_name)
    status, data = shakedown.run_command_on_master(cmd)

    assert status, "{} did not succeed".format(cmd)
    assert data.rstrip() == secret_value, "Got an unexpected secret data"

def assert_http_code(url, http_code='200'):
    cmd = r'curl -s -o /dev/null -w "%{http_code}"'
    cmd = cmd + ' {}'.format(url)
    status, output = shakedown.run_command_on_master(cmd)

    assert status
    assert output == http_code

def test_docker_dns_mapping(marathon_service_name):
    """ Tests that a running docker task is accessible from DNS.
    """

    app_id = uuid.uuid4().hex
    client = marathon.create_client()
    app_json = app_docker(app_id)
    client.add_app(app_json)
    shakedown.deployment_wait()

    tasks = client.get_tasks(app_id)
    host = tasks[0]['host']

    bad_cmd = 'ping -c 1 docker-test.marathon-user.mesos-bad'
    status, output = shakedown.run_command_on_master(bad_cmd)
    assert not status

    @retrying.retry(stop_max_attempt_number=30)
    def check_dns():
        cmd = 'ping -c 1 {}.{}.mesos'.format(app_id, marathon_service_name)
        shakedown.wait_for_dns('{}.{}.mesos'.format(app_id, marathon_service_name))
        status, output = shakedown.run_command_on_master(cmd)
        assert status

    check_dns()

    def get_master():
        exit_status, output = shakedown.run_command_on_master(
            "{}/_cat/master'".format(_curl_api(service_name, "GET")))
        if exit_status and len(output.split()) > 0:
            return output.split()[-1]

        return False

def test_mom_with_network_failure_bounce_master():
    """Marathon on Marathon (MoM) tests for DC/OS with network failures simulated by knocking out ports."""

    # get MoM ip
    mom_ip = common.ip_of_mom()
    print("MoM IP: {}".format(mom_ip))

    app_def = apps.sleep_app()
    app_id = app_def["id"]

    with shakedown.marathon_on_marathon():
        client = marathon.create_client()
        client.add_app(app_def)
        shakedown.wait_for_task("marathon-user", app_id.lstrip('/'))
        tasks = client.get_tasks(app_id)
        original_task_id = tasks[0]["id"]
        task_ip = tasks[0]['host']
        print("\nTask IP: " + task_ip)

    # PR for network partitioning in shakedown makes this better
    # take out the net
    partition_agent(mom_ip)
    partition_agent(task_ip)

    # wait for a min
    time.sleep(timedelta(minutes=1).total_seconds())

    # bounce master
    shakedown.run_command_on_master("sudo systemctl restart dcos-mesos-master")

    # bring the net up
    reconnect_agent(mom_ip)
    reconnect_agent(task_ip)

    time.sleep(timedelta(minutes=1).total_seconds())
    shakedown.wait_for_service_endpoint('marathon-user', timedelta(minutes=10).total_seconds())

    with shakedown.marathon_on_marathon():
        client = marathon.create_client()
        shakedown.wait_for_task("marathon-user", app_id.lstrip('/'), timedelta(minutes=10).total_seconds())

        @retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
        def check_task_is_back():
            tasks = client.get_tasks(app_id)
            assert tasks[0]['id'] == original_task_id, "The task ID has changed"

        check_task_is_back()

def test_mom_with_network_failure_bounce_master():
    """Marathon on Marathon (MoM) tests for DC/OS with network failures simulated by
    knocking out ports
    """

    # get MoM ip
    mom_ip = ip_of_mom()
    print("MoM IP: {}".format(mom_ip))

    app_def = get_resource("{}/large-sleep.json".format(fixture_dir()))

    with shakedown.marathon_on_marathon():
        client = marathon.create_client()
        client.add_app(app_def)
        shakedown.wait_for_task("marathon-user", "sleep")
        tasks = client.get_tasks('sleep')
        original_sleep_task_id = tasks[0]["id"]
        task_ip = tasks[0]['host']
        print("\nTask IP: " + task_ip)

    # PR for network partitioning in shakedown makes this better
    # take out the net
    partition_agent(mom_ip)
    partition_agent(task_ip)

    # wait for a min
    time.sleep(timedelta(minutes=1).total_seconds())

    # bounce master
    shakedown.run_command_on_master("sudo systemctl restart dcos-mesos-master")

    # bring the net up
    reconnect_agent(mom_ip)
    reconnect_agent(task_ip)

    time.sleep(timedelta(minutes=1).total_seconds())
    shakedown.wait_for_service_endpoint('marathon-user')
    shakedown.wait_for_task("marathon-user", "sleep")

    with shakedown.marathon_on_marathon():
        client = marathon.create_client()
        shakedown.wait_for_task("marathon-user", "sleep")
        tasks = client.get_tasks('sleep')
        current_sleep_task_id = tasks[0]["id"]

    assert current_sleep_task_id == original_sleep_task_id, "Task ID shouldn't change"

def run_hdfs_command(service_name, command):
    """
    Execute the command using the Docker client
    """
    full_command = 'docker run -e HDFS_SERVICE_NAME={} mesosphere/hdfs-client:2.6.4 /bin/bash -c "/configure-hdfs.sh && {}"'.format(service_name, command)

    rc, output = shakedown.run_command_on_master(full_command)
    return rc, output

def test_pod_secret_env_var(secret_fixture):
    # Install enterprise-cli since it's needed to create secrets
    if not common.is_enterprise_cli_package_installed():
        common.install_enterprise_cli_package()

    secret_name, secret_value = secret_fixture

    pod_id = '/{}'.format(uuid.uuid4().hex)
    pod_def = {
        "id": pod_id,
        "containers": [{
            "name": "container-1",
            "resources": {
                "cpus": 0.1,
                "mem": 64
            },
            "endpoints": [{
                "name": "http",
                "hostPort": 0,
                "protocol": [
                    "tcp"
                ]}
            ],
            "exec": {
                "command": {
                    "shell": "echo $SECRET_ENV && echo $SECRET_ENV >> $MESOS_SANDBOX/secret-env && /opt/mesosphere/bin/python -m http.server $ENDPOINT_HTTP"
                }
            }
        }],
        "environment": {
            "SECRET_ENV": {
                "secret": "secret1"
            }
        },
        "networks": [{
            "mode": "host"
        }],
        "secrets": {
            "secret1": {
                "source": secret_name
            }
        }
    }

    client = marathon.create_client()
    client.add_pod(pod_def)
    shakedown.deployment_wait()

    instances = client.show_pod(pod_id)['instances']
    assert len(instances) == 1, 'Failed to start the secret environment variable pod'

    port = instances[0]['containers'][0]['endpoints'][0]['allocatedHostPort']
    host = instances[0]['networks'][0]['addresses'][0]
    cmd = "curl {}:{}/secret-env".format(host, port)
    status, data = shakedown.run_command_on_master(cmd)

    assert status, "{} did not succeed".format(cmd)
    assert data.rstrip() == secret_value

def get_framework_srv_records(package_name):
    cmd = "curl localhost:8123/v1/enumerate"
    ok, out = shakedown.run_command_on_master(cmd)
    assert ok, "Failed to get srv records. command was {}".format(cmd)
    srvs = json.loads(out)
    framework_srvs = [f for f in srvs["frameworks"] if f["name"] == package_name]
    assert len(framework_srvs) == 1, "Got too many srv records matching package {}, got {}"\
        .format(package_name, framework_srvs)
    return framework_srvs[0]

def ee_version():

    version = "NA"
    # cat /opt/mesosphere/etc/bootstrap-config.json | jq '.["security"]'
    status, stdout = run_command_on_master('cat /opt/mesosphere/etc/bootstrap-config.json')
    if status:
        configuration = json.loads(stdout)
        version = configuration['security']
    return version

def master_ssh(cmd: str) -> tuple:
    '''
    Runs the provided command on the cluster master, using ssh.
    Returns a boolean (==success) and a string (output)
    '''
    log.info('(SSH:master) {}'.format(cmd))
    success, output = shakedown.run_command_on_master(cmd)
    log.info('Output (success={}):\n{}'.format(success, output))
    return success, output

    def fn():
        try:
            shakedown.uninstall_package_and_wait(PACKAGE_NAME)
        except (dcos.errors.DCOSException, json.decoder.JSONDecodeError):
            return False

        return shakedown.run_command_on_master(
            'docker run mesosphere/janitor /janitor.py '
            '-r cassandra-role -p cassandra-principal -z cassandra'
        )

def test_app_file_based_secret(secret_fixture):
    # Install enterprise-cli since it's needed to create secrets
    # if not common.is_enterprise_cli_package_installed():
    # common.install_enterprise_cli_package()

    secret_name, secret_value = secret_fixture
    secret_normalized_name = secret_name.replace('/', '')
    secret_container_path = 'mysecretpath'

    app_id = uuid.uuid4().hex
    # In case you're wondering about the `cmd`: secrets are mounted via tmpfs inside
    # the container and are not visible outside, hence the intermediate file
    app_def = {
        "id": app_id,
        "instances": 1,
        "cpus": 0.1,
        "mem": 64,
        "cmd": "cat {} >> {}_file && /opt/mesosphere/bin/python -m http.server $PORT_API".
            format(secret_container_path, secret_container_path),
        "container": {
            "type": "MESOS",
            "volumes": [{
                "containerPath": secret_container_path,
                "secret": "secret1"
            }]
        },
        "portDefinitions": [{
            "port": 0,
            "protocol": "tcp",
            "name": "api",
            "labels": {}
        }],
        "secrets": {
            "secret1": {
                "source": secret_name
            }
        }
    }

    client = marathon.create_client()
    client.add_app(app_def)
    shakedown.deployment_wait()

    tasks = client.get_tasks(app_id)
    assert len(tasks) == 1, 'Failed to start the file based secret app'

    port = tasks[0]['ports'][0]
    host = tasks[0]['host']
    # The secret by default is saved in $MESOS_SANDBOX/.secrets/path/to/secret
    cmd = "curl {}:{}/{}_file".format(host, port, secret_container_path)
    status, data = shakedown.run_command_on_master(cmd)

    assert status, "{} did not succeed".format(cmd)
    assert data == secret_value

    def fn():
        command = (
            "sudo kill -9 "
            "$(ps ax | grep {} | grep -v grep | tr -s ' ' | sed 's/^ *//g' | "
            "cut -d ' ' -f 1)".format(pattern))
        if agent_host is None:
            exit_status, _ = shakedown.run_command_on_master(command)
        else:
            exit_status, _ = shakedown.run_command_on_agent(agent_host, command)

        return exit_status

def test_restart_container_with_persistent_volume():
    """A task with a persistent volume, which writes to a file in the persistent volume, is launched.
       The app is killed and restarted and we can still read from the persistent volume what was written to it.
    """

    app_def = apps.persistent_volume_app()
    app_id = app_def['id']

    client = marathon.create_client()
    client.add_app(app_def)

    shakedown.deployment_wait()

    tasks = client.get_tasks(app_id)
    assert len(tasks) == 1, "The number of tasks is {} after deployment, but 1 was expected".format(len(tasks))

    port = tasks[0]['ports'][0]
    host = tasks[0]['host']
    cmd = "curl {}:{}/data/foo".format(host, port)
    run, data = shakedown.run_command_on_master(cmd)

    assert run, "{} did not succeed".format(cmd)
    assert data == 'hello\n', "'{}' was not equal to hello\\n".format(data)

    client.restart_app(app_id)
    shakedown.deployment_wait()

    @retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
    def check_task_recovery():
        tasks = client.get_tasks(app_id)
        assert len(tasks) == 1, "The number of tasks is {} after recovery, but 1 was expected".format(len(tasks))

    check_task_recovery()

    port = tasks[0]['ports'][0]
    host = tasks[0]['host']
    cmd = "curl {}:{}/data/foo".format(host, port)
    run, data = shakedown.run_command_on_master(cmd)

    assert run, "{} did not succeed".format(cmd)
    assert data == 'hello\nhello\n', "'{}' was not equal to hello\\nhello\\n".format(data)

def kill_task_with_pattern(pattern, host=None):
    command = (
        "sudo kill -9 "
        "$(ps ax | grep {} | grep -v grep | tr -s ' ' | sed 's/^ *//g' | "
        "cut -d ' ' -f 1)".format(pattern))
    if host is None:
        result = shakedown.run_command_on_master(command)
    else:
        result = shakedown.run_command_on_agent(host, command)

    if not result:
        raise RuntimeError('Failed to kill task with pattern "{}"'.format(pattern))