def test_unsupported_cluster():
    try:
        tools.get_command('quartz', 'exe', check_executable_existance=False)
    except Exception, e:
        actual = str(e)
        expected = 'Unsupported Cluster: quartz'
        assert actual == expected

def test_bad_data_filedir_15():
    try:
        tools.get_command('ray', 'exe', data_filename_test_default='e', check_executable_existance=False)
    except Exception, e:
        actual = str(e)
        expected = 'Invalid Usage: At least one of [data_filedir_train_default, data_filename_train_default, data_filedir_test_default, data_filename_test_default] is set, but neither data_reader_name or data_reader_path are.'
        assert actual == expected

def test_bad_data_filedir_8():
    try:
        tools.get_command('ray', 'exe', data_reader_path='path', data_filedir_default='filedir', data_filename_test_default='h', check_executable_existance=False)
    except Exception, e:
        actual = str(e)
        expected = 'Invalid Usage: data_fildir_default set but so is at least one of [data_filedir_train_default, data_filename_train_default, data_filedir_test_default, data_filename_test_default]'
        assert actual == expected

def test_bad_data_filedir_11():
    try:
        tools.get_command('ray', 'exe', data_filedir_default='filedir', check_executable_existance=False)
    except Exception, e:
        actual = str(e)
        expected = 'Invalid Usage: data_filedir_default set but neither data_reader_name or data_reader_path are.'
	assert actual == expected

def test_bad_model_5():
    try:
        tools.get_command('ray', 'exe', dir_name='dir', model_name='name', check_executable_existance=False)
    except Exception, e:
        actual = str(e)
        expected = 'Invalid Usage: model_name set but not model_folder.'
        assert actual == expected

def test_bad_dir_name_5():
    try:
        tools.get_command('ray', 'exe', optimizer_name='name', check_executable_existance=False)
    except Exception, e:
	actual = str(e)
	expected = 'Invalid Usage: dir_name is not set but at least one of model_folder, model_name, data_reader_name, optimizer_name is.'
        assert actual == expected

def test_bad_model_3():
    try:
        tools.get_command('ray', 'exe', dir_name='dir', model_name='name',  model_path='path', check_executable_existance=False)
    except Exception, e:
        actual = str(e)
        expected = 'Invalid Usage: model_path is set but so is at least one of model folder and model_name'
        assert actual == expected

def test_bad_data_reader():
    try:
        tools.get_command('catalyst', 'exe', dir_name='dir', data_reader_name='name', data_reader_path='path', check_executable_existance=False)
    except Exception, e:
        actual = str(e)
        expected = 'Invalid Usage: data_reader_path is set but so is data_reader_name , data_reader_name or data_reader_path is set but not data_filedir_default. If a data reader is provided, the default filedir must be set. This allows for determining what the filedir should be on each cluster. Alternatively, some or all of [data_filedir_train_default, data_filename_train_default, data_filedir_test_default, data_filename_test_default] can be set.'
        assert actual == expected

def test_bad_optimizer():
    try:
        tools.get_command('ray', 'exe', dir_name='dir', optimizer_name='name', optimizer_path='path', check_executable_existance=False)
    except Exception, e:
        actual = str(e)
        expected = 'Invalid Usage: optimizer_path is set but so is optimizer_name'
        assert actual == expected

def test_blacklisted_substrings():
    try:
        tools.get_command('ray', 'exe', partition=';', optimizer_path='--model=new_model', check_executable_existance=False)
    except Exception, e:
        actual = str(e)
        expected = 'Invalid character(s): ; contains ; , --model=new_model contains --'
        assert actual == expected

def skeleton_checkpoint_lenet_shared(cluster, executables, dir_name, compiler_name):
    if compiler_name not in executables:
      pytest.skip('default_exes[%s] does not exist' % compiler_name)
    exe = executables[compiler_name]
    output_file_name = '%s/bamboo/unit_tests/output/checkpoint_lenet_shared_no_checkpoint_%s_output.txt' % (dir_name, compiler_name)
    error_file_name  = '%s/bamboo/unit_tests/error/checkpoint_lenet_shared_no_checkpoint_%s_error.txt' % (dir_name, compiler_name)
    command = tools.get_command(
        cluster=cluster, executable=exe, num_nodes=1, num_processes=2,
        dir_name=dir_name,
        data_filedir_default='/p/lscratchh/brainusr/datasets/MNIST',
        data_reader_name='mnist', model_folder='tests',
        model_name='lenet_mnist_ckpt', num_epochs=2, optimizer_name='sgd',
        output_file_name=output_file_name, error_file_name=error_file_name)
    return_code_nockpt = os.system(command)
    if return_code_nockpt != 0:
        sys.stderr.write('LeNet (no checkpoint) execution failed, exiting with error')
        sys.exit(1)
    os.system('mv ckpt ckpt_baseline')

    output_file_name = '%s/bamboo/unit_tests/output/checkpoint_lenet_shared_checkpoint_%s_output.txt' % (dir_name, compiler_name)
    error_file_name  = '%s/bamboo/unit_tests/error/checkpoint_lenet_shared_checkpoint_%s_error.txt' % (dir_name, compiler_name)
    command = tools.get_command(
        cluster=cluster, executable=exe, num_nodes=1, num_processes=2,
        dir_name=dir_name,
        data_filedir_default='/p/lscratchh/brainusr/datasets/MNIST',
        data_reader_name='mnist', model_folder='tests',
        model_name='lenet_mnist_ckpt', num_epochs=1, optimizer_name='sgd',
        output_file_name=output_file_name, error_file_name=error_file_name)
    return_code_ckpt_1 = os.system(command)
    if return_code_ckpt_1 != 0:
        sys.stderr.write('LeNet (checkpoint) execution failed, exiting with error')
        sys.exit(1)

    output_file_name = '%s/bamboo/unit_tests/output/checkpoint_lenet_shared_restart_%s_output.txt' % (dir_name, compiler_name)
    error_file_name  = '%s/bamboo/unit_tests/error/checkpoint_lenet_shared_restart_%s_error.txt' % (dir_name, compiler_name)
    command = tools.get_command(
        cluster=cluster, executable=exe, num_nodes=1, num_processes=2,
        dir_name=dir_name,
        data_filedir_default='/p/lscratchh/brainusr/datasets/MNIST',
        data_reader_name='mnist', model_folder='tests',
        model_name='lenet_mnist_ckpt', num_epochs=2, optimizer_name='sgd',
        output_file_name=output_file_name, error_file_name=error_file_name)
    return_code_ckpt_2 = os.system(command)
    if return_code_ckpt_2 != 0:
        sys.stderr.write('LeNet execution (restart from checkpoint) failed, exiting with error')
        sys.exit(1)

    diff_test = os.system('diff -rq ckpt ckpt_baseline')
    os.system('rm -rf ckpt*')
    assert diff_test == 0

def test_unit_no_params_bad(cluster, exes, dirname):
    exe = exes['gcc4']
    sys.stderr.write('TESTING: run lbann with no params; lbann should throw exception\n')
    command = tools.get_command(
        cluster=cluster, executable=exe, exit_after_setup=True)
    return_code = os.system(command)
    assert return_code != 0

def test_unit_two_models_bad2(cluster, exes,  dirname):
    exe = exes['gcc4']
    sys.stderr.write('TESTING: run lbann with two models with missing {; lbann should throw exception\n')
    model_path='prototext/model_mnist_simple_1.prototext,prototext/model_mnist_simple_1.prototext}'
    command = tools.get_command(
        cluster=cluster, executable=exe, exit_after_setup=True,
        model_path=model_path)
    return_code = os.system(command)
    assert return_code != 0

def test_unit_one_model_bad(cluster, exes, dirname):
    exe = exes['gcc4']
    sys.stderr.write('TESTING: run lbann with no optimizer or reader; lbann should throw exception\n')
    model_path = 'prototext/model_mnist_simple_1.prototext'
    command = tools.get_command(
        cluster=cluster, executable=exe, exit_after_setup=True,
        model_path=model_path)
    return_code = os.system(command)
    assert return_code != 0

def test_unit_missing_reader(cluster, exes, dirname):
    exe = exes['gcc4']
    sys.stderr.write('TESTING: run lbann with two models, reader, but no reader; lbann should throw exception\n')
    model_path = '{prototext/model_mnist_simple_1.prototext,prototext/model_mnist_simple_1.prototext}'
    optimizer_path = 'prototext/opt_sgd.prototext'
    command = tools.get_command(
        cluster=cluster, executable=exe, exit_after_setup=True,
        model_path=model_path, optimizer_path=optimizer_path)
    return_code = os.system(command)
    assert return_code != 0

def test_unit_missing_optimizer(cluster, exes, dirname):
    exe = exes['gcc4']
    sys.stderr.write('TESTING: run lbann with two models, reader, but no optimizer; lbann should throw exception\n')
    model_path='{prototext/model_mnist_simple_1.prototext,prototext/model_mnist_simple_1.prototext}'
    data_reader_path='prototext/data_reader_mnist.prototext'
    command = tools.get_command(
        cluster=cluster, executable=exe, data_reader_path=data_reader_path,
        data_filedir_default='/p/lscratchh/brainusr/datasets/MNIST',
        exit_after_setup=True, model_path=model_path)
    return_code = os.system(command)
    assert return_code != 0

def skeleton_layer_log_sigmoid(cluster, executables, dir_name, compiler_name):
    if compiler_name not in executables:
      pytest.skip('default_exes[%s] does not exist' % compiler_name)
    output_file_name = '%s/bamboo/unit_tests/output/layer_log_sigmoid_%s_output.txt' % (dir_name, compiler_name)
    error_file_name  = '%s/bamboo/unit_tests/error/layer_log_sigmoid_%s_error.txt' % (dir_name, compiler_name)
    command = tools.get_command(
        cluster=cluster, executable=executables[compiler_name], num_nodes=1, num_processes=2, dir_name=dir_name,
        data_filedir_default='', data_reader_name='synthetic',
        model_folder='tests/layer_tests', model_name='log_sigmoid', optimizer_name='sgd',
        output_file_name=output_file_name, error_file_name=error_file_name)
    return_code = os.system(command)
    assert return_code == 0

def skeleton_mnist_ridge_regression(cluster, executables, dir_name, compiler_name):
    if compiler_name not in executables:
      pytest.skip('default_exes[%s] does not exist' % compiler_name)
    output_file_name = '%s/bamboo/unit_tests/output/mnist_ridge_regression_%s_output.txt' % (dir_name, compiler_name)
    error_file_name  = '%s/bamboo/unit_tests/error/mnist_ridge_regression_%s_error.txt' % (dir_name, compiler_name)
    command = tools.get_command(
        cluster=cluster, executable=executables[compiler_name], num_nodes=1, num_processes=1, dir_name=dir_name,
        data_filedir_default='/p/lscratchh/brainusr/datasets/MNIST', data_reader_name='mnist',
        model_folder='tests', model_name='mnist_ridge_regression', optimizer_name='adam',
        output_file_name=output_file_name, error_file_name=error_file_name)
    return_code = os.system(command)
    assert return_code == 0

def get_command(cluster, dir_name, model_folder, model_name, executable,
                output_file_name, error_file_name, compiler_name, weekly=False):
    if model_name in ['alexnet', 'conv_autoencoder_imagenet']:
        data_reader_percent = 0.01
        if weekly:
            data_reader_percent = 0.10
        command = tools.get_command(
            cluster=cluster, executable=executable, num_nodes=16,
            partition='pbatch', time_limit=600, num_processes=32,
            dir_name=dir_name,
            data_filedir_train_default='/p/lscratchh/brainusr/datasets/ILSVRC2012/original/train/',
            data_filename_train_default='/p/lscratchh/brainusr/datasets/ILSVRC2012/labels/train.txt',
            data_filedir_test_default='/p/lscratchh/brainusr/datasets/ILSVRC2012/original/val/',
            data_filename_test_default='/p/lscratchh/brainusr/datasets/ILSVRC2012/labels/val.txt',
            data_reader_name='imagenet', data_reader_percent=data_reader_percent,
            model_folder=model_folder, model_name=model_name, num_epochs=20,
            optimizer_name='adagrad', output_file_name=output_file_name,
            error_file_name=error_file_name)
    elif model_name in ['conv_autoencoder_mnist', 'lenet_mnist']:
        if (model_name == 'lenet_mnist') and (compiler_name in ['clang4', 'intel18']):
            partition = 'pbatch'
            time_limit = 600
        else:
            partition = 'pdebug'
            time_limit = 30
        if (cluster == 'ray') and (model_name == 'conv_autoencoder_mnist'):
            num_processes = 20
        else:
            num_processes = 2
	command = tools.get_command(
            cluster=cluster, executable=executable, num_nodes=1,
            partition=partition, time_limit=time_limit, num_processes=num_processes,
            dir_name=dir_name,
            data_filedir_default='/p/lscratchh/brainusr/datasets/MNIST',
            data_reader_name='mnist', model_folder=model_folder,
            model_name=model_name, num_epochs=5, optimizer_name='adagrad',
            output_file_name=output_file_name, error_file_name=error_file_name)
    else:
        raise Exception('Invalid model: %s' % model_name)
    return command

def skeleton_mnist_debug(cluster, dir_name, executables, compiler_name, weekly, debug, should_log=False):
    # If weekly or debug are true, then run the test.
    if (not weekly) and (not debug):
        pytest.skip('Not doing weekly or debug testing')
    if compiler_name not in executables:
      pytest.skip('default_exes[%s] does not exist' % compiler_name)
    model_name = 'lenet_mnist'
    output_file_name = '%s/bamboo/integration_tests/output/%s_%s_output.txt' %(dir_name, model_name, compiler_name)
    error_file_name = '%s/bamboo/integration_tests/error/%s_%s_error.txt' %(dir_name, model_name, compiler_name)
    command = tools.get_command(
        cluster=cluster, executable=executables[compiler_name], num_nodes=1,
        partition='pbatch', time_limit=100, dir_name=dir_name,
        data_filedir_default='/p/lscratchh/brainusr/datasets/MNIST',
        data_reader_name='mnist', model_folder='models/' + model_name,
        model_name=model_name, num_epochs=5, optimizer_name='adagrad',
        output_file_name=output_file_name, error_file_name=error_file_name)
    output_value = common_code.run_lbann(command, model_name, output_file_name, error_file_name)
    assert output_value == 0