def msrc():
    models_basedir = 'models/msrc/'
    crf = EdgeCRF(n_states=24, n_features=2028, n_edge_features=4,
                  inference_method='gco')
    clf = OneSlackSSVM(crf, max_iter=10000, C=0.01, verbose=2,
                       tol=0.1, n_jobs=4,
                       inference_cache=100)

    X, Y = load_msrc('train')
    Y = remove_areas(Y)

    start = time()
    clf.fit(X, Y)
    stop = time()

    np.savetxt(models_basedir + 'msrc_full.csv', clf.w)
    with open(models_basedir + 'msrc_full' + '.pickle', 'w') as f:
        pickle.dump(clf, f)

    X, Y = load_msrc('test')
    Y = remove_areas(Y)

    Y_pred = clf.predict(X)

    print('Error on test set: %f' % compute_error(Y, Y_pred))
    print('Score on test set: %f' % clf.score(X, Y))
    print('Norm of weight vector: |w|=%f' % np.linalg.norm(clf.w))
    print('Elapsed time: %f s' % (stop - start))

    return clf

def test_one_slack_constraint_caching():
    # testing cutting plane ssvm on easy multinomial dataset
    X, Y = generate_blocks_multinomial(n_samples=10, noise=0.5, seed=0,
                                       size_x=9)
    n_labels = len(np.unique(Y))
    exact_inference = get_installed([('ad3', {'branch_and_bound': True}), "lp"])[0]
    crf = GridCRF(n_states=n_labels, inference_method=exact_inference)
    clf = OneSlackSSVM(model=crf, max_iter=150, C=1,
                       check_constraints=True, break_on_bad=True,
                       inference_cache=50, inactive_window=0)
    clf.fit(X, Y)
    Y_pred = clf.predict(X)
    assert_array_equal(Y, Y_pred)
    assert_equal(len(clf.inference_cache_), len(X))
    # there should be 13 constraints, which are less than the 94 iterations
    # that are done
    # check that we didn't change the behavior of how we construct the cache
    constraints_per_sample = [len(cache) for cache in clf.inference_cache_]
    if exact_inference == "lp":
        assert_equal(len(clf.inference_cache_[0]), 18)
        assert_equal(np.max(constraints_per_sample), 18)
        assert_equal(np.min(constraints_per_sample), 18)
    else:
        assert_equal(len(clf.inference_cache_[0]), 13)
        assert_equal(np.max(constraints_per_sample), 20)
        assert_equal(np.min(constraints_per_sample), 11)

def syntetic_test():
    # test model on different train set size & on different train sets
    results = np.zeros((18, 5))
    full_labeled = np.array([2, 4, 10, 25, 100])
    train_size = 400

    for dataset in range(1, 19):
        X, Y = load_syntetic(dataset)

        for j, nfull in enumerate(full_labeled):
            crf = EdgeCRF(n_states=10, n_features=10, n_edge_features=2,
                          inference_method='qpbo')
            clf = OneSlackSSVM(crf, max_iter=10000, C=0.01, verbose=0,
                               tol=0.1, n_jobs=4, inference_cache=100)

            x_train = X[:nfull]
            y_train = Y[:nfull]
            x_test = X[(train_size + 1):]
            y_test = Y[(train_size + 1):]

            try:
                clf.fit(x_train, y_train)
                y_pred = clf.predict(x_test)

                results[dataset - 1, j] = compute_error(y_test, y_pred)

                print('dataset=%d, nfull=%d, error=%f' % (dataset, nfull,
                                                          results[dataset - 1, j]))
            except ValueError:
                print('dataset=%d, nfull=%d: Failed' % (dataset, nfull))

    np.savetxt('results/syntetic/full_labeled.txt', results)

def test_class_weights_rescale_C():
    # check that our crammer-singer implementation with class weights and
    # rescale_C=True is the same as LinearSVC's c-s class_weight implementation
    from sklearn.svm import LinearSVC
    X, Y = make_blobs(n_samples=210, centers=3, random_state=1, cluster_std=3,
                      shuffle=False)
    X = np.hstack([X, np.ones((X.shape[0], 1))])
    X, Y = X[:170], Y[:170]

    weights = 1. / np.bincount(Y)
    weights *= len(weights) / np.sum(weights)
    pbl_class_weight = MultiClassClf(n_features=3, n_classes=3,
                                     class_weight=weights, rescale_C=True)
    svm_class_weight = OneSlackSSVM(pbl_class_weight, C=10, tol=1e-5)
    svm_class_weight.fit(X, Y)

    try:
        linearsvm = LinearSVC(multi_class='crammer_singer',
                              fit_intercept=False, class_weight='auto', C=10)
        linearsvm.fit(X, Y)

        assert_array_almost_equal(svm_class_weight.w, linearsvm.coef_.ravel(),
                                  3)
    except TypeError:
        # travis has a really old sklearn version that doesn't support
        # class_weight in LinearSVC
        pass

def test_binary_blocks_one_slack_graph():
    #testing cutting plane ssvm on easy binary dataset
    # generate graphs explicitly for each example
    for inference_method in ["dai", "lp"]:
        print("testing %s" % inference_method)
        X, Y = toy.generate_blocks(n_samples=3)
        crf = GraphCRF(inference_method=inference_method)
        clf = OneSlackSSVM(problem=crf, max_iter=100, C=100, verbose=100,
                           check_constraints=True, break_on_bad=True,
                           n_jobs=1)
        x1, x2, x3 = X
        y1, y2, y3 = Y
        n_states = len(np.unique(Y))
        # delete some rows to make it more fun
        x1, y1 = x1[:, :-1], y1[:, :-1]
        x2, y2 = x2[:-1], y2[:-1]
        # generate graphs
        X_ = [x1, x2, x3]
        G = [make_grid_edges(x) for x in X_]

        # reshape / flatten x and y
        X_ = [x.reshape(-1, n_states) for x in X_]
        Y = [y.ravel() for y in [y1, y2, y3]]

        X = zip(X_, G)

        clf.fit(X, Y)
        Y_pred = clf.predict(X)
        for y, y_pred in zip(Y, Y_pred):
            assert_array_equal(y, y_pred)

def test_latent_node_boxes_edge_features():
    # learn the "easy" 2x2 boxes dataset.
    # smoketest using a single constant edge feature

    X, Y = make_simple_2x2(seed=1, n_samples=40)
    latent_crf = EdgeFeatureLatentNodeCRF(n_labels=2, n_hidden_states=2, n_features=1)
    base_svm = OneSlackSSVM(latent_crf)
    base_svm.C = 10
    latent_svm = LatentSSVM(base_svm,
                            latent_iter=10)

    G = [make_grid_edges(x) for x in X]

    # make edges for hidden states:
    edges = make_edges_2x2()

    G = [np.vstack([make_grid_edges(x), edges]) for x in X]

    # reshape / flatten x and y
    X_flat = [x.reshape(-1, 1) for x in X]
    Y_flat = [y.ravel() for y in Y]

    #X_ = zip(X_flat, G, [2 * 2 for x in X_flat])
    # add edge features
    X_ = [(x, g, np.ones((len(g), 1)), 4) for x, g in zip(X_flat, G)]
    latent_svm.fit(X_[:20], Y_flat[:20])

    assert_array_equal(latent_svm.predict(X_[:20]), Y_flat[:20])
    assert_equal(latent_svm.score(X_[:20], Y_flat[:20]), 1)

    # test that score is not always 1
    assert_true(.98 < latent_svm.score(X_[20:], Y_flat[20:]) < 1)

def test(nfiles):
    X = []
    Y = []
    X_tst = []
    Y_tst = []
    ntrain = nfiles
    ntest = 5*nfiles
    print("Training/testing with %d/%d files." % (ntrain,ntest))
    start = time.clock()   
 
    filename = '../maps/MAPS_AkPnCGdD_2/AkPnCGdD/MUS'
    #filename = '../maps/MAPS_AkPnCGdD_1/AkPnCGdD/ISOL/NO'
    files = dirs.get_files_with_extension(filename, '.mid')
    train_files = files[:ntrain]
    print("\t" + str(train_files))
    #test_files = files[ntrain:ntest+ntrain]
    # for legit testing
    test_files = files[-ntest:]
    map(per_file, train_files,
      it.repeat(X, ntrain), it.repeat(Y, ntrain))
    map(per_file, test_files,
      it.repeat(X_tst, ntest), it.repeat(Y_tst, ntest))

    end = time.clock()
    print("\tRead time: %f" % (end - start))
    print("\tnWindows train: " + str([X[i].shape[0] for i in range(len(X))]))
    start = time.clock()   

    crf = ChainCRF(n_states=2)
    clf = OneSlackSSVM(model=crf, C=100, n_jobs=-1,
          inference_cache=100, tol=.1)
    clf.fit(np.array(X), np.array(Y))

    end = time.clock()
    print("\tTrain time: %f" % (end - start))
    start = time.clock()   

    Y_pred = clf.predict(X_tst)
    comp = []
    for i in range(len(Y_tst)):
      for j in range(len(Y_tst[i])):
        comp.append((Y_tst[i][j], Y_pred[i][j]))
        print Y_tst[i][j],
      print
      for j in range(len(Y_tst[i])):
        print Y_pred[i][j],
      print
      print

    print("\tTrue positives: %d" % comp.count((1,1)))
    print("\tTrue negatives: %d" % comp.count((0,0)))
    print("\tFalse positives: %d" % comp.count((0,1)))
    print("\tFalse negatives: %d" % comp.count((1,0)))
    

    end = time.clock()
    print("\tTest time: %f" % (end - start))

def test_constraint_removal():
    digits = load_digits()
    X, y = digits.data, digits.target
    y = 2 * (y % 2) - 1  # even vs odd as +1 vs -1
    X = X / 16.
    pbl = BinaryClf(n_features=X.shape[1])
    clf_no_removal = OneSlackSSVM(model=pbl, max_iter=500, C=1,
                                  inactive_window=0, tol=0.01)
    clf_no_removal.fit(X, y)
    clf = OneSlackSSVM(model=pbl, max_iter=500, C=1, tol=0.01,
                       inactive_threshold=1e-8)
    clf.fit(X, y)
    # check that we learned something
    assert_greater(clf.score(X, y), .92)

    # results are mostly equal
    # if we decrease tol, they will get more similar
    assert_less(np.mean(clf.predict(X) != clf_no_removal.predict(X)), 0.02)

    # without removal, have as many constraints as iterations
    assert_equal(len(clf_no_removal.objective_curve_),
                 len(clf_no_removal.constraints_))

    # with removal, there are less constraints than iterations
    assert_less(len(clf.constraints_),
                len(clf.objective_curve_))

def test_multinomial_blocks_one_slack():
    #testing cutting plane ssvm on easy multinomial dataset
    X, Y = generate_blocks_multinomial(n_samples=10, noise=0.5, seed=0)
    n_labels = len(np.unique(Y))
    crf = GridCRF(n_states=n_labels, inference_method=inference_method)
    clf = OneSlackSSVM(model=crf, max_iter=150, C=1,
                       check_constraints=True, break_on_bad=True, tol=.1,
                       inference_cache=50)
    clf.fit(X, Y)
    Y_pred = clf.predict(X)
    assert_array_equal(Y, Y_pred)

def test_one_slack_attractive_potentials():
    # test that submodular SSVM can learn the block dataset
    X, Y = generate_blocks(n_samples=10)
    crf = GridCRF(inference_method=inference_method)
    submodular_clf = OneSlackSSVM(
        model=crf, max_iter=200, C=1, check_constraints=True, negativity_constraint=[5], inference_cache=50
    )
    submodular_clf.fit(X, Y)
    Y_pred = submodular_clf.predict(X)
    assert_array_equal(Y, Y_pred)
    assert_true(submodular_clf.w[5] < 0)

def test_one_slack_attractive_potentials():
    # test that submodular SSVM can learn the block dataset
    X, Y = toy.generate_blocks(n_samples=10)
    crf = GridCRF()
    submodular_clf = OneSlackSSVM(model=crf, max_iter=200, C=100, verbose=1,
                                  check_constraints=True,
                                  positive_constraint=[5], n_jobs=-1)
    submodular_clf.fit(X, Y)
    Y_pred = submodular_clf.predict(X)
    assert_array_equal(Y, Y_pred)
    assert_true(submodular_clf.w[5] < 0)  # don't ask me about signs

def syntetic_train_score_per_iter(result, only_weak=False, plot=True):
    w_history = result.data["w_history"]
    meta_data = result.meta
    n_full = meta_data["n_full"]
    n_train = meta_data["n_train"]
    n_inference_iter = meta_data["n_inference_iter"]
    n_full = meta_data["n_full"]
    n_train = meta_data["n_train"]
    dataset = meta_data["dataset"]
    C = meta_data["C"]
    latent_iter = meta_data["latent_iter"]
    max_iter = meta_data["max_iter"]
    inner_tol = meta_data["inner_tol"]
    outer_tol = meta_data["outer_tol"]
    alpha = meta_data["alpha"]
    min_changes = meta_data["min_changes"]
    initialize = meta_data["initialize"]

    crf = HCRF(
        n_states=10, n_features=10, n_edge_features=2, alpha=alpha, inference_method="gco", n_iter=n_inference_iter
    )
    base_clf = OneSlackSSVM(crf, max_iter=max_iter, C=C, verbose=0, tol=inner_tol, n_jobs=4, inference_cache=100)
    clf = LatentSSVM(base_clf, latent_iter=latent_iter, verbose=2, tol=outer_tol, min_changes=min_changes, n_jobs=4)

    X, Y = load_syntetic(dataset)

    Xtrain, Ytrain, Ytrain_full, Xtest, Ytest = split_test_train(X, Y, n_full, n_train)

    if only_weak:
        Xtrain = [x for (i, x) in enumerate(Xtrain) if not Ytrain[i].full_labeled]
        Ytrain_full = [y for (i, y) in enumerate(Ytrain_full) if not Ytrain[i].full_labeled]

    base_clf.w = None
    clf.w_history_ = w_history
    clf.iter_done = w_history.shape[0]

    train_scores = []
    for score in clf.staged_score(Xtrain, Ytrain_full):
        train_scores.append(score)
    train_scores = np.array(train_scores)

    if plot:
        x = np.arange(0, train_scores.size)
        pl.rc("text", usetex=True)
        pl.rc("font", family="serif")
        pl.figure(figsize=(10, 10), dpi=96)
        pl.title("score on train set")
        pl.plot(x, train_scores)
        pl.scatter(x, train_scores)
        pl.xlabel("iteration")
        pl.xlim([-0.5, train_scores.size + 1])

    return train_scores

def test_multinomial_blocks_one_slack():
    #testing cutting plane ssvm on easy multinomial dataset
    X, Y = toy.generate_blocks_multinomial(n_samples=10, noise=0.3,
                                           seed=0)
    n_labels = len(np.unique(Y))
    for inference_method in ['lp']:
        crf = GridCRF(n_states=n_labels, inference_method=inference_method)
        clf = OneSlackSSVM(problem=crf, max_iter=50, C=100, verbose=100,
                           check_constraints=True, break_on_bad=True)
        clf.fit(X, Y)
        Y_pred = clf.predict(X)
        assert_array_equal(Y, Y_pred)

def test_multilabel_yeast_independent():
    yeast = fetch_mldata("yeast")
    X = yeast.data
    y = yeast.target.toarray().T.astype(np.int)
    # no edges for the moment
    edges = np.zeros((0, 2), dtype=np.int)
    pbl = MultiLabelProblem(n_features=X.shape[1], n_labels=y.shape[1],
                            edges=edges)
    ssvm = OneSlackSSVM(pbl, verbose=10)
    ssvm.fit(X, y)
    from IPython.core.debugger import Tracer
    Tracer()()

def test_blobs_2d_one_slack():
    # make two gaussian blobs
    X, Y = make_blobs(n_samples=80, centers=3, random_state=42)
    # we have to add a constant 1 feature by hand :-/
    X = np.hstack([X, np.ones((X.shape[0], 1))])

    X_train, X_test, Y_train, Y_test = X[:40], X[40:], Y[:40], Y[40:]

    pbl = MultiClassClf(n_features=3, n_classes=3)
    svm = OneSlackSSVM(pbl, check_constraints=True, C=1000)

    svm.fit(X_train, Y_train)
    assert_array_equal(Y_test, np.hstack(svm.predict(X_test)))

def test_blobs_2d_one_slack():
    # make two gaussian blobs
    X, Y = make_blobs(n_samples=80, centers=2, random_state=1)
    Y = 2 * Y - 1
    # we have to add a constant 1 feature by hand :-/
    X = np.hstack([X, np.ones((X.shape[0], 1))])
    X_train, X_test, Y_train, Y_test = X[:40], X[40:], Y[:40], Y[40:]

    pbl = BinarySVMModel(n_features=3)
    svm = OneSlackSSVM(pbl, verbose=30, C=1000)

    svm.fit(X_train, Y_train)
    assert_array_equal(Y_test, np.hstack(svm.predict(X_test)))

def test_standard_svm_blobs_2d_class_weight():
    # no edges, reduce to crammer-singer svm
    X, Y = make_blobs(n_samples=210, centers=3, random_state=1, cluster_std=3,
                      shuffle=False)
    X = np.hstack([X, np.ones((X.shape[0], 1))])
    X, Y = X[:170], Y[:170]

    X_graphs = [(x[np.newaxis, :], np.empty((0, 2), dtype=np.int)) for x in X]

    pbl = GraphCRF(n_features=3, n_states=3, inference_method='unary')
    svm = OneSlackSSVM(pbl, check_constraints=False, C=1000)

    svm.fit(X_graphs, Y[:, np.newaxis])

    weights = 1. / np.bincount(Y)
    weights *= len(weights) / np.sum(weights)

    pbl_class_weight = GraphCRF(n_features=3, n_states=3, class_weight=weights,
                                inference_method='unary')
    svm_class_weight = OneSlackSSVM(pbl_class_weight, C=10,
                                    check_constraints=False,
                                    break_on_bad=False)
    svm_class_weight.fit(X_graphs, Y[:, np.newaxis])

    assert_greater(f1_score(Y, np.hstack(svm_class_weight.predict(X_graphs))),
                   f1_score(Y, np.hstack(svm.predict(X_graphs))))

def test_standard_svm_blobs_2d():
    # no edges, reduce to crammer-singer svm
    X, Y = make_blobs(n_samples=80, centers=3, random_state=42)
    # we have to add a constant 1 feature by hand :-/
    X = np.hstack([X, np.ones((X.shape[0], 1))])

    X_train, X_test, Y_train, Y_test = X[:40], X[40:], Y[:40], Y[40:]
    X_train_graphs = [(x[np.newaxis, :], np.empty((0, 2), dtype=np.int)) for x in X_train]

    X_test_graphs = [(x[np.newaxis, :], np.empty((0, 2), dtype=np.int)) for x in X_test]

    pbl = GraphCRF(n_features=3, n_states=3)
    svm = OneSlackSSVM(pbl, verbose=10, check_constraints=True, C=1000)

    svm.fit(X_train_graphs, Y_train[:, np.newaxis])
    assert_array_equal(Y_test, np.hstack(svm.predict(X_test_graphs)))

def main():
    print("Please be patient. Will take 5-20 minutes.")
    snakes = load_snakes()
    X_train, Y_train = snakes['X_train'], snakes['Y_train']

    X_train = [one_hot_colors(x) for x in X_train]
    Y_train_flat = [y_.ravel() for y_ in Y_train]

    X_train_directions, X_train_edge_features = prepare_data(X_train)

    # first, train on X with directions only:
    crf = EdgeFeatureGraphCRF(inference_method='qpbo')
    ssvm = OneSlackSSVM(crf, inference_cache=50, C=.1, tol=.1, switch_to='ad3',
                        n_jobs=-1)
    ssvm.fit(X_train_directions, Y_train_flat)

    # Evaluate using confusion matrix.
    # Clearly the middel of the snake is the hardest part.
    X_test, Y_test = snakes['X_test'], snakes['Y_test']
    X_test = [one_hot_colors(x) for x in X_test]
    Y_test_flat = [y_.ravel() for y_ in Y_test]
    X_test_directions, X_test_edge_features = prepare_data(X_test)
    Y_pred = ssvm.predict(X_test_directions)
    print("Results using only directional features for edges")
    print("Test accuracy: %.3f"
          % accuracy_score(np.hstack(Y_test_flat), np.hstack(Y_pred)))
    print(confusion_matrix(np.hstack(Y_test_flat), np.hstack(Y_pred)))

    # now, use more informative edge features:
    crf = EdgeFeatureGraphCRF(inference_method='qpbo')
    ssvm = OneSlackSSVM(crf, inference_cache=50, C=.1, tol=.1, switch_to='ad3',
                        n_jobs=-1)
    ssvm.fit(X_train_edge_features, Y_train_flat)
    Y_pred2 = ssvm.predict(X_test_edge_features)
    print("Results using also input features for edges")
    print("Test accuracy: %.3f"
          % accuracy_score(np.hstack(Y_test_flat), np.hstack(Y_pred2)))
    print(confusion_matrix(np.hstack(Y_test_flat), np.hstack(Y_pred2)))

    # plot stuff
    fig, axes = plt.subplots(2, 2)
    axes[0, 0].imshow(snakes['X_test'][0], interpolation='nearest')
    axes[0, 0].set_title('Input')
    y = Y_test[0].astype(np.int)
    bg = 2 * (y != 0)  # enhance contrast
    axes[0, 1].matshow(y + bg, cmap=plt.cm.Greys)
    axes[0, 1].set_title("Ground Truth")
    axes[1, 0].matshow(Y_pred[0].reshape(y.shape) + bg, cmap=plt.cm.Greys)
    axes[1, 0].set_title("Prediction w/o edge features")
    axes[1, 1].matshow(Y_pred2[0].reshape(y.shape) + bg, cmap=plt.cm.Greys)
    axes[1, 1].set_title("Prediction with edge features")
    for a in axes.ravel():
        a.set_xticks(())
        a.set_yticks(())
    plt.show()
    from IPython.core.debugger import Tracer
    Tracer()()

def test_svm_as_crf_pickling():
    iris = load_iris()
    X, y = iris.data, iris.target

    X_ = [(np.atleast_2d(x), np.empty((0, 2), dtype=np.int)) for x in X]
    Y = y.reshape(-1, 1)

    X_train, X_test, y_train, y_test = train_test_split(X_, Y, random_state=1)
    _, file_name = mkstemp()

    pbl = GraphCRF(n_features=4, n_states=3, inference_method="unary")
    logger = SaveLogger(file_name)
    svm = OneSlackSSVM(pbl, check_constraints=True, C=1, n_jobs=1, logger=logger)
    svm.fit(X_train, y_train)

    assert_less(0.97, svm.score(X_test, y_test))
    assert_less(0.97, logger.load().score(X_test, y_test))