def test_quadratic_weighted_kappa(self):
        kappa = metrics.quadratic_weighted_kappa([1,2,3],[1,2,3])
        self.assertAlmostEqual(kappa, 1.0)

        kappa = metrics.quadratic_weighted_kappa([1,2,1],[1,2,2],1,2)
        self.assertAlmostEqual(kappa, 0.4)

        kappa = metrics.quadratic_weighted_kappa([1,2,3,1,2,2,3],[1,2,3,1,2,3,2])
        self.assertAlmostEqual(kappa, 0.75)

    def predict(self, model, xg_train, xg_test, objective='reg:linear'):
        """
        Parameters
        ----------

        model : xgboost.Booster
            xgboost model ready for making predictions

        xg_train : xgboost.DMatrix
            training data

        xg_test : xgboost.DMatrix
            testing data


        Returns
        -------

        model_prediction : ModelPrediction (named tuple)

        """

        train_score = model.predict(
            xg_train, ntree_limit=model.best_iteration)
        test_score = model.predict(
            xg_test,  ntree_limit=model.best_iteration)

        train_label = np.asarray(xg_train.get_label())
        test_label = np.asarray(xg_test.get_label())

        if objective == 'reg:linear':
            # Cuttofs are optimized here
            best_cuts = optimize_cutoffs(train_score, train_label,
                                         verbose=False)
            train_prediction = classify_with_cutoffs(train_score, best_cuts)
            test_prediction = classify_with_cutoffs(test_score, best_cuts)
        else:
            train_prediction = train_score
            test_prediction = test_score

        train_qwk = quadratic_weighted_kappa(train_label, train_prediction)
        test_qwk = quadratic_weighted_kappa(test_label, test_prediction)

        return ModelPrediction(train_label, test_label,
                               train_score, test_score,
                               train_prediction, test_prediction,
                               train_qwk, test_qwk,
                               precision_score(train_label, train_prediction,
                                               average=None),
                               precision_score(test_label, test_prediction,
                                               average=None)
                               )

def minimize_quadratic_weighted_kappa(cutpoints,y_pred=None,y=None):
    cutpoints = np.sort(cutpoints)
    cutpoints = np.concatenate([[-99999999999999999],cutpoints,[999999999999999]])
    y_pred = pd.cut(y_pred,bins=cutpoints,labels=[1,2,3,4,5,6,7,8])
    score = quadratic_weighted_kappa(y,y_pred)
    print score
    return -score

def predict_score():
	file = open("model/predictions.txt")
	numarray = []
	while 1:
		line = file.readline()
		if not line:
			break
		numarray.append(int(float(line)))
	file = open("model/answers.txt")
	answerarray = []
	while 1:
		line = file.readline()
		if not line:
			break
		answerarray.append(int(float(line)))
	##print len(numarray)
	##print len(answerarray)
	solutionarray = []
	for x in range(0, len(numarray)):
		if numarray[x] == answerarray[x]:
			solutionarray.append(1)
		else:
			solutionarray.append(0)
	onecounter = solutionarray.count(1)
	print "QWK_Score: " + str(metrics.quadratic_weighted_kappa(answerarray,numarray))

def eval_dag(dag, filename, dag_id=None):

    dag = normalize_dag(dag)

    if filename not in input_cache:
        input_cache[filename] = pd.read_csv('data/'+filename, sep=';')

    data = input_cache[filename]

    feats = data[data.columns[:-1]]
    targets = data[data.columns[-1]]

    le = preprocessing.LabelEncoder()

    ix = targets.index
    targets = pd.Series(le.fit_transform(targets), index=ix)

    errors = []

    start_time = time.time()

    for train_idx, test_idx in cross_validation.StratifiedKFold(targets, n_folds=5):
        train_data = (feats.iloc[train_idx], targets.iloc[train_idx])
        test_data = (feats.iloc[test_idx], targets.iloc[test_idx])

        ms = train_dag(dag, train_data)
        preds = test_dag(dag, ms, test_data)

        acc = mm.quadratic_weighted_kappa(test_data[1], preds)
        errors.append(acc)

    m_errors = float(np.mean(errors))
    s_errors = float(np.std(errors))

    return m_errors, s_errors, time.time() - start_time

    def _score_offset(self, bin_offset, sv):
        flg = self._data[:, 0].astype(int) == sv
        self._data[flg, 1] = self._data[flg, 0] + bin_offset
        offset_pred = np.clip(np.round(self._data[:, 1]), 1, 8)\
            .astype(int)
        kappa = quadratic_weighted_kappa(self._data[:, 2], offset_pred)

        return -kappa

def evalerror_softmax_cdf(preds, dtrain, cdf):
    ## label are in [0,1,2,3]
    labels = dtrain.get_label() + 1
    preds = getClfScore(preds, cdf)
    kappa = quadratic_weighted_kappa(labels, preds)
    ## we return -kappa for using early stopping
    kappa *= -1.
    return 'kappa', float(kappa)

def keras_model():

    import pandas as pd
    import numpy as np

    from keras.preprocessing import sequence
    from keras.models import Sequential
    from keras.layers.core import Dense, Dropout, Activation, Flatten
    from keras.layers.convolutional import Convolution1D, MaxPooling1D
    from keras.callbacks import EarlyStopping
    from keras.utils import np_utils

    from data_util import load_csvs, load_other
    import ml_metrics as metrics

    nb_words = 6500
    maxlen = 175
    filter_length = 10
    other_col_dim = 4

    X_train, Y_train, X_test, Y_test, nb_classes = load_csvs('data/tpov4/train_1.csv',
                                                             'data/tpov4/test_1.csv',
                                                              nb_words, maxlen, 'self', w2v=None)

    # read _other.csv
    other_train = load_other('data/tpov4/train_1_other.csv', maxlen, other_col_dim)
    other_test = load_other('data/tpov4/test_1_other.csv', maxlen, other_col_dim)

    print('other tensor:', other_train.shape)

    pool_length = maxlen - filter_length + 1

    model = Sequential()
    model.add(Convolution1D(nb_filter=50,
                            filter_length=filter_length,
                            border_mode="valid", activation="relu",
                            input_shape=(maxlen, other_col_dim)))
    model.add(MaxPooling1D(pool_length=pool_length))
    model.add(Flatten())
    model.add(Dropout(0.05))
    model.add(Dense(nb_classes))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy',
                  optimizer={{choice(['rmsprop', 'adam', 'adadelta', 'adagrad'])}})

    earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)

    model.fit(other_train, Y_train, batch_size=32, nb_epoch=25,
              validation_split=0.1, show_accuracy=True, callbacks=[earlystop])

    classes = earlystop.model.predict_classes(other_test, batch_size=32)
    org_classes = np_utils.categorical_probas_to_classes(Y_test)

    acc = np_utils.accuracy(classes, org_classes)  # accuracy only supports classes
    print('Test accuracy:', acc)
    kappa = metrics.quadratic_weighted_kappa(classes, org_classes)
    print('Test Kappa:', kappa)
    return {'loss': -acc, 'status': STATUS_OK}

def eval_wrapper(yhat, y):
    """
    Evaluation metric for the competition : quad weighted kappa
    """  
    y = np.array(y)
    y = y.astype(int)
    yhat = np.array(yhat)
    yhat = np.clip(np.round(yhat), np.min(y), np.max(y)).astype(int)   
    return quadratic_weighted_kappa(yhat, y)

def xgb_regression_quadratic_weighted_kappa(preds,dtrain):
    labels = dtrain.get_label()
    cutpoints = [1.886638,3.303624,4.152756,4.825063,5.653934,6.236325,6.765184]  
    res = minimize(minimize_quadratic_weighted_kappa,cutpoints,(preds,labels),method='BFGS')
    cutpoints = np.sort(res.x)
    cutpoints = np.concatenate([[-99999999999999999],cutpoints,[999999999999999]])
    y_pred = pd.cut(preds,bins=cutpoints,labels=[1,2,3,4,5,6,7,8])
    kappa = quadratic_weighted_kappa(labels,y_pred)
    ## we return -kappa for using early stopping
    kappa *= -1.
    return 'kappa', float(kappa)

def evalerror_cocr_cdf(preds, dtrain, cdf):
    labels = dtrain.get_label() + 1
    #print preds.shape
    ## get prediction
    #preds = sigmoid(preds)
    preds = applyCOCRRule(preds)
    preds = getScore(preds, cdf)
    kappa = quadratic_weighted_kappa(labels, preds)
    ## we return -kappa for using early stopping
    kappa *= -1.
    return 'kappa', float(kappa)

def evalerror(preds, dtrain):
    ## label are in [0,1,2,3] as required by XGBoost for multi-classification
    labels = dtrain.get_label() + 1
    ## class probability
    preds = softmax(preds)
    ## decoding (naive argmax decoding)
    pred_labels = np.argmax(preds, axis=1) + 1
    ## compute quadratic weighted kappa (using implementation from @Ben Hamner
    ## https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/quadratic_weighted_kappa.py
    kappa = quadratic_weighted_kappa(labels, pred_labels)
    return 'kappa', kappa

    def _offset_qwk_score(self, offset):
        """

        :param numpy.array offset:
        :param numpy.array y_true:
        :param numpy.array y_pred:
        :rtype: float
        """
        offset_pred = self._apply_offset(self._data, offset)
        kappa = quadratic_weighted_kappa(self._data[:, 2], offset_pred)

        return -kappa

def cnn1d_selfembd(X_train, Y_train, X_test, Y_test, nb_classes,
                   maxlen, vocab_size, embd_dim,
                   nb_filter, filter_length, batch_size, nb_epoch, optm):
    """
    - CNN-1d on text input (represented in int)
    - MOT
    - dropout + L2 softmax

    :param <X, Y> train and test sets
    :param nb_classes # of classes
    :param maxlen max of n char in a sentence
    :param vocab_size
    :param embd_dim
    :param nb_filter
    :param filter_length
    :param batch_size
    :param nb_epoch
    :param optm optimizer options, e.g., adam, rmsprop, etc.
    :return:
    """
    pool_length = maxlen - filter_length + 1

    model = Sequential()
    model.add(Embedding(vocab_size, embd_dim, input_length=maxlen))
    model.add(Dropout(0.25))

    model.add(Convolution1D(nb_filter=nb_filter,
                            filter_length=filter_length,
                            border_mode="valid",
                            activation="relu"))
    model.add(MaxPooling1D(pool_length=pool_length))

    model.add(Flatten())
    model.add(Dropout(0.5))

    model.add(Dense(nb_classes))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer=optm)

    earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)

    model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,
              validation_split=0.1, show_accuracy=True, callbacks=[earlystop])

    classes = earlystop.model.predict_classes(X_test, batch_size=batch_size)
    acc = np_utils.accuracy(classes, np_utils.categorical_probas_to_classes(Y_test))
    print('Test accuracy:', acc)
    # return(acc)
    kappa = metrics.quadratic_weighted_kappa(classes, np_utils.categorical_probas_to_classes(Y_test))
    print('Test Kappa:', kappa)
    return (kappa)

    def on_epoch_end(self, epoch, logs={}):
        p = self.model.predict(self.X_val.values, verbose=0)
        current = ml_metrics.quadratic_weighted_kappa(self.y_val.values.ravel(),np.clip(np.round(p.astype(int).ravel()), 1, 8))

        if current > self.best:
            self.best = current
            self.wait = 0
        else:
            if self.wait >= self.patience:
                self.model.stop_training = True
                print('Epoch %05d: early stopping' % (epoch))

            self.wait += 1 #incremental the number of times without improvement
        print('Epoch %d Kappa: %f | Best Kappa: %f \n' % (epoch,current,self.best))

    def default_errorfun(p, ysc, ytr):
        """
        Parameters
        ----------

        p : array of 8 cutoff values

        ysc : array of scores [array(double)]

        ytr : array of true labels [array(int)]
        """
        errors = quadratic_weighted_kappa(
            classify_with_cutoffs(ysc, p).astype(np.int64), ytr)
        return 1 - errors

def ensembleSelectionObj(param, p1_list, weight1, p2_list, true_label_list, cdf_list, numValidMatrix):

    weight2 = param['weight2']
    kappa_cv = np.zeros((config.n_runs, config.n_folds), dtype=float)
    for run in range(config.n_runs):
        for fold in range(config.n_folds):
            numValid = numValidMatrix[run][fold]
            p1 = p1_list[run,fold,:numValid]
            p2 = p2_list[run,fold,:numValid]
            true_label = true_label_list[run,fold,:numValid]
            cdf = cdf_list[run,fold,:]
            p_ens = (weight1 * p1 + weight2 * p2) / (weight1 + weight2)
            p_ens_score = getScore(p_ens, cdf)
            kappa_cv[run][fold] = quadratic_weighted_kappa(p_ens_score, true_label)
    kappa_cv_mean = np.mean(kappa_cv)
    return {'loss': -kappa_cv_mean, 'status': STATUS_OK}

def cnn1d_w2vembd(X_train, Y_train, X_test, Y_test, nb_classes,
                  maxlen,
                  nb_filter, filter_length, batch_size, nb_epoch, optm):
    """
    - CNN-1d on 3d sensor which uses word2vec embedding
    - MOT

    :param <X, Y> train and test sets
    :param nb_classes # of classes
    :param maxlen max of n char in a sentence
    :param nb_filter
    :param filter_length
    :param batch_size
    :param nb_epoch
    :param optm
    :return:
    """
    pool_length = maxlen - filter_length + 1

    model = Sequential()

    model.add(Convolution1D(nb_filter=nb_filter,
                            filter_length=filter_length,
                            border_mode="valid",
                            activation="relu", input_shape=(maxlen, 300)))
    model.add(MaxPooling1D(pool_length=pool_length))
    model.add(Flatten())
    model.add(Dropout(0.5))
    model.add(Dense(nb_classes))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer=optm)

    earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)

    model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,
              validation_split=0.1, show_accuracy=True, callbacks=[earlystop])

    classes = earlystop.model.predict_classes(X_test, batch_size=batch_size)
    acc = np_utils.accuracy(classes, np_utils.categorical_probas_to_classes(Y_test))  # accuracy only supports classes
    print('Test accuracy:', acc)
    # return(acc)
    kappa = metrics.quadratic_weighted_kappa(classes, np_utils.categorical_probas_to_classes(Y_test))
    print('Test Kappa:', kappa)
    return (kappa)

def lstm_selfembd(X_train, Y_train, X_test, Y_test, nb_classes,
                  maxlen, vocab_size, embd_dim,
                  batch_size, nb_epoch, optm):
    """
    - LSTM  on text input (represented in int)
    - fully-connected model

    :param <X, Y> train and test sets
    :param nb_classes # of classes
    :param maxlen max of n char in a sentence
    :param vocab_size
    :param embd_dim
    :param batch_size
    :param nb_epoch
    :param optm optimizer options, e.g., adam, rmsprop, etc.
    :return:
    """

    model = Sequential()
    model.add(Embedding(vocab_size, embd_dim, input_length=maxlen))
    model.add(Dropout(0.25))

    # model.add(LSTM(100, return_sequences=True))
    model.add(LSTM(50))

    model.add(Flatten())
    model.add(Dropout(0.5))
    model.add(Dense(nb_classes))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer=optm)

    earlystop = EarlyStopping(monitor='val_loss', patience=2, verbose=1)

    model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,
              validation_split=0.1, show_accuracy=True, callbacks=[earlystop])

    classes = earlystop.model.predict_classes(X_test, batch_size=batch_size)
    acc = np_utils.accuracy(classes, np_utils.categorical_probas_to_classes(Y_test))  # accuracy only supports classes
    print('Test accuracy:', acc)
    kappa = metrics.quadratic_weighted_kappa(classes, np_utils.categorical_probas_to_classes(Y_test))
    print('Test Kappa:', kappa)
    return (kappa)

def cnn_var_selfembd(X_train, Y_train, X_test, Y_test, nb_classes,
                     maxlen, vocab_size, embd_size,
                     nb_filter, batch_size, nb_epoches, optm):
    ngram_filters = [2, 5, 8]

    input = Input(shape=(maxlen,), name='input', dtype='int32')
    embedded = Embedding(input_dim=vocab_size, output_dim=embd_size, input_length=maxlen)(input)

    convs = [None, None, None]
    # three CNNs
    for i, n_gram in enumerate(ngram_filters):
        pool_length = maxlen - n_gram + 1
        convs[i] = Convolution1D(nb_filter=nb_filter,
                                 filter_length=n_gram,
                                 border_mode="valid",
                                 activation="relu")(embedded)
        convs[i] = MaxPooling1D(pool_length=pool_length)(convs[i])
        convs[i] = Flatten()(convs[i])

    merged = merge([convs[0], convs[1], convs[2]], mode='concat', concat_axis=1)
    merged = Dropout(0.5)(merged)
    output = Dense(nb_classes, activation='softmax', name='output')(merged)

    model = Model(input, output)
    model.compile(optm, loss={'output': 'categorical_crossentropy'})
    earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)
    model.fit(X_train, Y_train,
              nb_epoch=nb_epoches, batch_size=batch_size,
              validation_split=0.1, callbacks=[earlystop])

    probs = earlystop.model.predict(X_test, batch_size=batch_size)
    classes = np_utils.categorical_probas_to_classes(probs)

    acc = np_utils.accuracy(classes,
                            np_utils.categorical_probas_to_classes(Y_test))
    print('Test accuracy:', acc)
    kappa = metrics.quadratic_weighted_kappa(classes,
                                             np_utils.categorical_probas_to_classes(Y_test))
    print('Test Kappa:', kappa)
    return acc