本文整理汇总了Python中ml_metrics.quadratic_weighted_kappa函数的典型用法代码示例。如果您正苦于以下问题:Python quadratic_weighted_kappa函数的具体用法?Python quadratic_weighted_kappa怎么用?Python quadratic_weighted_kappa使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了quadratic_weighted_kappa函数的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Python代码示例。
示例1: test_quadratic_weighted_kappa
def test_quadratic_weighted_kappa(self):
kappa = metrics.quadratic_weighted_kappa([1,2,3],[1,2,3])
self.assertAlmostEqual(kappa, 1.0)
kappa = metrics.quadratic_weighted_kappa([1,2,1],[1,2,2],1,2)
self.assertAlmostEqual(kappa, 0.4)
kappa = metrics.quadratic_weighted_kappa([1,2,3,1,2,2,3],[1,2,3,1,2,3,2])
self.assertAlmostEqual(kappa, 0.75)
开发者ID:9G1IC,项目名称:Metrics,代码行数:9,代码来源:test_kappa.py
示例2: predict
def predict(self, model, xg_train, xg_test, objective='reg:linear'):
"""
Parameters
----------
model : xgboost.Booster
xgboost model ready for making predictions
xg_train : xgboost.DMatrix
training data
xg_test : xgboost.DMatrix
testing data
Returns
-------
model_prediction : ModelPrediction (named tuple)
"""
train_score = model.predict(
xg_train, ntree_limit=model.best_iteration)
test_score = model.predict(
xg_test, ntree_limit=model.best_iteration)
train_label = np.asarray(xg_train.get_label())
test_label = np.asarray(xg_test.get_label())
if objective == 'reg:linear':
# Cuttofs are optimized here
best_cuts = optimize_cutoffs(train_score, train_label,
verbose=False)
train_prediction = classify_with_cutoffs(train_score, best_cuts)
test_prediction = classify_with_cutoffs(test_score, best_cuts)
else:
train_prediction = train_score
test_prediction = test_score
train_qwk = quadratic_weighted_kappa(train_label, train_prediction)
test_qwk = quadratic_weighted_kappa(test_label, test_prediction)
return ModelPrediction(train_label, test_label,
train_score, test_score,
train_prediction, test_prediction,
train_qwk, test_qwk,
precision_score(train_label, train_prediction,
average=None),
precision_score(test_label, test_prediction,
average=None)
)
开发者ID:PedroMDuarte,项目名称:kaggle-prudential-201512,代码行数:52,代码来源:xgboostmodel.py
示例3: minimize_quadratic_weighted_kappa
def minimize_quadratic_weighted_kappa(cutpoints,y_pred=None,y=None):
cutpoints = np.sort(cutpoints)
cutpoints = np.concatenate([[-99999999999999999],cutpoints,[999999999999999]])
y_pred = pd.cut(y_pred,bins=cutpoints,labels=[1,2,3,4,5,6,7,8])
score = quadratic_weighted_kappa(y,y_pred)
print score
return -score
开发者ID:wawltor,项目名称:Preudential,代码行数:7,代码来源:utils.py
示例4: predict_score
def predict_score():
file = open("model/predictions.txt")
numarray = []
while 1:
line = file.readline()
if not line:
break
numarray.append(int(float(line)))
file = open("model/answers.txt")
answerarray = []
while 1:
line = file.readline()
if not line:
break
answerarray.append(int(float(line)))
##print len(numarray)
##print len(answerarray)
solutionarray = []
for x in range(0, len(numarray)):
if numarray[x] == answerarray[x]:
solutionarray.append(1)
else:
solutionarray.append(0)
onecounter = solutionarray.count(1)
print "QWK_Score: " + str(metrics.quadratic_weighted_kappa(answerarray,numarray))
开发者ID:smartinsightsfromdata,项目名称:Autograder,代码行数:25,代码来源:estimateaccuracy.py
示例5: eval_dag
def eval_dag(dag, filename, dag_id=None):
dag = normalize_dag(dag)
if filename not in input_cache:
input_cache[filename] = pd.read_csv('data/'+filename, sep=';')
data = input_cache[filename]
feats = data[data.columns[:-1]]
targets = data[data.columns[-1]]
le = preprocessing.LabelEncoder()
ix = targets.index
targets = pd.Series(le.fit_transform(targets), index=ix)
errors = []
start_time = time.time()
for train_idx, test_idx in cross_validation.StratifiedKFold(targets, n_folds=5):
train_data = (feats.iloc[train_idx], targets.iloc[train_idx])
test_data = (feats.iloc[test_idx], targets.iloc[test_idx])
ms = train_dag(dag, train_data)
preds = test_dag(dag, ms, test_data)
acc = mm.quadratic_weighted_kappa(test_data[1], preds)
errors.append(acc)
m_errors = float(np.mean(errors))
s_errors = float(np.std(errors))
return m_errors, s_errors, time.time() - start_time
开发者ID:Undin,项目名称:dag-evaluate,代码行数:35,代码来源:eval.py
示例6: _score_offset
def _score_offset(self, bin_offset, sv):
flg = self._data[:, 0].astype(int) == sv
self._data[flg, 1] = self._data[flg, 0] + bin_offset
offset_pred = np.clip(np.round(self._data[:, 1]), 1, 8)\
.astype(int)
kappa = quadratic_weighted_kappa(self._data[:, 2], offset_pred)
return -kappa
开发者ID:haisland0909,项目名称:PrudentialLifeInsuranceAssessment,代码行数:8,代码来源:optoffset.py
示例7: evalerror_softmax_cdf
def evalerror_softmax_cdf(preds, dtrain, cdf):
## label are in [0,1,2,3]
labels = dtrain.get_label() + 1
preds = getClfScore(preds, cdf)
kappa = quadratic_weighted_kappa(labels, preds)
## we return -kappa for using early stopping
kappa *= -1.
return 'kappa', float(kappa)
开发者ID:wawltor,项目名称:Preudential,代码行数:8,代码来源:utils.py
示例8: keras_model
def keras_model():
import pandas as pd
import numpy as np
from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution1D, MaxPooling1D
from keras.callbacks import EarlyStopping
from keras.utils import np_utils
from data_util import load_csvs, load_other
import ml_metrics as metrics
nb_words = 6500
maxlen = 175
filter_length = 10
other_col_dim = 4
X_train, Y_train, X_test, Y_test, nb_classes = load_csvs('data/tpov4/train_1.csv',
'data/tpov4/test_1.csv',
nb_words, maxlen, 'self', w2v=None)
# read _other.csv
other_train = load_other('data/tpov4/train_1_other.csv', maxlen, other_col_dim)
other_test = load_other('data/tpov4/test_1_other.csv', maxlen, other_col_dim)
print('other tensor:', other_train.shape)
pool_length = maxlen - filter_length + 1
model = Sequential()
model.add(Convolution1D(nb_filter=50,
filter_length=filter_length,
border_mode="valid", activation="relu",
input_shape=(maxlen, other_col_dim)))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(Flatten())
model.add(Dropout(0.05))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer={{choice(['rmsprop', 'adam', 'adadelta', 'adagrad'])}})
earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)
model.fit(other_train, Y_train, batch_size=32, nb_epoch=25,
validation_split=0.1, show_accuracy=True, callbacks=[earlystop])
classes = earlystop.model.predict_classes(other_test, batch_size=32)
org_classes = np_utils.categorical_probas_to_classes(Y_test)
acc = np_utils.accuracy(classes, org_classes) # accuracy only supports classes
print('Test accuracy:', acc)
kappa = metrics.quadratic_weighted_kappa(classes, org_classes)
print('Test Kappa:', kappa)
return {'loss': -acc, 'status': STATUS_OK}
开发者ID:leocnj,项目名称:dl_response_rater,代码行数:58,代码来源:try_hyperas.py
示例9: eval_wrapper
def eval_wrapper(yhat, y):
"""
Evaluation metric for the competition : quad weighted kappa
"""
y = np.array(y)
y = y.astype(int)
yhat = np.array(yhat)
yhat = np.clip(np.round(yhat), np.min(y), np.max(y)).astype(int)
return quadratic_weighted_kappa(yhat, y)
开发者ID:VinACE,项目名称:Kaggle,代码行数:9,代码来源:class_utilities_lv2.py
示例10: xgb_regression_quadratic_weighted_kappa
def xgb_regression_quadratic_weighted_kappa(preds,dtrain):
labels = dtrain.get_label()
cutpoints = [1.886638,3.303624,4.152756,4.825063,5.653934,6.236325,6.765184]
res = minimize(minimize_quadratic_weighted_kappa,cutpoints,(preds,labels),method='BFGS')
cutpoints = np.sort(res.x)
cutpoints = np.concatenate([[-99999999999999999],cutpoints,[999999999999999]])
y_pred = pd.cut(preds,bins=cutpoints,labels=[1,2,3,4,5,6,7,8])
kappa = quadratic_weighted_kappa(labels,y_pred)
## we return -kappa for using early stopping
kappa *= -1.
return 'kappa', float(kappa)
开发者ID:wawltor,项目名称:Preudential,代码行数:11,代码来源:utils.py
示例11: evalerror_cocr_cdf
def evalerror_cocr_cdf(preds, dtrain, cdf):
labels = dtrain.get_label() + 1
#print preds.shape
## get prediction
#preds = sigmoid(preds)
preds = applyCOCRRule(preds)
preds = getScore(preds, cdf)
kappa = quadratic_weighted_kappa(labels, preds)
## we return -kappa for using early stopping
kappa *= -1.
return 'kappa', float(kappa)
开发者ID:wawltor,项目名称:Preudential,代码行数:11,代码来源:utils.py
示例12: evalerror
def evalerror(preds, dtrain):
## label are in [0,1,2,3] as required by XGBoost for multi-classification
labels = dtrain.get_label() + 1
## class probability
preds = softmax(preds)
## decoding (naive argmax decoding)
pred_labels = np.argmax(preds, axis=1) + 1
## compute quadratic weighted kappa (using implementation from @Ben Hamner
## https://github.com/benhamner/Metrics/blob/master/Python/ml_metrics/quadratic_weighted_kappa.py
kappa = quadratic_weighted_kappa(labels, pred_labels)
return 'kappa', kappa
开发者ID:ldamewood,项目名称:crowdflower,代码行数:11,代码来源:softkappa.py
示例13: _offset_qwk_score
def _offset_qwk_score(self, offset):
"""
:param numpy.array offset:
:param numpy.array y_true:
:param numpy.array y_pred:
:rtype: float
"""
offset_pred = self._apply_offset(self._data, offset)
kappa = quadratic_weighted_kappa(self._data[:, 2], offset_pred)
return -kappa
开发者ID:haisland0909,项目名称:PrudentialLifeInsuranceAssessment,代码行数:12,代码来源:optoffset.py
示例14: cnn1d_selfembd
def cnn1d_selfembd(X_train, Y_train, X_test, Y_test, nb_classes,
maxlen, vocab_size, embd_dim,
nb_filter, filter_length, batch_size, nb_epoch, optm):
"""
- CNN-1d on text input (represented in int)
- MOT
- dropout + L2 softmax
:param <X, Y> train and test sets
:param nb_classes # of classes
:param maxlen max of n char in a sentence
:param vocab_size
:param embd_dim
:param nb_filter
:param filter_length
:param batch_size
:param nb_epoch
:param optm optimizer options, e.g., adam, rmsprop, etc.
:return:
"""
pool_length = maxlen - filter_length + 1
model = Sequential()
model.add(Embedding(vocab_size, embd_dim, input_length=maxlen))
model.add(Dropout(0.25))
model.add(Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode="valid",
activation="relu"))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer=optm)
earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)
model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,
validation_split=0.1, show_accuracy=True, callbacks=[earlystop])
classes = earlystop.model.predict_classes(X_test, batch_size=batch_size)
acc = np_utils.accuracy(classes, np_utils.categorical_probas_to_classes(Y_test))
print('Test accuracy:', acc)
# return(acc)
kappa = metrics.quadratic_weighted_kappa(classes, np_utils.categorical_probas_to_classes(Y_test))
print('Test Kappa:', kappa)
return (kappa)
开发者ID:leocnj,项目名称:dl_response_rater,代码行数:51,代码来源:cnn1d.py
示例15: on_epoch_end
def on_epoch_end(self, epoch, logs={}):
p = self.model.predict(self.X_val.values, verbose=0)
current = ml_metrics.quadratic_weighted_kappa(self.y_val.values.ravel(),np.clip(np.round(p.astype(int).ravel()), 1, 8))
if current > self.best:
self.best = current
self.wait = 0
else:
if self.wait >= self.patience:
self.model.stop_training = True
print('Epoch %05d: early stopping' % (epoch))
self.wait += 1 #incremental the number of times without improvement
print('Epoch %d Kappa: %f | Best Kappa: %f \n' % (epoch,current,self.best))
开发者ID:computational-class,项目名称:DM-Competition-Getting-Started,代码行数:14,代码来源:Neural+Network.py
示例16: default_errorfun
def default_errorfun(p, ysc, ytr):
"""
Parameters
----------
p : array of 8 cutoff values
ysc : array of scores [array(double)]
ytr : array of true labels [array(int)]
"""
errors = quadratic_weighted_kappa(
classify_with_cutoffs(ysc, p).astype(np.int64), ytr)
return 1 - errors
开发者ID:PedroMDuarte,项目名称:kaggle-prudential-201512,代码行数:14,代码来源:xgboostmodel.py
示例17: ensembleSelectionObj
def ensembleSelectionObj(param, p1_list, weight1, p2_list, true_label_list, cdf_list, numValidMatrix):
weight2 = param['weight2']
kappa_cv = np.zeros((config.n_runs, config.n_folds), dtype=float)
for run in range(config.n_runs):
for fold in range(config.n_folds):
numValid = numValidMatrix[run][fold]
p1 = p1_list[run,fold,:numValid]
p2 = p2_list[run,fold,:numValid]
true_label = true_label_list[run,fold,:numValid]
cdf = cdf_list[run,fold,:]
p_ens = (weight1 * p1 + weight2 * p2) / (weight1 + weight2)
p_ens_score = getScore(p_ens, cdf)
kappa_cv[run][fold] = quadratic_weighted_kappa(p_ens_score, true_label)
kappa_cv_mean = np.mean(kappa_cv)
return {'loss': -kappa_cv_mean, 'status': STATUS_OK}
开发者ID:0x0all,项目名称:Kaggle_CrowdFlower,代码行数:16,代码来源:ensemble_selection.py
示例18: cnn1d_w2vembd
def cnn1d_w2vembd(X_train, Y_train, X_test, Y_test, nb_classes,
maxlen,
nb_filter, filter_length, batch_size, nb_epoch, optm):
"""
- CNN-1d on 3d sensor which uses word2vec embedding
- MOT
:param <X, Y> train and test sets
:param nb_classes # of classes
:param maxlen max of n char in a sentence
:param nb_filter
:param filter_length
:param batch_size
:param nb_epoch
:param optm
:return:
"""
pool_length = maxlen - filter_length + 1
model = Sequential()
model.add(Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode="valid",
activation="relu", input_shape=(maxlen, 300)))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer=optm)
earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)
model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,
validation_split=0.1, show_accuracy=True, callbacks=[earlystop])
classes = earlystop.model.predict_classes(X_test, batch_size=batch_size)
acc = np_utils.accuracy(classes, np_utils.categorical_probas_to_classes(Y_test)) # accuracy only supports classes
print('Test accuracy:', acc)
# return(acc)
kappa = metrics.quadratic_weighted_kappa(classes, np_utils.categorical_probas_to_classes(Y_test))
print('Test Kappa:', kappa)
return (kappa)
开发者ID:leocnj,项目名称:dl_response_rater,代码行数:44,代码来源:cnn1d.py
示例19: lstm_selfembd
def lstm_selfembd(X_train, Y_train, X_test, Y_test, nb_classes,
maxlen, vocab_size, embd_dim,
batch_size, nb_epoch, optm):
"""
- LSTM on text input (represented in int)
- fully-connected model
:param <X, Y> train and test sets
:param nb_classes # of classes
:param maxlen max of n char in a sentence
:param vocab_size
:param embd_dim
:param batch_size
:param nb_epoch
:param optm optimizer options, e.g., adam, rmsprop, etc.
:return:
"""
model = Sequential()
model.add(Embedding(vocab_size, embd_dim, input_length=maxlen))
model.add(Dropout(0.25))
# model.add(LSTM(100, return_sequences=True))
model.add(LSTM(50))
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer=optm)
earlystop = EarlyStopping(monitor='val_loss', patience=2, verbose=1)
model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,
validation_split=0.1, show_accuracy=True, callbacks=[earlystop])
classes = earlystop.model.predict_classes(X_test, batch_size=batch_size)
acc = np_utils.accuracy(classes, np_utils.categorical_probas_to_classes(Y_test)) # accuracy only supports classes
print('Test accuracy:', acc)
kappa = metrics.quadratic_weighted_kappa(classes, np_utils.categorical_probas_to_classes(Y_test))
print('Test Kappa:', kappa)
return (kappa)
开发者ID:leocnj,项目名称:dl_response_rater,代码行数:42,代码来源:cnn1d.py
示例20: cnn_var_selfembd
def cnn_var_selfembd(X_train, Y_train, X_test, Y_test, nb_classes,
maxlen, vocab_size, embd_size,
nb_filter, batch_size, nb_epoches, optm):
ngram_filters = [2, 5, 8]
input = Input(shape=(maxlen,), name='input', dtype='int32')
embedded = Embedding(input_dim=vocab_size, output_dim=embd_size, input_length=maxlen)(input)
convs = [None, None, None]
# three CNNs
for i, n_gram in enumerate(ngram_filters):
pool_length = maxlen - n_gram + 1
convs[i] = Convolution1D(nb_filter=nb_filter,
filter_length=n_gram,
border_mode="valid",
activation="relu")(embedded)
convs[i] = MaxPooling1D(pool_length=pool_length)(convs[i])
convs[i] = Flatten()(convs[i])
merged = merge([convs[0], convs[1], convs[2]], mode='concat', concat_axis=1)
merged = Dropout(0.5)(merged)
output = Dense(nb_classes, activation='softmax', name='output')(merged)
model = Model(input, output)
model.compile(optm, loss={'output': 'categorical_crossentropy'})
earlystop = EarlyStopping(monitor='val_loss', patience=1, verbose=1)
model.fit(X_train, Y_train,
nb_epoch=nb_epoches, batch_size=batch_size,
validation_split=0.1, callbacks=[earlystop])
probs = earlystop.model.predict(X_test, batch_size=batch_size)
classes = np_utils.categorical_probas_to_classes(probs)
acc = np_utils.accuracy(classes,
np_utils.categorical_probas_to_classes(Y_test))
print('Test accuracy:', acc)
kappa = metrics.quadratic_weighted_kappa(classes,
np_utils.categorical_probas_to_classes(Y_test))
print('Test Kappa:', kappa)
return acc
开发者ID:leocnj,项目名称:dl_response_rater,代码行数:40,代码来源:cnn1d.py
注:本文中的ml_metrics.quadratic_weighted_kappa函数示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。 |
请发表评论