具有交叉验证的神经网络模型的多个指标
Multiple metrics for neural network model with cross validation
我正在尝试获取 LSTM 模型交叉验证的 F1、精度和召回率。
我知道如何显示准确度，但是当我尝试使用 cross_validate 显示其他指标时，我遇到了很多不同的错误。
我的代码如下：
def nn_model():
model_lstm1 = Sequential()
model_lstm1.add(Embedding(20000, 100, input_length=49))
model_lstm1.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model_lstm1.add(Dense(2, activation='sigmoid'))
model_lstm1.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
return model_lstm1
classifier = KerasClassifier(build_fn=nn_model, batch_size=10,nb_epoch=10)
scoring = {'precision' : make_scorer(precision_score),
'recall' : make_scorer(recall_score),
'f1_score' : make_scorer(f1_score)}
results = cross_validate(classifier, X_train, y_train, cv=skf, scoring = scoring)
print("F1 score SVM: %0.2f (+/- %0.2f)" % (np.mean(results[f1_score]), np.std(results[f1_score])))
print("precision score SVM: %0.2f (+/- %0.2f)" % (np.mean(results[precision]), np.std(results[precision])))
print("recall macro SVM: %0.2f (+/- %0.2f)" % (np.mean(results[recall]), np.std(results[recall])))
我得到的错误如下：
Epoch 1/1 1086/1086 [==============================] - 18s 17ms/step -
loss: 0.6014 - acc: 0.7035
--------------------------------------------------------------------------- ValueError Traceback (most recent call
last) in
6 'f1_score' : make_scorer(f1_score)}
7
----> 8 results = cross_validate(classifier, X_train, y_train, cv=skf, scoring = scoring)
9
10 print("F1 score SVM: %0.2f (+/- %0.2f)" % (np.mean(results[f1_score]), np.std(results[f1_score])))
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/model_selection/_validation.py in cross_validate(estimator, X, y, groups, scoring, cv, n_jobs,
verbose, fit_params, pre_dispatch, return_train_score,
return_estimator, error_score)
229 return_times=True, return_estimator=return_estimator,
230 error_score=error_score)
--> 231 for train, test in cv.split(X, y, groups))
232
233 zipped_scores = list(zip(*scores))
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/joblib/parallel.py
in call(self, iterable)
919 # remaining jobs.
920 self._iterating = False
--> 921 if self.dispatch_one_batch(iterator):
922 self._iterating = self._original_iterator is not None
923
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/joblib/parallel.py
in dispatch_one_batch(self, iterator)
757 return False
758 else:
--> 759 self._dispatch(tasks)
760 return True
761
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/joblib/parallel.py
in _dispatch(self, batch)
714 with self._lock:
715 job_idx = len(self._jobs)
--> 716 job = self._backend.apply_async(batch, callback=cb)
717 # A job can complete so quickly than its callback is
718 # called before we get here, causing self._jobs to
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/joblib/_parallel_backends.py
in apply_async(self, func, callback)
180 def apply_async(self, func, callback=None):
181 """Schedule a func to be run"""
--> 182 result = ImmediateResult(func)
183 if callback:
184 callback(result)
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/joblib/_parallel_backends.py
in init(self, batch)
547 # Don't delay the application, to avoid keeping the input
548 # arguments in memory
--> 549 self.results = batch()
550
551 def get(self):
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/joblib/parallel.py
in call(self)
223 with parallel_backend(self._backend, n_jobs=self._n_jobs):
224 return [func(*args, **kwargs)
--> 225 for func, args, kwargs in self.items]
226
227 def len(self):
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/joblib/parallel.py
in (.0)
223 with parallel_backend(self._backend, n_jobs=self._n_jobs):
224 return [func(*args, **kwargs)
--> 225 for func, args, kwargs in self.items]
226
227 def len(self):
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/model_selection/_validation.py in _fit_and_score(estimator, X, y, scorer, train, test, verbose,
parameters, fit_params, return_train_score, return_parameters,
return_n_test_samples, return_times, return_estimator, error_score)
552 fit_time = time.time() - start_time
553 # _score will return dict if is_multimetric is True
--> 554 test_scores = _score(estimator, X_test, y_test, scorer, is_multimetric)
555 score_time = time.time() - start_time - fit_time
556 if return_train_score:
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/model_selection/_validation.py in _score(estimator, X_test, y_test, scorer, is_multimetric)
595 """
596 if is_multimetric:
--> 597 return _multimetric_score(estimator, X_test, y_test, scorer)
598 else:
599 if y_test is None:
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/model_selection/_validation.py in _multimetric_score(estimator, X_test, y_test, scorers)
625 score = scorer(estimator, X_test)
626 else:
--> 627 score = scorer(estimator, X_test, y_test)
628
629 if hasattr(score, 'item'):
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/metrics/scorer.py
in call(self, estimator, X, y_true, sample_weight)
95 else:
96 return self._sign * self._score_func(y_true, y_pred,
---> 97 **self._kwargs)
98
99
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/metrics/classification.py
in precision_score(y_true, y_pred, labels, pos_label, average,
sample_weight) 1567
average=average, 1568
warn_for=('precision',),
-> 1569 sample_weight=sample_weight) 1570 return p 1571
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/metrics/classification.py
in precision_recall_fscore_support(y_true, y_pred, beta, labels,
pos_label, average, warn_for, sample_weight) 1413 raise
ValueError("beta should be >0 in the F-beta score") 1414 labels
= _check_set_wise_labels(y_true, y_pred, average, labels,
-> 1415 pos_label) 1416 1417 # Calculate tp_sum, pred_sum, true_sum ###
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/metrics/classification.py
in _check_set_wise_labels(y_true, y_pred, average, labels, pos_label)
1237 str(average_options)) 1238
-> 1239 y_type, y_true, y_pred = _check_targets(y_true, y_pred) 1240 present_labels = unique_labels(y_true, y_pred) 1241 if
average == 'binary':
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/metrics/classification.py
in _check_targets(y_true, y_pred)
79 if len(y_type) > 1:
80 raise ValueError("Classification metrics can't handle a mix of {0} "
---> 81 "and {1} targets".format(type_true, type_pred))
82
83 # We can't have more than one value on y_type => The set is no more needed
ValueError: Classification metrics can't handle a mix of
multilabel-indicator and binary targets
我做错了什么？
你的代码有问题
- 您不能使用热编码标签link。使用原始标签。您可以对原始标签使用
sparse_categorical_crossentropy 损失。
- cross_validate returns 得分为
test_scores。对于火车分数集 return_train_score
更正代码
def nn_model():
model_lstm1 = Sequential()
model_lstm1.add(Embedding(200, 100, input_length=10))
model_lstm1.add(LSTM(10, dropout=0.2, recurrent_dropout=0.2))
model_lstm1.add(Dense(2, activation='sigmoid'))
model_lstm1.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model_lstm1
classifier = KerasClassifier(build_fn=nn_model, batch_size=10,nb_epoch=10)
scoring = {'precision' : make_scorer(precision_score),
'recall' : make_scorer(recall_score),
'f1_score' : make_scorer(f1_score)}
results = cross_validate(classifier, np.random.randint(0,100,(1000,10)),
np.random.np.random.randint(0,2,1000), scoring = scoring, cv=3, return_train_score=True)
print("F1 score SVM: %0.2f (+/- %0.2f)" % (np.mean(results['test_f1_score']), np.std(results['test_f1_score'])))
print("precision score SVM: %0.2f (+/- %0.2f)" % (np.mean(results['test_precision']), np.std(results['test_precision'])))
print("recall macro SVM: %0.2f (+/- %0.2f)" % (np.mean(results['test_recall']), np.std(results['test_recall'])))
输出
Epoch 1/1
666/666 [==============================] - 5s 7ms/step - loss: 0.6932 - acc: 0.5075
Epoch 1/1
667/667 [==============================] - 5s 7ms/step - loss: 0.6929 - acc: 0.5127
Epoch 1/1
667/667 [==============================] - 5s 7ms/step - loss: 0.6934 - acc: 0.5007
F1 score SVM: 0.10 (+/- 0.09)
precision score SVM: 0.43 (+/- 0.07)
recall macro SVM: 0.06 (+/- 0.06)
你可能会得到
UndefinedMetricWarning: ....
首字母纪元中的警告（如果数据不足），您可以忽略。这是因为 classifier 正在 class 将所有数据转化为一个 class 而没有数据转化为另一个 class。
我正在尝试获取 LSTM 模型交叉验证的 F1、精度和召回率。
我知道如何显示准确度，但是当我尝试使用 cross_validate 显示其他指标时，我遇到了很多不同的错误。
我的代码如下：
def nn_model():
model_lstm1 = Sequential()
model_lstm1.add(Embedding(20000, 100, input_length=49))
model_lstm1.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model_lstm1.add(Dense(2, activation='sigmoid'))
model_lstm1.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
return model_lstm1
classifier = KerasClassifier(build_fn=nn_model, batch_size=10,nb_epoch=10)
scoring = {'precision' : make_scorer(precision_score),
'recall' : make_scorer(recall_score),
'f1_score' : make_scorer(f1_score)}
results = cross_validate(classifier, X_train, y_train, cv=skf, scoring = scoring)
print("F1 score SVM: %0.2f (+/- %0.2f)" % (np.mean(results[f1_score]), np.std(results[f1_score])))
print("precision score SVM: %0.2f (+/- %0.2f)" % (np.mean(results[precision]), np.std(results[precision])))
print("recall macro SVM: %0.2f (+/- %0.2f)" % (np.mean(results[recall]), np.std(results[recall])))
我得到的错误如下：
Epoch 1/1 1086/1086 [==============================] - 18s 17ms/step - loss: 0.6014 - acc: 0.7035 --------------------------------------------------------------------------- ValueError Traceback (most recent call last) in 6 'f1_score' : make_scorer(f1_score)} 7 ----> 8 results = cross_validate(classifier, X_train, y_train, cv=skf, scoring = scoring) 9 10 print("F1 score SVM: %0.2f (+/- %0.2f)" % (np.mean(results[f1_score]), np.std(results[f1_score])))
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/model_selection/_validation.py in cross_validate(estimator, X, y, groups, scoring, cv, n_jobs, verbose, fit_params, pre_dispatch, return_train_score, return_estimator, error_score) 229 return_times=True, return_estimator=return_estimator, 230 error_score=error_score) --> 231 for train, test in cv.split(X, y, groups)) 232 233 zipped_scores = list(zip(*scores))
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/joblib/parallel.py in call(self, iterable) 919 # remaining jobs. 920 self._iterating = False --> 921 if self.dispatch_one_batch(iterator): 922 self._iterating = self._original_iterator is not None 923
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/joblib/parallel.py in dispatch_one_batch(self, iterator) 757 return False 758 else: --> 759 self._dispatch(tasks) 760 return True 761
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/joblib/parallel.py in _dispatch(self, batch) 714 with self._lock: 715 job_idx = len(self._jobs) --> 716 job = self._backend.apply_async(batch, callback=cb) 717 # A job can complete so quickly than its callback is 718 # called before we get here, causing self._jobs to
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/joblib/_parallel_backends.py in apply_async(self, func, callback) 180 def apply_async(self, func, callback=None): 181 """Schedule a func to be run""" --> 182 result = ImmediateResult(func) 183 if callback: 184 callback(result)
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/joblib/_parallel_backends.py in init(self, batch) 547 # Don't delay the application, to avoid keeping the input 548 # arguments in memory --> 549 self.results = batch() 550 551 def get(self):
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/joblib/parallel.py in call(self) 223 with parallel_backend(self._backend, n_jobs=self._n_jobs): 224 return [func(*args, **kwargs) --> 225 for func, args, kwargs in self.items] 226 227 def len(self):
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/joblib/parallel.py in (.0) 223 with parallel_backend(self._backend, n_jobs=self._n_jobs): 224 return [func(*args, **kwargs) --> 225 for func, args, kwargs in self.items] 226 227 def len(self):
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/model_selection/_validation.py in _fit_and_score(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score, return_parameters, return_n_test_samples, return_times, return_estimator, error_score) 552 fit_time = time.time() - start_time 553 # _score will return dict if is_multimetric is True --> 554 test_scores = _score(estimator, X_test, y_test, scorer, is_multimetric) 555 score_time = time.time() - start_time - fit_time 556 if return_train_score:
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/model_selection/_validation.py in _score(estimator, X_test, y_test, scorer, is_multimetric) 595 """ 596 if is_multimetric: --> 597 return _multimetric_score(estimator, X_test, y_test, scorer) 598 else: 599 if y_test is None:
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/model_selection/_validation.py in _multimetric_score(estimator, X_test, y_test, scorers) 625 score = scorer(estimator, X_test) 626 else: --> 627 score = scorer(estimator, X_test, y_test) 628 629 if hasattr(score, 'item'):
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/metrics/scorer.py in call(self, estimator, X, y_true, sample_weight) 95 else: 96 return self._sign * self._score_func(y_true, y_pred, ---> 97 **self._kwargs) 98 99
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/metrics/classification.py in precision_score(y_true, y_pred, labels, pos_label, average, sample_weight) 1567
average=average, 1568
warn_for=('precision',), -> 1569 sample_weight=sample_weight) 1570 return p 1571/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/metrics/classification.py in precision_recall_fscore_support(y_true, y_pred, beta, labels, pos_label, average, warn_for, sample_weight) 1413 raise ValueError("beta should be >0 in the F-beta score") 1414 labels = _check_set_wise_labels(y_true, y_pred, average, labels, -> 1415 pos_label) 1416 1417 # Calculate tp_sum, pred_sum, true_sum ###
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/metrics/classification.py in _check_set_wise_labels(y_true, y_pred, average, labels, pos_label) 1237 str(average_options)) 1238 -> 1239 y_type, y_true, y_pred = _check_targets(y_true, y_pred) 1240 present_labels = unique_labels(y_true, y_pred) 1241 if average == 'binary':
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/metrics/classification.py in _check_targets(y_true, y_pred) 79 if len(y_type) > 1: 80 raise ValueError("Classification metrics can't handle a mix of {0} " ---> 81 "and {1} targets".format(type_true, type_pred)) 82 83 # We can't have more than one value on y_type => The set is no more needed
ValueError: Classification metrics can't handle a mix of multilabel-indicator and binary targets
我做错了什么？
你的代码有问题
- 您不能使用热编码标签link。使用原始标签。您可以对原始标签使用
sparse_categorical_crossentropy损失。 - cross_validate returns 得分为
test_scores。对于火车分数集return_train_score
更正代码
def nn_model():
model_lstm1 = Sequential()
model_lstm1.add(Embedding(200, 100, input_length=10))
model_lstm1.add(LSTM(10, dropout=0.2, recurrent_dropout=0.2))
model_lstm1.add(Dense(2, activation='sigmoid'))
model_lstm1.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model_lstm1
classifier = KerasClassifier(build_fn=nn_model, batch_size=10,nb_epoch=10)
scoring = {'precision' : make_scorer(precision_score),
'recall' : make_scorer(recall_score),
'f1_score' : make_scorer(f1_score)}
results = cross_validate(classifier, np.random.randint(0,100,(1000,10)),
np.random.np.random.randint(0,2,1000), scoring = scoring, cv=3, return_train_score=True)
print("F1 score SVM: %0.2f (+/- %0.2f)" % (np.mean(results['test_f1_score']), np.std(results['test_f1_score'])))
print("precision score SVM: %0.2f (+/- %0.2f)" % (np.mean(results['test_precision']), np.std(results['test_precision'])))
print("recall macro SVM: %0.2f (+/- %0.2f)" % (np.mean(results['test_recall']), np.std(results['test_recall'])))
输出
Epoch 1/1
666/666 [==============================] - 5s 7ms/step - loss: 0.6932 - acc: 0.5075
Epoch 1/1
667/667 [==============================] - 5s 7ms/step - loss: 0.6929 - acc: 0.5127
Epoch 1/1
667/667 [==============================] - 5s 7ms/step - loss: 0.6934 - acc: 0.5007
F1 score SVM: 0.10 (+/- 0.09)
precision score SVM: 0.43 (+/- 0.07)
recall macro SVM: 0.06 (+/- 0.06)
你可能会得到
UndefinedMetricWarning: ....
首字母纪元中的警告（如果数据不足），您可以忽略。这是因为 classifier 正在 class 将所有数据转化为一个 class 而没有数据转化为另一个 class。