mlr3 中的自定义 Precision-Recall AUC 度量
Custom Precision-Recall AUC measure in mlr3
我想在 mlr3 中创建自定义 Precision-Recall AUC 度量。
我正在关注mlr3 book chapter on creating custom measures.
我感觉我差不多了，但是 R 抛出了一个烦人的错误，我不知道如何解释。
让我们定义度量：
PRAUC = R6::R6Class("PRAUC",
inherit = mlr3::MeasureClassif,
public = list(
initialize = function() {
super$initialize(
# custom id for the measure
id = "classif.prauc",
# additional packages required to calculate this measure
packages = c('PRROC'),
# properties, see below
properties = character(),
# required predict type of the learner
predict_type = "prob",
# feasible range of values
range = c(0, 1),
# minimize during tuning?
minimize = FALSE
)
}
),
private = list(
# custom scoring function operating on the prediction object
.score = function(prediction, ...) {
truth1 <- ifelse(prediction$truth == levels(prediction$truth)[1], 1, 0) # Function PRROC::pr.curve assumes binary response is numeric, positive class is 1, negative class is 0
PRROC::pr.curve(scores.class0 = prediction$prob, weights.class0 = truth1)
}
)
)
mlr3::mlr_measures$add("classif.prauc", PRAUC)
让我们看看它是否有效：
task_sonar <- tsk('sonar')
learner <- lrn('classif.rpart', predict_type = 'prob')
learner$train(task_sonar)
pred <- learner$predict(task_sonar)
pred$score(msr('classif.prauc'))
# Error in if (sum(weights < 0) != 0) { :
# missing value where TRUE/FALSE needed
这是回溯：
11.
check(length(sorted.scores.class0), weights.class0)
10.
compute.pr(scores.class0, scores.class1, weights.class0, weights.class1,
curve, minStepSize, max.compute, min.compute, rand.compute,
dg.compute)
9.
PRROC::pr.curve(scores.class0 = prediction$prob, weights.class0 = truth1)
8.
measure$.__enclos_env__$private$.score(prediction = prediction,
task = task, learner = learner, train_set = train_set)
7.
measure_score(self, prediction, task, learner, train_set)
6.
m$score(prediction = self, task = task, learner = learner, train_set = train_set)
5.
FUN(X[[i]], ...)
4.
vapply(.x, .f, FUN.VALUE = .value, USE.NAMES = FALSE, ...)
3.
map_mold(.x, .f, NA_real_, ...)
2.
map_dbl(measures, function(m) m$score(prediction = self, task = task,
learner = learner, train_set = train_set))
1.
pred$score(msr("classif.prauc"))
故障似乎来自 PRROC::pr.curve。然而，当在实际预测对象 pred 上尝试这个函数时，它工作得很好：
PRROC::pr.curve(
scores.class0 = pred$prob[, 1],
weights.class0 = ifelse(pred$truth == levels(pred$truth)[1], 1, 0)
)
# Precision-recall curve
#
# Area under curve (Integral):
# 0.9081261
#
# Area under curve (Davis & Goadrich):
# 0.9081837
#
# Curve not computed ( can be done by using curve=TRUE )
发生错误的一种可能情况是，在 PRAUC 中，PRROC::pr.curve 的参数 weights.class0 是 NA。我无法确认这一点，但我怀疑 weights.class0 正在接收 NA 而不是数字，导致 PRROC::pr.curve 在 PRAUC 内发生故障。如果是这样的话，我不知道为什么会这样。
可能还有其他我没有想到的场景。任何帮助将不胜感激。
编辑
missuse 的回答帮助我意识到为什么我的措施不起作用。首先，
PRROC::pr.curve(scores.class0 = prediction$prob, weights.class0 = truth1)
应该是
PRROC::pr.curve(scores.class0 = prediction$prob[, 1], weights.class0 = truth1)。
其次，函数pr.curve returns class PRROC 的一个对象，而我定义的mlr3 度量实际上期望numeric.所以应该是
PRROC::pr.curve(scores.class0 = prediction$prob[, 1], weights.class0 = truth1)[[2]]
或
PRROC::pr.curve(scores.class0 = prediction$prob[, 1], weights.class0 = truth1)[[3]],
取决于用于计算 AUC 的方法（参见 ?PRROC::pr.curve）。
请注意，虽然 MLmetrics::PRAUC 远没有 PRROC::pr.curve 混乱，但它看起来像 the former is poorly implemented.
这是 PRROC::pr.curve 实际有效的措施实施：
PRAUC = R6::R6Class("PRAUC",
inherit = mlr3::MeasureClassif,
public = list(
initialize = function() {
super$initialize(
# custom id for the measure
id = "classif.prauc",
# additional packages required to calculate this measure
packages = c('PRROC'),
# properties, see below
properties = character(),
# required predict type of the learner
predict_type = "prob",
# feasible range of values
range = c(0, 1),
# minimize during tuning?
minimize = FALSE
)
}
),
private = list(
# custom scoring function operating on the prediction object
.score = function(prediction, ...) {
truth1 <- ifelse(prediction$truth == levels(prediction$truth)[1], 1, 0) # Looks like in mlr3 the positive class in binary classification is always the first factor level
PRROC::pr.curve(
scores.class0 = prediction$prob[, 1], # Looks like in mlr3 the positive class in binary classification is always the first of two columns
weights.class0 = truth1
)[[2]]
}
)
)
mlr3::mlr_measures$add("classif.prauc", PRAUC)
示例：
task_sonar <- tsk('sonar')
learner <- lrn('classif.rpart', predict_type = 'prob')
learner$train(task_sonar)
pred <- learner$predict(task_sonar)
pred$score(msr('classif.prauc'))
#classif.prauc
# 0.923816
但是，现在的问题是改变正面 class 会导致不同的分数：
task_sonar <- tsk('sonar')
task_sonar$positive <- 'R' # Now R is the positive class
learner <- lrn('classif.rpart', predict_type = 'prob')
learner$train(task_sonar)
pred <- learner$predict(task_sonar)
pred$score(msr('classif.prauc'))
#classif.prauc
# 0.9081261
?PRROC::pr.curve比较混乱，所以我会用MLmetrics::PRAUC来计算PRAUC：
library(mlr3measures)
library(mlr3)
PRAUC = R6::R6Class("PRAUC",
inherit = mlr3::MeasureClassif,
public = list(
initialize = function() {
super$initialize(
# custom id for the measure
id = "classif.prauc",
# additional packages required to calculate this measure
packages = c('MLmetrics'),
# properties, see below
properties = character(),
# required predict type of the learner
predict_type = "prob",
# feasible range of values
range = c(0, 1),
# minimize during tuning?
minimize = FALSE
)
}
),
private = list(
# custom scoring function operating on the prediction object
.score = function(prediction, ...) {
MLmetrics::PRAUC(prediction$prob[,1], #probs for 1st (positive class is in first column) class
as.integer(prediction$truth == levels(prediction$truth)[1])) #truth for 1st class
}
)
)
验证它是否有效：
mlr3::mlr_measures$add("classif.prauc", PRAUC)
task_sonar <- tsk('sonar')
learner <- lrn('classif.rpart', predict_type = 'prob')
learner$train(task_sonar)
pred <- learner$predict(task_sonar)
pred$score(msr('classif.prauc'))
classif.prauc
0.8489383
MLmetrics::PRAUC(pred$data$prob[,1],
as.integer(pred$truth == "M"))
0.8489383
编辑：使用 PRROC::pr.curve 的措施实施作为对上述问题的编辑给出。建议使用该实现，因为 PRROC::pr.curve 比 MLmetrics::PRAUC.
更精确
我想在 mlr3 中创建自定义 Precision-Recall AUC 度量。
我正在关注mlr3 book chapter on creating custom measures.
我感觉我差不多了，但是 R 抛出了一个烦人的错误，我不知道如何解释。
让我们定义度量：
PRAUC = R6::R6Class("PRAUC",
inherit = mlr3::MeasureClassif,
public = list(
initialize = function() {
super$initialize(
# custom id for the measure
id = "classif.prauc",
# additional packages required to calculate this measure
packages = c('PRROC'),
# properties, see below
properties = character(),
# required predict type of the learner
predict_type = "prob",
# feasible range of values
range = c(0, 1),
# minimize during tuning?
minimize = FALSE
)
}
),
private = list(
# custom scoring function operating on the prediction object
.score = function(prediction, ...) {
truth1 <- ifelse(prediction$truth == levels(prediction$truth)[1], 1, 0) # Function PRROC::pr.curve assumes binary response is numeric, positive class is 1, negative class is 0
PRROC::pr.curve(scores.class0 = prediction$prob, weights.class0 = truth1)
}
)
)
mlr3::mlr_measures$add("classif.prauc", PRAUC)
让我们看看它是否有效：
task_sonar <- tsk('sonar')
learner <- lrn('classif.rpart', predict_type = 'prob')
learner$train(task_sonar)
pred <- learner$predict(task_sonar)
pred$score(msr('classif.prauc'))
# Error in if (sum(weights < 0) != 0) { :
# missing value where TRUE/FALSE needed
这是回溯：
11.
check(length(sorted.scores.class0), weights.class0)
10.
compute.pr(scores.class0, scores.class1, weights.class0, weights.class1,
curve, minStepSize, max.compute, min.compute, rand.compute,
dg.compute)
9.
PRROC::pr.curve(scores.class0 = prediction$prob, weights.class0 = truth1)
8.
measure$.__enclos_env__$private$.score(prediction = prediction,
task = task, learner = learner, train_set = train_set)
7.
measure_score(self, prediction, task, learner, train_set)
6.
m$score(prediction = self, task = task, learner = learner, train_set = train_set)
5.
FUN(X[[i]], ...)
4.
vapply(.x, .f, FUN.VALUE = .value, USE.NAMES = FALSE, ...)
3.
map_mold(.x, .f, NA_real_, ...)
2.
map_dbl(measures, function(m) m$score(prediction = self, task = task,
learner = learner, train_set = train_set))
1.
pred$score(msr("classif.prauc"))
故障似乎来自 PRROC::pr.curve。然而，当在实际预测对象 pred 上尝试这个函数时，它工作得很好：
PRROC::pr.curve(
scores.class0 = pred$prob[, 1],
weights.class0 = ifelse(pred$truth == levels(pred$truth)[1], 1, 0)
)
# Precision-recall curve
#
# Area under curve (Integral):
# 0.9081261
#
# Area under curve (Davis & Goadrich):
# 0.9081837
#
# Curve not computed ( can be done by using curve=TRUE )
发生错误的一种可能情况是，在 PRAUC 中，PRROC::pr.curve 的参数 weights.class0 是 NA。我无法确认这一点，但我怀疑 weights.class0 正在接收 NA 而不是数字，导致 PRROC::pr.curve 在 PRAUC 内发生故障。如果是这样的话，我不知道为什么会这样。
可能还有其他我没有想到的场景。任何帮助将不胜感激。
编辑
missuse 的回答帮助我意识到为什么我的措施不起作用。首先，
PRROC::pr.curve(scores.class0 = prediction$prob, weights.class0 = truth1)
应该是
PRROC::pr.curve(scores.class0 = prediction$prob[, 1], weights.class0 = truth1)。
其次，函数pr.curve returns class PRROC 的一个对象，而我定义的mlr3 度量实际上期望numeric.所以应该是
PRROC::pr.curve(scores.class0 = prediction$prob[, 1], weights.class0 = truth1)[[2]]
或
PRROC::pr.curve(scores.class0 = prediction$prob[, 1], weights.class0 = truth1)[[3]],
取决于用于计算 AUC 的方法（参见 ?PRROC::pr.curve）。
请注意，虽然 MLmetrics::PRAUC 远没有 PRROC::pr.curve 混乱，但它看起来像 the former is poorly implemented.
这是 PRROC::pr.curve 实际有效的措施实施：
PRAUC = R6::R6Class("PRAUC",
inherit = mlr3::MeasureClassif,
public = list(
initialize = function() {
super$initialize(
# custom id for the measure
id = "classif.prauc",
# additional packages required to calculate this measure
packages = c('PRROC'),
# properties, see below
properties = character(),
# required predict type of the learner
predict_type = "prob",
# feasible range of values
range = c(0, 1),
# minimize during tuning?
minimize = FALSE
)
}
),
private = list(
# custom scoring function operating on the prediction object
.score = function(prediction, ...) {
truth1 <- ifelse(prediction$truth == levels(prediction$truth)[1], 1, 0) # Looks like in mlr3 the positive class in binary classification is always the first factor level
PRROC::pr.curve(
scores.class0 = prediction$prob[, 1], # Looks like in mlr3 the positive class in binary classification is always the first of two columns
weights.class0 = truth1
)[[2]]
}
)
)
mlr3::mlr_measures$add("classif.prauc", PRAUC)
示例：
task_sonar <- tsk('sonar')
learner <- lrn('classif.rpart', predict_type = 'prob')
learner$train(task_sonar)
pred <- learner$predict(task_sonar)
pred$score(msr('classif.prauc'))
#classif.prauc
# 0.923816
但是，现在的问题是改变正面 class 会导致不同的分数：
task_sonar <- tsk('sonar')
task_sonar$positive <- 'R' # Now R is the positive class
learner <- lrn('classif.rpart', predict_type = 'prob')
learner$train(task_sonar)
pred <- learner$predict(task_sonar)
pred$score(msr('classif.prauc'))
#classif.prauc
# 0.9081261
?PRROC::pr.curve比较混乱，所以我会用MLmetrics::PRAUC来计算PRAUC：
library(mlr3measures)
library(mlr3)
PRAUC = R6::R6Class("PRAUC",
inherit = mlr3::MeasureClassif,
public = list(
initialize = function() {
super$initialize(
# custom id for the measure
id = "classif.prauc",
# additional packages required to calculate this measure
packages = c('MLmetrics'),
# properties, see below
properties = character(),
# required predict type of the learner
predict_type = "prob",
# feasible range of values
range = c(0, 1),
# minimize during tuning?
minimize = FALSE
)
}
),
private = list(
# custom scoring function operating on the prediction object
.score = function(prediction, ...) {
MLmetrics::PRAUC(prediction$prob[,1], #probs for 1st (positive class is in first column) class
as.integer(prediction$truth == levels(prediction$truth)[1])) #truth for 1st class
}
)
)
验证它是否有效：
mlr3::mlr_measures$add("classif.prauc", PRAUC)
task_sonar <- tsk('sonar')
learner <- lrn('classif.rpart', predict_type = 'prob')
learner$train(task_sonar)
pred <- learner$predict(task_sonar)
pred$score(msr('classif.prauc'))
classif.prauc
0.8489383
MLmetrics::PRAUC(pred$data$prob[,1],
as.integer(pred$truth == "M"))
0.8489383
编辑：使用 PRROC::pr.curve 的措施实施作为对上述问题的编辑给出。建议使用该实现，因为 PRROC::pr.curve 比 MLmetrics::PRAUC.