mlr3，基准测试和嵌套重采样：如何从基准对象中提取调优模型来计算特征重要性

Question

我正在使用 mlr3 中的 benchmark() 函数来比较几种 ML 算法。其中之一是具有超参数调整的 XGB。因此，我有一个外部重采样来评估整体性能（保留样本）和一个用于超参数调整（5 折交叉验证）的内部重采样。除了对所有 ML 算法的准确性进行估计外，我还想了解调整后的 XGB 的特征重要性。为此，我必须访问调整后的模型（在基准对象中）。我不知道如何去做。 benchmark() 返回的对象是一个深度嵌套的列表，我不明白它的结构。

这个 对我没有帮助，因为它使用了不同的设置（管道中的学习器而不是基准对象）。

这个 answer on github 对我没有帮助，因为它展示了如何一次提取关于基准测试的所有信息，而不是如何提取基准测试中的一个学习者的一个（调整的）模型。

下面是我用来执行嵌套重采样的代码。在基准测试之后，我想按照 here 所述估计特征重要性，这需要访问调整后的 XGB 模型。

require(mlr3verse)

### Parameters

## Tuning

n_folds = 5

grid_search_resolution = 2

measure = msr("classif.acc")

task = tsk("iris")

# Messages mlr3
# 
options("mlr3.debug" = TRUE)

### Set up hyperparameter tuning
# AutoTuner for the inner resampling

## inner-resampling design
inner_resampling = rsmp("cv", folds = n_folds)
terminator = trm("none")
 
## XGB: no Hyperparameter Tuning
xgb_no_tuning = lrn("classif.xgboost", eval_metric = "mlogloss")
set_threads(xgb_no_tuning, n = 6)

## XGB: AutoTuner
# Setting up Hyperparameter Tuning

xgb_learner_tuning = lrn("classif.xgboost", eval_metric = "mlogloss")
xgb_search_space = ps(nrounds = p_int(lower = 100, upper= 500),
                      max_depth = p_int(lower = 3, upper= 10),
                      colsample_bytree = p_dbl(lower = 0.6, upper = 1)
                  )
xgb_tuner = tnr("grid_search", resolution = grid_search_resolution)

# implicit parallelisation
set_threads(xgb_learner_tuning, n = 6)

xgb_tuned = AutoTuner$new(xgb_learner_tuning, inner_resampling, measure, terminator, xgb_tuner, xgb_search_space, store_tuning_instance = TRUE)

## Outer re-sampling: hold-out
outer_resampling = rsmp("holdout")
outer_resampling$instantiate(task)

bm_design = benchmark_grid(
  tasks = task,
  learners = c(lrn("classif.featureless"), 
               xgb_no_tuning,
               xgb_tuned 
  ),
  resamplings = outer_resampling
)

begin_time = Sys.time()
bmr = benchmark(bm_design, store_models = TRUE)
duration = Sys.time() - begin_time

print(duration)

## Results of benchmarking
benchmark_results = bmr$aggregate(measure)
print(benchmark_results)


## Overview

mlr3misc::map(as.data.table(bmr)$learner, "model")

## Detailed results

# Specification of learners
print(bmr$learners$learner)

解决方案

根据 be-marc

的评论

require(mlr3verse)
require(mlr3tuning)
require(mlr3misc)

### Parameters

## Tuning

n_folds = 5

grid_search_resolution = 2

measure = msr("classif.acc")

task = tsk("iris")

# Messages mlr3
# 
options("mlr3.debug" = TRUE)

### Set up hyperparameter tuning
# AutoTuner for the inner resampling

## inner-resampling design
inner_resampling = rsmp("cv", folds = n_folds)
terminator = trm("none")
 
## XGB: no Hyperparameter Tuning
xgb_no_tuning = lrn("classif.xgboost", eval_metric = "mlogloss")
set_threads(xgb_no_tuning, n = 6)

## XGB: AutoTuner
# Setting up Hyperparameter Tuning

xgb_learner_tuning = lrn("classif.xgboost", eval_metric = "mlogloss")
xgb_search_space = ps(nrounds = p_int(lower = 100, upper= 500),
                      max_depth = p_int(lower = 3, upper= 10),
                      colsample_bytree = p_dbl(lower = 0.6, upper = 1)
                  )
xgb_tuner = tnr("grid_search", resolution = grid_search_resolution)

# implicit parallelisation
set_threads(xgb_learner_tuning, n = 6)

xgb_tuned = AutoTuner$new(xgb_learner_tuning, inner_resampling, measure, terminator, xgb_tuner, xgb_search_space, store_tuning_instance = TRUE)

## Outer re-sampling: hold-out
outer_resampling = rsmp("holdout")
outer_resampling$instantiate(task)

bm_design = benchmark_grid(
  tasks = task,
  learners = c(lrn("classif.featureless"), 
               xgb_no_tuning,
               xgb_tuned 
  ),
  resamplings = outer_resampling
)

begin_time = Sys.time()
bmr = benchmark(bm_design, store_models = TRUE)
duration = Sys.time() - begin_time

print(duration)

## Results of benchmarking
benchmark_results = bmr$aggregate(measure)
print(benchmark_results)


## Overview

mlr3misc::map(as.data.table(bmr)$learner, "model")

## Detailed results

# Specification of learners
print(bmr$learners$learner)

## Feature Importance

# extract models from outer sampling
# 

data = as.data.table(bmr)
outer_learners = map(data$learner, "learner")

xgb_tuned_model = outer_learners[[3]]

print(xgb_tuned_model)

# print feature importance 
# (presumably gain - mlr3 documentation not clear)
print(xgb_tuned_model$importance())

Answer 1

library(mlr3tuning)
library(mlr3learners)
library(mlr3misc)

learner = lrn("classif.xgboost", nrounds = to_tune(100, 500), eval_metric = "logloss")

at = AutoTuner$new(
  learner = learner,
  resampling = rsmp("cv", folds = 3),
  measure = msr("classif.ce"),
  terminator = trm("evals", n_evals = 5),
  tuner = tnr("random_search"),
  store_models = TRUE
)

design = benchmark_grid(task = tsk("pima"), learner = at, resampling = rsmp("cv", folds = 5))
bmr = benchmark(design, store_models = TRUE)

提取适合外环的学习者

data = as.data.table(bmr)
outer_learners = map(data$learner, "learner")

提取适合内循环的学习器

archives = extract_inner_tuning_archives(bmr)
inner_learners = map(archives$resample_result, "learners")