ranger: Binary Classification

# nolint start
library(mlexperiments)
library(mllrnrs)

See https://github.com/kapsner/mllrnrs/blob/main/R/learner_ranger.R for implementation details.

Preprocessing

Import and Prepare Data

library(mlbench)
data("BreastCancer")
dataset <- BreastCancer |>
  data.table::as.data.table() |>
  na.omit()
feature_cols <- colnames(dataset)[2:10]
target_col <- "Class"

General Configurations

seed <- 123
if (isTRUE(as.logical(Sys.getenv("_R_CHECK_LIMIT_CORES_")))) {
  # on cran
  ncores <- 2L
} else {
  ncores <- ifelse(
    test = parallel::detectCores() > 4,
    yes = 4L,
    no = ifelse(
      test = parallel::detectCores() < 2L,
      yes = 1L,
      no = parallel::detectCores()
    )
  )
}
options("mlexperiments.bayesian.max_init" = 4L)

Generate Training- and Test Data

data_split <- splitTools::partition(
  y = dataset[, get(target_col)],
  p = c(train = 0.7, test = 0.3),
  type = "stratified",
  seed = seed
)

train_x <- model.matrix(
  ~ -1 + .,
  dataset[data_split$train, .SD, .SDcols = feature_cols]
)
train_y <- dataset[data_split$train, get(target_col)]


test_x <- model.matrix(
  ~ -1 + .,
  dataset[data_split$test, .SD, .SDcols = feature_cols]
)
test_y <- dataset[data_split$test, get(target_col)]

Generate Training Data Folds

fold_list <- splitTools::create_folds(
  y = train_y,
  k = 3,
  type = "stratified",
  seed = seed
)

Experiments

Prepare Experiments

# required learner arguments, not optimized
learner_args <- list(probability = TRUE)

# set arguments for predict function and performance metric,
# required for mlexperiments::MLCrossValidation and
# mlexperiments::MLNestedCV
predict_args <- list(prob = TRUE, positive = "malignant")
performance_metric <- mlexperiments::metric("AUC")
performance_metric_args <- list(positive = "malignant", negative = "benign")
return_models <- FALSE

# required for grid search and initialization of bayesian optimization
parameter_grid <- expand.grid(
  num.trees = seq(500, 1000, 500),
  mtry = seq(2, 6, 2),
  min.node.size = seq(2, 9, 4),
  max.depth = seq(1, 9, 4),
  sample.fraction = seq(0.5, 0.8, 0.3)
)
# reduce to a maximum of 10 rows
if (nrow(parameter_grid) > 10) {
  set.seed(123)
  sample_rows <- sample(seq_len(nrow(parameter_grid)), 10, FALSE)
  parameter_grid <- kdry::mlh_subset(parameter_grid, sample_rows)
}

# required for bayesian optimization
parameter_bounds <- list(
  num.trees = c(100L, 1000L),
  mtry = c(2L, 9L),
  min.node.size = c(2L, 20L),
  max.depth = c(1L, 40L),
  sample.fraction = c(0.3, 1.)
)
optim_args <- list(
  n_iter = ncores,
  kappa = 3.5,
  acq = "ucb"
)

Hyperparameter Tuning

tuner <- mlexperiments::MLTuneParameters$new(
  learner = mllrnrs::LearnerRanger$new(),
  strategy = "grid",
  ncores = ncores,
  seed = seed
)

tuner$parameter_grid <- parameter_grid
tuner$learner_args <- learner_args
tuner$split_type <- "stratified"

tuner$set_data(
  x = train_x,
  y = train_y
)

tuner_results_grid <- tuner$execute(k = 3)
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [================================>------------------------------------------------------------------------------] 3/10 ( 30%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [===========================================>-------------------------------------------------------------------] 4/10 ( 40%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [=======================================================>-------------------------------------------------------] 5/10 ( 50%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [==============================================================================================================] 10/10 (100%)
#>  Classification: using 'mean classification error' as optimization metric.

head(tuner_results_grid)
#>    setting_id metric_optim_mean num.trees  mtry min.node.size max.depth sample.fraction probability
#>         <int>             <num>     <num> <num>         <num>     <num>           <num>      <lgcl>
#> 1:          1        0.04398668       500     2             6         9             0.5        TRUE
#> 2:          2        0.04189024       500     4             2         5             0.8        TRUE
#> 3:          3        0.04398668      1000     2             2         5             0.5        TRUE
#> 4:          4        0.04189024       500     2             6         9             0.8        TRUE
#> 5:          5        0.04613602      1000     6             2         1             0.8        TRUE
#> 6:          6        0.04398668      1000     2             2         5             0.8        TRUE

Bayesian Optimization

tuner <- mlexperiments::MLTuneParameters$new(
  learner = mllrnrs::LearnerRanger$new(),
  strategy = "bayesian",
  ncores = ncores,
  seed = seed
)

tuner$parameter_grid <- parameter_grid
tuner$parameter_bounds <- parameter_bounds

tuner$learner_args <- learner_args
tuner$optim_args <- optim_args

tuner$split_type <- "stratified"

tuner$set_data(
  x = train_x,
  y = train_y
)

tuner_results_bayesian <- tuner$execute(k = 3)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.169  Round = 1   num.trees = 1000.0000   mtry = 2.0000   min.node.size = 2.0000  max.depth = 5.0000  sample.fraction = 0.5000    Value = -0.04398668
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.116  Round = 2   num.trees = 500.0000    mtry = 2.0000   min.node.size = 2.0000  max.depth = 9.0000  sample.fraction = 0.5000    Value = -0.04189024
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.107  Round = 3   num.trees = 500.0000    mtry = 4.0000   min.node.size = 2.0000  max.depth = 5.0000  sample.fraction = 0.8000    Value = -0.04189024
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.169  Round = 4   num.trees = 1000.0000   mtry = 4.0000   min.node.size = 6.0000  max.depth = 9.0000  sample.fraction = 0.8000    Value = -0.04189024
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.17   Round = 5   num.trees = 952.0000    mtry = 7.0000   min.node.size = 5.0000  max.depth = 21.0000 sample.fraction = 0.8216647 Value = -0.04817972
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.174  Round = 6   num.trees = 952.0000    mtry = 7.0000   min.node.size = 5.0000  max.depth = 21.0000 sample.fraction = 0.8216647 Value = -0.04817972
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.18   Round = 7   num.trees = 952.0000    mtry = 7.0000   min.node.size = 5.0000  max.depth = 21.0000 sample.fraction = 0.8216647 Value = -0.04817972
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.183  Round = 8   num.trees = 952.0000    mtry = 7.0000   min.node.size = 5.0000  max.depth = 21.0000 sample.fraction = 0.8216647 Value = -0.04817972
#>
#>  Best Parameters Found:
#> Round = 2    num.trees = 500.0000    mtry = 2.0000   min.node.size = 2.0000  max.depth = 9.0000  sample.fraction = 0.5000    Value = -0.04189024

head(tuner_results_bayesian)
#>    setting_id num.trees  mtry min.node.size max.depth sample.fraction       Value probability metric_optim_mean
#>         <int>     <num> <num>         <num>     <num>           <num>       <num>      <lgcl>             <num>
#> 1:          1      1000     2             2         5       0.5000000 -0.04398668        TRUE        0.04398668
#> 2:          2       500     2             2         9       0.5000000 -0.04189024        TRUE        0.04189024
#> 3:          3       500     4             2         5       0.8000000 -0.04189024        TRUE        0.04189024
#> 4:          4      1000     4             6         9       0.8000000 -0.04189024        TRUE        0.04189024
#> 5:          5       952     7             5        21       0.8216647 -0.04817972        TRUE        0.04817972
#> 6:          6       952     7             5        21       0.8216647 -0.04817972        TRUE        0.04817972

k-Fold Cross Validation

validator <- mlexperiments::MLCrossValidation$new(
  learner = mllrnrs::LearnerRanger$new(),
  fold_list = fold_list,
  ncores = ncores,
  seed = seed
)

validator$learner_args <- tuner$results$best.setting[-1]

validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models

validator$set_data(
  x = train_x,
  y = train_y
)

validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> CV fold: Fold2
#>
#> CV fold: Fold3

head(validator_results)
#>      fold performance  mtry min.node.size max.depth sample.fraction probability
#>    <char>       <num> <num>         <num>     <num>           <num>      <lgcl>
#> 1:  Fold1   0.9925861     2             2         9             0.5        TRUE
#> 2:  Fold2   0.9935853     2             2         9             0.5        TRUE
#> 3:  Fold3   0.9888393     2             2         9             0.5        TRUE

Nested Cross Validation

validator <- mlexperiments::MLNestedCV$new(
  learner = mllrnrs::LearnerRanger$new(),
  strategy = "grid",
  fold_list = fold_list,
  k_tuning = 3L,
  ncores = ncores,
  seed = seed
)

validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"

validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models

validator$set_data(
  x = train_x,
  y = train_y
)

validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [===========================================>-------------------------------------------------------------------] 4/10 ( 40%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [=======================================================>-------------------------------------------------------] 5/10 ( 50%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [==============================================================================================================] 10/10 (100%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> CV fold: Fold2
#> CV progress [==============================================================================>----------------------------------------] 2/3 ( 67%)
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [===========================================>-------------------------------------------------------------------] 4/10 ( 40%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [=======================================================>-------------------------------------------------------] 5/10 ( 50%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [==============================================================================================================] 10/10 (100%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> CV fold: Fold3
#> CV progress [=======================================================================================================================] 3/3 (100%)
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [===========================================>-------------------------------------------------------------------] 4/10 ( 40%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [=======================================================>-------------------------------------------------------] 5/10 ( 50%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#>  Classification: using 'mean classification error' as optimization metric.
#>
#> Parameter settings [==============================================================================================================] 10/10 (100%)
#>  Classification: using 'mean classification error' as optimization metric.

head(validator_results)
#>      fold performance num.trees  mtry min.node.size max.depth sample.fraction probability
#>    <char>       <num>     <num> <num>         <num>     <num>           <num>      <lgcl>
#> 1:  Fold1   0.9931156       500     2             6         9             0.8        TRUE
#> 2:  Fold2   0.9935853      1000     2             2         5             0.5        TRUE
#> 3:  Fold3   0.9886676       500     2             6         9             0.5        TRUE

Inner Bayesian Optimization

validator <- mlexperiments::MLNestedCV$new(
  learner = mllrnrs::LearnerRanger$new(),
  strategy = "bayesian",
  fold_list = fold_list,
  k_tuning = 3L,
  ncores = ncores,
  seed = 312
)

validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"


validator$parameter_bounds <- parameter_bounds
validator$optim_args <- optim_args

validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- TRUE

validator$set_data(
  x = train_x,
  y = train_y
)

validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.086  Round = 1   num.trees = 500.0000    mtry = 2.0000   min.node.size = 2.0000  max.depth = 9.0000  sample.fraction = 0.5000    Value = -0.05331217
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.14   Round = 2   num.trees = 1000.0000   mtry = 4.0000   min.node.size = 6.0000  max.depth = 9.0000  sample.fraction = 0.8000    Value = -0.05645683
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.096  Round = 3   num.trees = 500.0000    mtry = 2.0000   min.node.size = 6.0000  max.depth = 9.0000  sample.fraction = 0.8000    Value = -0.05645683
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.104  Round = 4   num.trees = 1000.0000   mtry = 6.0000   min.node.size = 2.0000  max.depth = 1.0000  sample.fraction = 0.8000    Value = -0.05960148
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.116  Round = 5   num.trees = 807.0000    mtry = 9.0000   min.node.size = 2.0000  max.depth = 10.0000 sample.fraction = 0.4920744 Value = -0.05645683
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.105  Round = 6   num.trees = 767.0000    mtry = 3.0000   min.node.size = 16.0000 max.depth = 23.0000 sample.fraction = 0.3697985 Value = -0.05960148
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.095  Round = 7   num.trees = 767.0000    mtry = 3.0000   min.node.size = 16.0000 max.depth = 23.0000 sample.fraction = 0.3697985 Value = -0.05960148
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.099  Round = 8   num.trees = 767.0000    mtry = 3.0000   min.node.size = 16.0000 max.depth = 23.0000 sample.fraction = 0.3697985 Value = -0.05960148
#>
#>  Best Parameters Found:
#> Round = 1    num.trees = 500.0000    mtry = 2.0000   min.node.size = 2.0000  max.depth = 9.0000  sample.fraction = 0.5000    Value = -0.05331217
#>
#> CV fold: Fold2
#> CV progress [==============================================================================>----------------------------------------] 2/3 ( 67%)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.081  Round = 1   num.trees = 500.0000    mtry = 2.0000   min.node.size = 2.0000  max.depth = 9.0000  sample.fraction = 0.5000    Value = -0.05031447
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.133  Round = 2   num.trees = 1000.0000   mtry = 4.0000   min.node.size = 6.0000  max.depth = 9.0000  sample.fraction = 0.8000    Value = -0.05031447
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.083  Round = 3   num.trees = 500.0000    mtry = 2.0000   min.node.size = 6.0000  max.depth = 9.0000  sample.fraction = 0.8000    Value = -0.05345912
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.095  Round = 4   num.trees = 1000.0000   mtry = 6.0000   min.node.size = 2.0000  max.depth = 1.0000  sample.fraction = 0.8000    Value = -0.04716981
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.116  Round = 5   num.trees = 826.0000    mtry = 9.0000   min.node.size = 15.0000 max.depth = 40.0000 sample.fraction = 1.0000    Value = -0.05660377
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.118  Round = 6   num.trees = 1000.0000   mtry = 9.0000   min.node.size = 2.0000  max.depth = 25.0000 sample.fraction = 0.3404917 Value = -0.05660377
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.102  Round = 7   num.trees = 767.0000    mtry = 3.0000   min.node.size = 16.0000 max.depth = 23.0000 sample.fraction = 0.3697985 Value = -0.04716981
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.096  Round = 8   num.trees = 913.0000    mtry = 5.0000   min.node.size = 16.0000 max.depth = 1.0000  sample.fraction = 0.3000    Value = -0.04716981
#>
#>  Best Parameters Found:
#> Round = 4    num.trees = 1000.0000   mtry = 6.0000   min.node.size = 2.0000  max.depth = 1.0000  sample.fraction = 0.8000    Value = -0.04716981
#>
#> CV fold: Fold3
#> CV progress [=======================================================================================================================] 3/3 (100%)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.091  Round = 1   num.trees = 500.0000    mtry = 2.0000   min.node.size = 2.0000  max.depth = 9.0000  sample.fraction = 0.5000    Value = -0.04414495
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.13   Round = 2   num.trees = 1000.0000   mtry = 4.0000   min.node.size = 6.0000  max.depth = 9.0000  sample.fraction = 0.8000    Value = -0.04731956
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.098  Round = 3   num.trees = 500.0000    mtry = 2.0000   min.node.size = 6.0000  max.depth = 9.0000  sample.fraction = 0.8000    Value = -0.04414495
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.096  Round = 4   num.trees = 1000.0000   mtry = 6.0000   min.node.size = 2.0000  max.depth = 1.0000  sample.fraction = 0.8000    Value = -0.04103025
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.096  Round = 5   num.trees = 767.0000    mtry = 3.0000   min.node.size = 16.0000 max.depth = 23.0000 sample.fraction = 0.3697985 Value = -0.04731956
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.065  Round = 6   num.trees = 253.0000    mtry = 8.0000   min.node.size = 12.0000 max.depth = 15.0000 sample.fraction = 0.4823285 Value = -0.0441749
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.064  Round = 7   num.trees = 245.0000    mtry = 7.0000   min.node.size = 18.0000 max.depth = 14.0000 sample.fraction = 0.3033942 Value = -0.04103025
#>
#>  Classification: using 'mean classification error' as optimization metric.
#> elapsed = 0.131  Round = 8   num.trees = 994.0000    mtry = 6.0000   min.node.size = 15.0000 max.depth = 15.0000 sample.fraction = 0.5716413 Value = -0.0441749
#>
#>  Best Parameters Found:
#> Round = 4    num.trees = 1000.0000   mtry = 6.0000   min.node.size = 2.0000  max.depth = 1.0000  sample.fraction = 0.8000    Value = -0.04103025

head(validator_results)
#>      fold performance num.trees  mtry min.node.size max.depth sample.fraction probability
#>    <char>       <num>     <num> <num>         <num>     <num>           <num>      <lgcl>
#> 1:  Fold1   0.9932921       500     2             2         9             0.5        TRUE
#> 2:  Fold2   0.9913315      1000     6             2         1             0.8        TRUE
#> 3:  Fold3   0.9876374      1000     6             2         1             0.8        TRUE

Holdout Test Dataset Performance

Predict Outcome in Holdout Test Dataset

preds_ranger <- mlexperiments::predictions(
  object = validator,
  newdata = test_x
)

Evaluate Performance on Holdout Test Dataset

perf_ranger <- mlexperiments::performance(
  object = validator,
  prediction_results = preds_ranger,
  y_ground_truth = test_y,
  type = "binary"
)
perf_ranger
#>     model performance       AUC      Brier BrierScaled       BAC    TP    TN    FP    FN       TPR       TNR        FPR        FNR       PPV
#>    <char>       <num>     <num>      <num>       <num>     <num> <int> <int> <int> <int>     <num>     <num>      <num>      <num>     <num>
#> 1:  Fold1   0.9888060 0.9888060 0.04307263   0.8105483 0.9508706    66   132     2     6 0.9166667 0.9850746 0.01492537 0.08333333 0.9705882
#> 2:  Fold2   0.9795813 0.9795813 0.07807664   0.6565858 0.9257877    64   129     5     8 0.8888889 0.9626866 0.03731343 0.11111111 0.9275362
#> 3:  Fold3   0.9795813 0.9795813 0.07868587   0.6539061 0.9220564    64   128     6     8 0.8888889 0.9552239 0.04477612 0.11111111 0.9142857
#>          NPV        FDR       MCC        F1     GMEAN       GPR       ACC       MMCE        BER
#>        <num>      <num>     <num>     <num>     <num>     <num>     <num>      <num>      <num>
#> 1: 0.9565217 0.02941176 0.9143377 0.9428571 0.9502553 0.9432422 0.9611650 0.03883495 0.04912935
#> 2: 0.9416058 0.07246377 0.8603139 0.9078014 0.9250521 0.9080070 0.9368932 0.06310680 0.07421227
#> 3: 0.9411765 0.08571429 0.8497685 0.9014085 0.9214597 0.9014979 0.9320388 0.06796117 0.07794362