AE 21: Monitor models
Suggested answers
Application exercise
Answers
Example model monitoring dashboard
Load the data
Split the data
# split the data into training/validation/test sets
housing_split <- initial_validation_time_split(data = housing, prop = c(0.6, 0.095))
housing_train <- training(housing_split) |>
write_csv(file = "data/housing_train.csv")
housing_val <- validation(housing_split) |>
write_csv(file = "data/housing_val.csv")
housing_monitor <- testing(housing_split) |>
write_csv(file = "data/housing_monitor.csv")
# split housing_train into rolling origin folds
housing_folds <- rolling_origin(housing_train, initial = 100, assess = 100, skip = 50)Fit the model
# tune a boosted tree model
boost_rec <- recipe(price ~ ., data = housing_train) |>
step_impute_mean(all_numeric_predictors()) |>
step_impute_mode(all_nominal_predictors())
boost_spec <- boost_tree(
mtry = tune(), trees = 1000, min_n = tune(), tree_depth = tune(),
learn_rate = tune(), loss_reduction = tune(), sample_size = tune(),
stop_iter = tune()
) |>
set_engine("lightgbm") |>
set_mode("regression")
boost_wf <- workflow() |>
add_recipe(boost_rec) |>
add_model(boost_spec)
housing_tune <- tune_bayes(
boost_wf,
housing_folds,
iter = 10,
initial = 8,
param_info = extract_parameter_set_dials(boost_wf) |>
finalize(x = housing_train),
control = control_bayes(
verbose_iter = TRUE,
save_workflow = TRUE
)
)# evaluate performance
autoplot(housing_tune)
show_best(housing_tune)# A tibble: 5 × 14
mtry min_n tree_depth learn_rate loss_reduction sample_size stop_iter .metric
<int> <int> <int> <dbl> <dbl> <dbl> <int> <chr>
1 1 6 8 0.0510 5.72e-10 0.683 20 rmse
2 4 9 5 0.0762 3.71e-10 0.989 19 rmse
3 1 6 7 0.0131 1.32e-10 0.266 16 rmse
4 3 11 15 0.0876 2.47e-10 0.623 20 rmse
5 4 20 4 0.0769 1.24e-10 0.917 3 rmse
# ℹ 6 more variables: .estimator <chr>, mean <dbl>, n <int>, std_err <dbl>,
# .config <chr>, .iter <int>housing_fit <- fit_best(housing_tune)
# calculate validation set metrics
housing_metrics <- housing_fit |>
augment(new_data = housing_val) |>
metrics(truth = price, estimate = .pred)Pin the model
v <- vetiver_model(
model = housing_fit,
model_name = "tompkins-housing",
description = "A LightGBM model to predict Tompkins County home prices. Trained on data from 2022-2023.",
metadata = list(metrics = housing_metrics)
)
v
── tompkins-housing ─ <bundled_workflow> model for deployment
A LightGBM model to predict Tompkins County home prices. Trained on data from
2022-2023. using 11 featureslibrary(googleCloudStorageR)
board <- board_gcs(bucket = "info-4940-models", prefix = "bcs88/")
board |>
vetiver_pin_write(v)
board |>
pin_meta("tompkins-housing")List of 13
$ file : chr "tompkins-housing.rds"
$ file_size : 'fs_bytes' int 425K
$ pin_hash : chr "17d9f38cb713073a"
$ type : chr "rds"
$ title : chr "tompkins-housing: a pinned list"
$ description: chr "A LightGBM model to predict Tompkins County home prices. Trained on data from 2022-2023."
$ tags : NULL
$ urls : NULL
$ created : POSIXct[1:1], format: "2024-12-05 11:58:45"
$ api_version: int 1
$ user :List of 3
..$ metrics :List of 3
.. ..$ .metric : chr [1:3] "rmse" "rsq" "mae"
.. ..$ .estimator: chr [1:3] "standard" "standard" "standard"
.. ..$ .estimate : num [1:3] 0.172 0.65 0.12
..$ required_pkgs: chr [1:5] "bonsai" "lightgbm" "parsnip" "recipes" ...
..$ renv_lock : NULL
$ name : chr "tompkins-housing"
$ local :List of 3
..$ dir : 'fs_path' chr "~/Library/Caches/pins/gcs-info-4940-models/tompkins-housing/20241205T165845Z-17d9f"
..$ url : NULL
..$ version: chr "20241205T165845Z-17d9f"board |>
pin_write(x = housing_monitor, name = "tompkins-housing-new-sales")Generate Docker artifacts
vetiver_write_plumber(
board = board,
name = "tompkins-housing"
)
vetiver_write_docker(
vetiver_model = v,
docker_args = list(port = 2300),
additional_pkgs = required_pkgs(board)
)Monitor your model’s inputs
Your turn: Create a plot or table comparing the development vs. monitoring distributions of a model input/feature. How might you make this comparison if you didn’t have all the model development data available when monitoring? What summary statistics might you record during model development, to prepare for monitoring?
ggplot(data = housing, mapping = aes(x = 10^price, y = after_stat(density), fill = monitor)) +
geom_histogram(alpha = 0.5, position = "identity") +
scale_x_log10(labels = label_dollar(scale_cut = cut_short_scale())) +
scale_fill_discrete_qualitative() +
labs(x = "Price", fill = NULL)
ggplot(data = housing, mapping = aes(y = town |> fct_rev(), x = 10^price, fill = monitor)) +
geom_boxplot(alpha = 0.8, position = "dodge") +
scale_x_log10(labels = label_dollar(scale_cut = cut_short_scale())) +
scale_fill_discrete_qualitative() +
labs(x = "Price", y = "Town", fill = NULL)
Monitor your model’s outputs
Your turn: Use the functions for metrics monitoring from {vetiver} to create a monitoring visualization. Choose a different set of metrics or time aggregation.
library(vetiver)
library(tidymodels)
url <- "http://appliedml.infosci.cornell.edu:2300/predict"
endpoint <- vetiver_endpoint(url)
augment(endpoint, new_data = housing_monitor) |>
vetiver_compute_metrics(
date_var = sold_date,
period = "month",
truth = price,
estimate = .pred,
metric_set = metric_set(rmse, rsq, mae)
) |>
vetiver_plot_metrics()
Compute ML metrics
Your turn: Compute the mean absolute percentage error with the monitoring data, and aggregate by week/month, number of bedrooms/bathrooms, or town location.
If you have time, make a visualization showing your results.
# convert back to raw dollars for calculating percentage error
augment(endpoint, housing_monitor) |>
mutate(
.pred = 10^.pred,
price = 10^price
) |>
group_by(town) |>
mape(price, .pred)# A tibble: 12 × 4
town .metric .estimator .estimate
<chr> <chr> <chr> <dbl>
1 Caroline mape standard 22.8
2 Cortlandville mape standard 5.88
3 Danby mape standard 47.1
4 Dryden mape standard 31.5
5 Enfield mape standard 21.1
6 Groton mape standard 47.2
7 Harford mape standard 113.
8 Hector mape standard 19.8
9 Ithaca mape standard 20.3
10 Lansing mape standard 24.4
11 Newfield mape standard 24.3
12 Ulysses mape standard 27.4 Acknowledgments
- Materials derived in part from Intro to MLOps with {vetiver} and licensed under a Creative Commons Attribution 4.0 International (CC BY) License.
Session information
sessioninfo::session_info()─ Session info ───────────────────────────────────────────────────────────────
setting value
version R version 4.4.1 (2024-06-14)
os macOS Sonoma 14.6.1
system aarch64, darwin20
ui X11
language (EN)
collate en_US.UTF-8
ctype en_US.UTF-8
tz America/New_York
date 2024-12-05
pandoc 3.4 @ /usr/local/bin/ (via rmarkdown)
─ Packages ───────────────────────────────────────────────────────────────────
package * version date (UTC) lib source
archive 1.1.9 2024-09-12 [1] CRAN (R 4.4.1)
askpass 1.2.1 2024-10-04 [1] CRAN (R 4.4.1)
assertthat 0.2.1 2019-03-21 [1] CRAN (R 4.3.0)
backports 1.5.0 2024-05-23 [1] CRAN (R 4.4.0)
bit 4.0.5 2022-11-15 [1] CRAN (R 4.3.0)
bit64 4.0.5 2020-08-30 [1] CRAN (R 4.3.0)
bonsai * 0.3.1 2024-07-23 [1] CRAN (R 4.4.0)
broom * 1.0.6 2024-05-17 [1] CRAN (R 4.4.0)
bundle 0.1.1 2023-09-09 [1] CRAN (R 4.4.0)
butcher 0.3.4 2024-04-11 [1] CRAN (R 4.4.0)
cachem 1.1.0 2024-05-16 [1] CRAN (R 4.4.0)
class 7.3-22 2023-05-03 [1] CRAN (R 4.4.0)
cli 3.6.3 2024-06-21 [1] CRAN (R 4.4.0)
codetools 0.2-20 2024-03-31 [1] CRAN (R 4.4.1)
colorspace * 2.1-1 2024-07-26 [1] CRAN (R 4.4.0)
crayon 1.5.3 2024-06-20 [1] CRAN (R 4.4.0)
curl 5.2.3 2024-09-20 [1] CRAN (R 4.4.1)
data.table 1.15.4 2024-03-30 [1] CRAN (R 4.3.1)
dials * 1.3.0 2024-07-30 [1] CRAN (R 4.4.0)
DiceDesign 1.10 2023-12-07 [1] CRAN (R 4.3.1)
dichromat 2.0-0.1 2022-05-02 [1] CRAN (R 4.3.0)
digest 0.6.35 2024-03-11 [1] CRAN (R 4.3.1)
dplyr * 1.1.4 2023-11-17 [1] CRAN (R 4.3.1)
ellipsis 0.3.2 2021-04-29 [1] CRAN (R 4.3.0)
evaluate 0.24.0 2024-06-10 [1] CRAN (R 4.4.0)
fansi 1.0.6 2023-12-08 [1] CRAN (R 4.3.1)
farver 2.1.2 2024-05-13 [1] CRAN (R 4.3.3)
fastmap 1.2.0 2024-05-15 [1] CRAN (R 4.4.0)
forcats * 1.0.0 2023-01-29 [1] CRAN (R 4.3.0)
foreach 1.5.2 2022-02-02 [1] CRAN (R 4.3.0)
fs 1.6.4 2024-04-25 [1] CRAN (R 4.4.0)
furrr 0.3.1 2022-08-15 [1] CRAN (R 4.3.0)
future * 1.33.2 2024-03-26 [1] CRAN (R 4.3.1)
future.apply 1.11.2 2024-03-28 [1] CRAN (R 4.3.1)
gargle 1.5.2 2023-07-20 [1] CRAN (R 4.3.0)
generics 0.1.3 2022-07-05 [1] CRAN (R 4.3.0)
ggplot2 * 3.5.1 2024-04-23 [1] CRAN (R 4.3.1)
globals 0.16.3 2024-03-08 [1] CRAN (R 4.3.1)
glue 1.8.0 2024-09-30 [1] CRAN (R 4.4.1)
googleAuthR 2.0.2 2024-05-22 [1] CRAN (R 4.4.0)
googleCloudStorageR * 0.7.0 2021-12-16 [1] CRAN (R 4.4.0)
gower 1.0.1 2022-12-22 [1] CRAN (R 4.3.0)
GPfit 1.0-8 2019-02-08 [1] CRAN (R 4.3.0)
gtable 0.3.5 2024-04-22 [1] CRAN (R 4.3.1)
hardhat 1.4.0 2024-06-02 [1] CRAN (R 4.4.0)
here 1.0.1 2020-12-13 [1] CRAN (R 4.3.0)
hms 1.1.3 2023-03-21 [1] CRAN (R 4.3.0)
htmltools 0.5.8.1 2024-04-04 [1] CRAN (R 4.3.1)
htmlwidgets 1.6.4 2023-12-06 [1] CRAN (R 4.3.1)
httr 1.4.7 2023-08-15 [1] CRAN (R 4.3.0)
infer * 1.0.7 2024-03-25 [1] CRAN (R 4.3.1)
ipred 0.9-14 2023-03-09 [1] CRAN (R 4.3.0)
iterators 1.0.14 2022-02-05 [1] CRAN (R 4.3.0)
jsonlite 1.8.9 2024-09-20 [1] CRAN (R 4.4.1)
knitr 1.47 2024-05-29 [1] CRAN (R 4.4.0)
labeling 0.4.3 2023-08-29 [1] CRAN (R 4.3.0)
later 1.3.2 2023-12-06 [1] CRAN (R 4.3.1)
lattice 0.22-6 2024-03-20 [1] CRAN (R 4.4.0)
lava 1.8.0 2024-03-05 [1] CRAN (R 4.3.1)
lhs 1.1.6 2022-12-17 [1] CRAN (R 4.3.0)
lifecycle 1.0.4 2023-11-07 [1] CRAN (R 4.3.1)
lightgbm 4.4.0 2024-06-15 [1] CRAN (R 4.4.0)
listenv 0.9.1 2024-01-29 [1] CRAN (R 4.3.1)
lubridate * 1.9.3 2023-09-27 [1] CRAN (R 4.3.1)
magrittr 2.0.3 2022-03-30 [1] CRAN (R 4.3.0)
MASS 7.3-61 2024-06-13 [1] CRAN (R 4.4.0)
Matrix 1.7-0 2024-03-22 [1] CRAN (R 4.4.0)
memoise 2.0.1 2021-11-26 [1] CRAN (R 4.3.0)
mime 0.12 2021-09-28 [1] CRAN (R 4.3.0)
modeldata * 1.4.0 2024-06-19 [1] CRAN (R 4.4.0)
nnet 7.3-19 2023-05-03 [1] CRAN (R 4.4.0)
openssl 2.2.2 2024-09-20 [1] CRAN (R 4.4.1)
parallelly 1.37.1 2024-02-29 [1] CRAN (R 4.3.1)
parsnip * 1.2.1 2024-03-22 [1] CRAN (R 4.3.1)
pillar 1.9.0 2023-03-22 [1] CRAN (R 4.3.0)
pins * 1.3.0 2023-11-09 [1] CRAN (R 4.4.0)
pkgconfig 2.0.3 2019-09-22 [1] CRAN (R 4.3.0)
plumber 1.2.2 2024-03-26 [1] CRAN (R 4.4.0)
prodlim 2023.08.28 2023-08-28 [1] CRAN (R 4.3.0)
promises 1.3.0 2024-04-05 [1] CRAN (R 4.3.1)
purrr * 1.0.2 2023-08-10 [1] CRAN (R 4.3.0)
R6 2.5.1 2021-08-19 [1] CRAN (R 4.3.0)
rappdirs 0.3.3 2021-01-31 [1] CRAN (R 4.3.0)
RColorBrewer 1.1-3 2022-04-03 [1] CRAN (R 4.3.0)
Rcpp 1.0.13 2024-07-17 [1] CRAN (R 4.4.0)
readr * 2.1.5 2024-01-10 [1] CRAN (R 4.3.1)
recipes * 1.0.10 2024-02-18 [1] CRAN (R 4.3.1)
rlang 1.1.4 2024-06-04 [1] CRAN (R 4.3.3)
rmarkdown 2.27 2024-05-17 [1] CRAN (R 4.4.0)
rpart 4.1.23 2023-12-05 [1] CRAN (R 4.4.0)
rprojroot 2.0.4 2023-11-05 [1] CRAN (R 4.3.1)
rsample * 1.2.1 2024-03-25 [1] CRAN (R 4.3.1)
rstudioapi 0.17.0 2024-10-16 [1] CRAN (R 4.4.1)
scales * 1.3.0.9000 2024-11-14 [1] Github (r-lib/scales@ee03582)
sessioninfo 1.2.2 2021-12-06 [1] CRAN (R 4.3.0)
slider 0.3.1 2023-10-12 [1] CRAN (R 4.3.1)
stringi 1.8.4 2024-05-06 [1] CRAN (R 4.3.1)
stringr * 1.5.1 2023-11-14 [1] CRAN (R 4.3.1)
survival 3.7-0 2024-06-05 [1] CRAN (R 4.4.0)
swagger 5.17.14.1 2024-06-28 [1] CRAN (R 4.4.0)
tibble * 3.2.1 2023-03-20 [1] CRAN (R 4.3.0)
tidymodels * 1.2.0 2024-03-25 [1] CRAN (R 4.3.1)
tidyr * 1.3.1 2024-01-24 [1] CRAN (R 4.3.1)
tidyselect 1.2.1 2024-03-11 [1] CRAN (R 4.3.1)
tidyverse * 2.0.0 2023-02-22 [1] CRAN (R 4.3.0)
timechange 0.3.0 2024-01-18 [1] CRAN (R 4.3.1)
timeDate 4032.109 2023-12-14 [1] CRAN (R 4.3.1)
tune * 1.2.1 2024-04-18 [1] CRAN (R 4.3.1)
tzdb 0.4.0 2023-05-12 [1] CRAN (R 4.3.0)
utf8 1.2.4 2023-10-22 [1] CRAN (R 4.3.1)
vctrs 0.6.5 2023-12-01 [1] CRAN (R 4.3.1)
vetiver * 0.2.5 2023-11-16 [1] CRAN (R 4.4.0)
vroom 1.6.5 2023-12-05 [1] CRAN (R 4.3.1)
warp 0.2.1 2023-11-02 [1] CRAN (R 4.3.1)
webutils 1.2.0 2023-11-24 [1] CRAN (R 4.4.0)
withr 3.0.2 2024-10-28 [1] CRAN (R 4.4.1)
workflows * 1.1.4 2024-02-19 [1] CRAN (R 4.4.0)
workflowsets * 1.1.0 2024-03-21 [1] CRAN (R 4.3.1)
xfun 0.45 2024-06-16 [1] CRAN (R 4.4.0)
yaml 2.3.10 2024-07-26 [1] CRAN (R 4.4.0)
yardstick * 1.3.1 2024-03-21 [1] CRAN (R 4.3.1)
zip 2.3.1 2024-01-27 [1] CRAN (R 4.4.0)
[1] /Library/Frameworks/R.framework/Versions/4.4-arm64/Resources/library
──────────────────────────────────────────────────────────────────────────────