start model card

Author: isabelizimm

vetiver/__init__.py

@@ -17,6 +17,7 @@
 from .handlers.statsmodels import StatsmodelsHandler  # noqa
 from .rsconnect import deploy_rsconnect  # noqa
 from .monitor import compute_metrics, pin_metrics, plot_metrics, _rolling_df  # noqa
+from .model_card import model_card  # noqa
 
 __author__ = "Isabel Zimmerman <isabel.zimmerman@rstudio.com>"
 __all__ = []

vetiver/model_card.py

@@ -0,0 +1,21 @@
+import logging
+from importlib_resources import files as _files
+import shutil
+
+_log = logging.getLogger(__name__)
+
+
+def model_card(path="."):
+    """Create a model card for documentation
+
+    Parameters
+    ----------
+    path : str
+       Path to save model card
+    """
+    src_path = _files("vetiver") / "templates/model_card.qmd"
+
+    if _log is not None:
+        _log.info("Writing model card template...")
+
+    return shutil.copy(src=src_path, dst=path)

vetiver/pin_read_write.py

@@ -37,7 +37,8 @@ def vetiver_pin_write(board, model: VetiverModel, versioned: bool = True):
     # message = """
     # Create a Model Card for your published model.
     # Model Cards provide a framework for transparent, responsible reporting.
-    # Use the vetiver `.Rmd` template as a place to start."""
+    # Use the vetiver `.qmd` Quarto template as a place to start,
+    # with vetiver.model_card()"""
 
     # warnings.warn(message=message)
 

vetiver/templates/model_card.qmd

@@ -0,0 +1,129 @@
+---
+title: "Model Card: your model title"
+date: '`python date.today()`'
+output:
+  html_document
+params:
+    board: !python pins.board_rsconnect()
+    name: isabel.zimmerman/sacramento_rf
+    version: NULL
+---
+
+```{python setup, include=FALSE}
+from datetime import date
+from sklearn import metrics
+import vetiver
+import pins
+
+# library(yardstick)
+# knitr::opts_chunk$set(echo = FALSE)
+v = vetiver.VetiverModel.from_pin(pins.board_rsconnect(server_url = "https://colorado.rstudio.com/rsc/"), "isabel.zimmerman/cars_mpg")
+```
+
+A [model card](https://doi.org/10.1145/3287560.3287596) provides brief, transparent, responsible reporting for a trained machine learning model.
+
+## Model details
+
+- Developed by PERSON AND/OR TEAM
+- `python print(f"{v.description} using {v.ptype} features")`
+- More details about how model was developed and what it is predicting
+- More details on feature engineering and/or data preprocessing for model
+- Version `python v.metadata.version` of this model was published at `r v_meta.created`
+- Citation and/or license details for the model
+- If you have questions about this model, please contact PERSON@ORG.ORG
+
+## Intended use
+
+- The primary intended uses of this model are ...
+- The primary intended users of this model are ...
+- Some use cases are out of scope for this model, such as ...
+
+## Important aspects/factors
+
+- Aspects or factors (demographic, environmental, technical) that are relevant to the context of this model are ...
+- In evaluating this model, we examined aspects such as ...
+
+## Metrics
+
+- The metrics used to evaluate this model are ...
+- These metrics are computed via ...
+- We chose these metrics because ...
+
+## Training data & evaluation data
+
+- The training dataset for this model was ...
+- The training dataset for this model has the "prototype" or signature:
+
+```{python}
+v.ptype
+```
+
+- The evaluation dataset used in this model card is ...
+- We chose this evaluation data because ...
+
+```{python}
+## EVALUATION DATA:
+from vetiver.data import mtcars
+## consider using a package like skimr or DataExplorer for automated
+## presentation of evaluation data characteristics
+```
+
+
+## Quantitative analyses {.tabset}
+
+```{python}
+## compute predictions for your evaluation data
+## `handler_startup` is designed to get the R process ready to make predictions
+#suppressPackageStartupMessages(handler_startup(v))
+new_data["preds"] = v.model.predict(mtcars.drop(columns="mpg"))
+```
+
+
+### Overall model performance
+
+```{python}
+
+metric_set = [metrics.mean_absolute_error, metrics.mean_squared_error]
+
+```
+
+### Disaggregated model performance
+
+```{python}
+
+preds %>%
+    group_by(type) %>%
+    metrics(price, .pred)
+```
+
+### Visualize model performance
+
+```{python, fig.height=3}
+preds %>%
+    ggplot(aes(price, .pred, color = type)) +
+    geom_abline(slope = 1, lty = 2, color = "gray60", size = 1.2) +
+    geom_point(alpha = 0.5, show.legend = FALSE) +
+    facet_wrap(vars(type))
+```
+
+### Make a custom plot
+
+```{r}
+preds %>%
+    mutate(.resid = price - .pred) %>%
+    ggplot(aes(longitude, latitude, color = .resid)) +
+    geom_point(alpha = 0.8) +
+    scale_color_gradient2() +
+    coord_fixed()
+```
+
+
+## Ethical considerations
+
+- We considered ...
+
+## Caveats & recommendations
+
+- This model does ...
+- This model does not ...
+- We recommend ...