pin_metrics to facet

Author: isabelizimm

vetiver/__init__.py

@@ -4,6 +4,7 @@
 __author__ = "Isabel Zimmerman <isabel.zimmerman@rstudio.com>"
 __all__ = []
 
+<<<<<<< HEAD
 import importlib  # noqa
 from .ptype import *  # noqa
 from .vetiver_model import VetiverModel  # noqa
@@ -20,3 +21,19 @@
 from .handlers.torch import TorchHandler  # noqa
 from .rsconnect import deploy_rsconnect # noqa
 from .monitor import compute_metrics, pin_metrics, plot_metrics # noqa
+=======
+import importlib
+from .ptype import *
+from .vetiver_model import *
+from .server import *
+from .mock import *
+from .pin_read_write import *
+from .attach_pkgs import *
+from .meta import *
+from .write_docker import *
+from .write_fastapi import *
+from .handlers._interface import *
+from .handlers.sklearn_vt import *
+from .handlers.pytorch_vt import *
+from .monitor import compute_metrics, plot_metrics, pin_metrics
+>>>>>>> f936189 (pin_metrics to facet)

vetiver/monitor.py

@@ -1,80 +1,118 @@
+import datetime
 import pins
+from pins import PinsError
 import matplotlib.pyplot as plt
 import seaborn as sns
-import numpy as np
 import pandas as pd
 
 def compute_metrics(data, 
                     date_var, 
                     period,
                     metric_set, 
                     truth, 
-                    pred):
-    
+                    estimate):
+    """
+    Compute metrics for given time period
+
+    Parameters
+    ----------
+    data : DataFrame
+        Pandas dataframe
+    date_var: 
+        Column in `data` containing dates
+    period:
+        Defining period to group by 
+        https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases
+    metric_set: list
+        List of metrics to compute 
+        https://scikit-learn.org/stable/modules/classes.html#module-sklearn.metrics
+    truth:
+        Column identifier for true results
+    estimate:
+        Column identifier for predicted results
+
+    """
     output = pd.DataFrame()
-    rol = data.set_index(date_var)
-    rol = rol.rolling(window=period)
+    dates_sorted = data.set_index(date_var).sort_index()
+    rol = dates_sorted.set_index(truth).rolling(window=period)
 
     for metric in metric_set:
-        rol.apply(
+        rol[metric.__name__] =rol.apply(
             _compute, 
             raw=False, 
-            kwargs = {'df':output,
-            'truth': truth,
-            'pred': pred,
+            kwargs = {
             'metric': metric}
             )
 
+    rol.set_index(date_var)
+
     return output
 
 def _compute(window, 
-            df, 
-            truth, 
-            pred, 
             metric):
-    output = metric(y_true=window, 
-                    # TO DO: change y pred to correct piece
-                    y_pred=np.random.randint(0,1000,size=(len(window),1)))
-    df.loc[window.index.max(), metric.__name__] = output
-    return output
-
-
-# def pin_metrics(board,
-#                                 df_metrics,
-#                                 metrics_pin_name,
-#                                 #.index = .index,
-#                                 overwrite = True):
-    
-# #    date_var <- quo_name(enquo(date_var)) #enquo?
 
-#     new_metrics = df_metrics.sort()
-
-#     new_dates = df_metrics.index.unique()
+    output = metric(y_true = window.index,
+                    y_pred=window)
+    return output
 
-#     old_metrics = pins.pin_read(board, metrics_pin_name)
-#     overlapping_dates = old_metrics.index in new_dates
 
-#     if overwrite is True:
-#         old_metrics = old_metrics not in overlapping_dates
-#     else:
-#         if overlapping_dates:
-#             raise ValueError(f"The new metrics overlap with dates \
-#                      already stored in {repr(metrics_pin_name)} \
-#                      Check the aggregated dates or use `overwrite = True`"
-#             )
+def pin_metrics(board,
+                df_metrics,
+                metrics_pin_name,
+                overwrite = False):
+    """
+    Update an existing pin storing model metrics over time
+
+    Parameters
+    ----------
+    board : 
+        Pins board
+    df_metrics: pd.DataFrame
+        Dataframe of metrics over time, such as created by `vetiver_compute_metrics()`
+    metrics_pin_name:
+        Pin name for where the metrics are stored
+    overwrite: bool
+        If TRUE (the default), overwrite any metrics for 
+        dates that exist both in the existing pin and 
+        new metrics with the new values. If FALSE, error 
+        when the new metrics contain overlapping dates with 
+        the existing pin.
+    """
+    date_types = (datetime.date, datetime.time, datetime.datetime)
+    if not isinstance(df_metrics.index, date_types):
+        try: 
+            df_metrics = df_metrics.index.astype('datetime')
+        except TypeError:
+            raise TypeError("Index must be a date type")
+
+    new_metrics = df_metrics.sort_index()
+
+    new_dates = df_metrics.index.unique()
+
+    try:
+        old_metrics = board.pin_read(metrics_pin_name)
+    except PinsError:
+        board.pin_write(metrics_pin_name)
+
+    overlapping_dates = old_metrics.index in new_dates
+
+    if overwrite is True:
+        old_metrics = old_metrics not in overlapping_dates
+    else:
+        if overlapping_dates:
+            raise ValueError(f"The new metrics overlap with dates \
+                     already stored in {repr(metrics_pin_name)} \
+                     Check the aggregated dates or use `overwrite = True`"
+            )
 
-#     new_metrics = old_metrics + df_metrics
-#     new_metrics <- vec_slice(
-#         new_metrics,
-#         vctrs::vec_order(new_metrics.index)
-#     )
+    new_metrics = old_metrics + df_metrics
+    new_metrics = new_metrics.sort_index()
 
-#     pins.pin_write(board, new_metrics, metrics_pin_name)
+    pins.pin_write(board, new_metrics, metrics_pin_name)
 
 
-def vetiver_plot_metrics(df_metrics,
-                        date_var,
-                        metric):
+def plot_metrics(df_metrics,
+                        date_var, metric):
 
     ncols = 1
     nrows = len(metric)
@@ -83,11 +121,24 @@ def vetiver_plot_metrics(df_metrics,
         nrows=nrows,
         ncols=ncols
     )
+    """
+    Plot metrics over a given time period
+
+    Parameters
+    ----------
+    df_metrics : DataFrame
+        Pandas dataframe of metrics over time, such as created by `compute_metircs()`
+    date_var: 
+        Column in `data` containing dates
+    metric_set: list
+        List of metrics to compute
+
+    """
 
     # loop for plotting each column
     for i, col in enumerate(df_metrics.columns):
         sns.lineplot(
-            x=df_metrics[date_var], 
+            x=date_var, 
             y=df_metrics[col], 
             ax=axes[i], 
             color='royalblue'