Merge pull request #266 from interpretml/gaugup/ReplaceBostonHousingDataset

Replace load_boston() with fetch_california_housing()

Author: gaugup

docs/source/notebooks/DiCE_multiclass_classification_and_regression.ipynb

@@ -19,7 +19,7 @@
     "import dice_ml\n",
     "from dice_ml import Dice\n",
     "\n",
-    "from sklearn.datasets import load_iris, load_boston\n",
+    "from sklearn.datasets import load_iris, fetch_california_housing\n",
     "from sklearn.pipeline import Pipeline\n",
     "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
     "from sklearn.model_selection import train_test_split\n",
@@ -162,7 +162,7 @@
    "outputs": [],
    "source": [
     "# Single input\n",
-    "query_instances_iris = x_train[2:3]\n",
+    "query_instances_iris = x_test[2:3]\n",
     "genetic_iris = exp_genetic_iris.generate_counterfactuals(query_instances_iris, total_CFs=7, desired_class=2)\n",
     "genetic_iris.visualize_as_dataframe()"
    ]
@@ -174,7 +174,7 @@
    "outputs": [],
    "source": [
     "# Multiple queries can be given as input at once\n",
-    "query_instances_iris = x_train[17:19]\n",
+    "query_instances_iris = x_test[17:19]\n",
     "genetic_iris = exp_genetic_iris.generate_counterfactuals(query_instances_iris, total_CFs=7, desired_class=2)\n",
     "genetic_iris.visualize_as_dataframe(show_only_changes=True)"
    ]
@@ -190,7 +190,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "For regression, we will use sklearn's boston dataset. This dataset contains boston house-prices. More information at https://scikit-learn.org/stable/datasets/toy_dataset.html#boston-house-prices-dataset"
+    "For regression, we will use sklearn's California Housing dataset. This dataset contains California house prices. More information at https://scikit-learn.org/stable/modules/generated/sklearn.datasets.fetch_california_housing.html"
    ]
   },
   {
@@ -199,10 +199,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "boston_data = load_boston()\n",
-    "df_boston = pd.DataFrame(boston_data.data, columns=boston_data.feature_names)\n",
-    "df_boston[outcome_name] = pd.Series(boston_data.target)\n",
-    "df_boston.head()"
+    "housing_data = fetch_california_housing()\n",
+    "df_housing = pd.DataFrame(housing_data.data, columns=housing_data.feature_names)\n",
+    "df_housing[outcome_name] = pd.Series(housing_data.target)\n",
+    "df_housing.head()"
    ]
   },
   {
@@ -211,7 +211,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "df_boston.info()"
+    "df_housing.info()"
    ]
   },
   {
@@ -220,8 +220,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "continuous_features_boston = df_boston.drop(outcome_name, axis=1).columns.tolist()\n",
-    "target = df_boston[outcome_name]"
+    "continuous_features_housing = df_housing.drop(outcome_name, axis=1).columns.tolist()\n",
+    "target = df_housing[outcome_name]"
    ]
   },
   {
@@ -231,13 +231,13 @@
    "outputs": [],
    "source": [
     "# Split data into train and test\n",
-    "datasetX = df_boston.drop(outcome_name, axis=1)\n",
+    "datasetX = df_housing.drop(outcome_name, axis=1)\n",
     "x_train, x_test, y_train, y_test = train_test_split(datasetX,\n",
     "                                                    target,\n",
     "                                                    test_size=0.2,\n",
     "                                                    random_state=0)\n",
     "\n",
-    "categorical_features = x_train.columns.difference(continuous_features_boston)\n",
+    "categorical_features = x_train.columns.difference(continuous_features_housing)\n",
     "\n",
     "# We create the preprocessing pipelines for both numeric and categorical data.\n",
     "numeric_transformer = Pipeline(steps=[\n",
@@ -248,14 +248,14 @@
     "\n",
     "transformations = ColumnTransformer(\n",
     "    transformers=[\n",
-    "        ('num', numeric_transformer, continuous_features_boston),\n",
+    "        ('num', numeric_transformer, continuous_features_housing),\n",
     "        ('cat', categorical_transformer, categorical_features)])\n",
     "\n",
     "# Append classifier to preprocessing pipeline.\n",
     "# Now we have a full prediction pipeline.\n",
-    "regr_boston = Pipeline(steps=[('preprocessor', transformations),\n",
-    "                              ('regressor', RandomForestRegressor())])\n",
-    "model_boston = regr_boston.fit(x_train, y_train)"
+    "regr_housing = Pipeline(steps=[('preprocessor', transformations),\n",
+    "                               ('regressor', RandomForestRegressor())])\n",
+    "model_housing = regr_housing.fit(x_train, y_train)"
    ]
   },
   {
@@ -264,9 +264,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "d_boston = dice_ml.Data(dataframe=df_boston, continuous_features=continuous_features_boston, outcome_name=outcome_name)\n",
+    "d_housing = dice_ml.Data(dataframe=df_housing, continuous_features=continuous_features_housing, outcome_name=outcome_name)\n",
     "# We provide the type of model as a parameter (model_type)\n",
-    "m_boston = dice_ml.Model(model=model_boston, backend=\"sklearn\", model_type='regressor')"
+    "m_housing = dice_ml.Model(model=model_housing, backend=\"sklearn\", model_type='regressor')"
    ]
   },
   {
@@ -275,7 +275,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "exp_genetic_boston = Dice(d_boston, m_boston, method=\"genetic\")"
+    "exp_genetic_housing = Dice(d_housing, m_housing, method=\"genetic\")"
    ]
   },
   {
@@ -292,23 +292,11 @@
    "outputs": [],
    "source": [
     "# Multiple queries can be given as input at once\n",
-    "query_instances_boston = x_train[2:3]\n",
-    "genetic_boston = exp_genetic_boston.generate_counterfactuals(query_instances_boston,\n",
-    "                                                             total_CFs=2,\n",
-    "                                                             desired_range=[30, 45])\n",
-    "genetic_boston.visualize_as_dataframe(show_only_changes=True)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Multiple queries can be given as input at once\n",
-    "query_instances_boston = x_train[17:19]\n",
-    "genetic_boston = exp_genetic_boston.generate_counterfactuals(query_instances_boston, total_CFs=4, desired_range=[40, 50])\n",
-    "genetic_boston.visualize_as_dataframe(show_only_changes=True)"
+    "query_instances_housing = x_test[2:4]\n",
+    "genetic_housing = exp_genetic_housing.generate_counterfactuals(query_instances_housing,\n",
+    "                                                               total_CFs=2,\n",
+    "                                                               desired_range=[3.0, 5.0])\n",
+    "genetic_housing.visualize_as_dataframe(show_only_changes=True)"
    ]
   }
  ],

tests/conftest.py

@@ -2,7 +2,7 @@
 
 import pandas as pd
 import pytest
-from sklearn.datasets import load_boston, load_iris
+from sklearn.datasets import fetch_california_housing, load_iris
 from sklearn.model_selection import train_test_split
 
 import dice_ml
@@ -219,11 +219,11 @@ def create_iris_data():
 
 
 @pytest.fixture
-def create_boston_data():
-    boston = load_boston()
+def create_housing_data():
+    housing = fetch_california_housing()
     x_train, x_test, y_train, y_test = train_test_split(
-        boston.data, boston.target,
+        housing.data, housing.target,
         test_size=0.2, random_state=7)
-    x_train = pd.DataFrame(data=x_train, columns=boston.feature_names)
-    x_test = pd.DataFrame(data=x_test, columns=boston.feature_names)
-    return x_train, x_test, y_train, y_test, boston.feature_names.tolist()
+    x_train = pd.DataFrame(data=x_train, columns=housing.feature_names)
+    x_test = pd.DataFrame(data=x_test, columns=housing.feature_names)
+    return x_train, x_test, y_train, y_test, housing.feature_names

tests/test_dice_interface/test_explainer_base.py

@@ -398,18 +398,17 @@ def test_zero_totalcfs(self, desired_range, regression_exp_object, sample_custom
                     desired_range=desired_range)
 
     @pytest.mark.parametrize("desired_range, method",
-                             [([10, 100], 'random')])
-    def test_numeric_categories(self, desired_range, method, create_boston_data):
+                             [([3, 5], 'random')])
+    def test_numeric_categories(self, desired_range, method, create_housing_data):
         x_train, x_test, y_train, y_test, feature_names = \
-            create_boston_data
+            create_housing_data
 
         rfc = RandomForestRegressor(n_estimators=10, max_depth=4,
                                     random_state=777)
         model = rfc.fit(x_train, y_train)
 
         dataset_train = x_train.copy()
         dataset_train['Outcome'] = y_train
-        feature_names.remove('CHAS')
 
         d = dice_ml.Data(dataframe=dataset_train, continuous_features=feature_names, outcome_name='Outcome')
         m = dice_ml.Model(model=model, backend='sklearn', model_type='regressor')

tests/test_model_interface/test_base_model.py

@@ -47,9 +47,9 @@ def create_sklearn_random_forest_regressor(self, X, y):
         model = rfc.fit(X, y)
         return model
 
-    def test_base_model_regression(self, create_boston_data):
+    def test_base_model_regression(self, create_housing_data):
         x_train, x_test, y_train, y_test, feature_names = \
-            create_boston_data
+            create_housing_data
         trained_model = self.create_sklearn_random_forest_regressor(x_train, y_train)
 
         diceml_model = dice_ml.Model(model=trained_model, model_type='regressor', backend='sklearn')