made imputer add constant instead for removed columns

added testcase

Author: janvanrijn

openml/datasets/dataset.py

@@ -312,13 +312,14 @@ def retrieve_class_labels(self, target_name='class'):
             return None
 
     def get_features_by_type(self, data_type, exclude=None):
-        assert type(exclude) is list, "Exclude should be a list of indeces"
         assert data_type in OpenMLDataFeature.LEGAL_DATA_TYPES, "Illegal feature type requested"
+        if exclude is not None:
+            assert type(exclude) is list, "Exclude should be a list of indeces"
 
         result = []
         for idx in self.features:
             # in many cases we want to exclude, for example, the target feature
-            if idx not in exclude:
+            if exclude is None or idx not in exclude:
                 if self.features[idx].data_type == data_type:
                     result.append(idx)
         return result

openml/utils/preprocessing.py

@@ -1,17 +1,19 @@
-
-from sklearn.preprocessing.imputation import Imputer, check_array, _get_mask, _most_frequent
+from scipy.integrate.tests.test_bvp import emden_bc
+from sklearn.preprocessing.imputation import Imputer, _get_mask
 
 import warnings
-
+import math
 import numpy as np
-import numpy.ma as ma
 from scipy import sparse
+
+from sklearn.utils import check_array
 from sklearn.utils.fixes import astype
 from sklearn.utils.sparsefuncs import _get_median
 from sklearn.utils.validation import check_is_fitted
 from sklearn.utils.validation import FLOAT_DTYPES
 
 
+
 class ConditionalImputer(Imputer):
     """Imputation transformer for completing missing values.
 
@@ -34,6 +36,13 @@ class ConditionalImputer(Imputer):
         - If "most_frequent", then replace missing using the most frequent
           value along the axis.
 
+    strategy_nominal : string, optional (default="most_frequent")
+        The imputation strategy for nominal attributes. For values, see "strategy"
+
+    indices_nominal : list (int)
+        An array of indices determining which are treated as nominal. If None,
+        the Conditional Imputer will guess based on the values
+
     axis : integer, optional (default=0)
         The axis along which to impute.
 
@@ -68,12 +77,15 @@ class ConditionalImputer(Imputer):
     """
     def __init__(self, missing_values="NaN", strategy="mean",
                  strategy_nominal="most_frequent",
-                 indeces_nominal=None,
+                 categorical_features=None,
+                 empty_attribute_constant=None,
                  axis=0, verbose=0, copy=True):
         self.missing_values = missing_values
         self.strategy = strategy
         self.strategy_nominal = strategy_nominal
-        self.indeces_nominal = indeces_nominal
+        self.categorical_features = categorical_features
+        self.categorical_features_implied = None
+        self.empty_attribute_constant = empty_attribute_constant
         self.axis = axis
         self.verbose = verbose
         self.copy = copy
@@ -125,8 +137,78 @@ def fit(self, X, y=None):
 
         # here the indexes of nominal values get set
         self.statistics_ = statistics_general
-        if self.indeces_nominal is not None:
-            for i in self.indeces_nominal:
+        if self.categorical_features is not None:
+            for i in self.categorical_features:
                 self.statistics_[i] = statistics_nominal[i]
+        else:
+            # iterate over all attributes
+            self.categorical_features_implied = []
+            for iAtt in range(len(statistics_general)):
+                isNominal = True
+                for iInst in range(len(X)):
+                    if not np.isnan(X[iInst][iAtt]) and math.floor(X[iInst][iAtt]) != X[iInst][iAtt]:
+                        isNominal = False
+                        break
+                if isNominal:
+                    # book keeping, for testing purposes
+                    self.categorical_features_implied.append(iAtt)
+                    self.statistics_[iAtt] = statistics_nominal[iAtt]
+
+        return self
+
+
+    def transform(self, X):
+        """Impute all missing values in X.
+        Parameters
+        ----------
+        X : {array-like, sparse matrix}, shape = [n_samples, n_features]
+            The input data to complete.
+        """
+        check_is_fitted(self, 'statistics_')
+        X = check_array(X, accept_sparse='csc', dtype=FLOAT_DTYPES,
+                        force_all_finite=False, copy=self.copy)
+        statistics = self.statistics_
+        if X.shape[1] != statistics.shape[0]:
+            raise ValueError("X has %d features per sample, expected %d"
+                             % (X.shape[1], self.statistics_.shape[0]))
+
+        # impute completelly empty columns with constant
+        if self.empty_attribute_constant is not None:
+            invalid_mask = np.isnan(statistics)
+            X[:, invalid_mask] = self.empty_attribute_constant
+            self.statistics_[invalid_mask] = self.empty_attribute_constant
+
+        # Delete the invalid rows/columns
+        invalid_mask = np.isnan(statistics)
+        valid_mask = np.logical_not(invalid_mask)
+        valid_statistics = statistics[valid_mask]
+        valid_statistics_indexes = np.where(valid_mask)[0]
+        missing = np.arange(X.shape[not self.axis])[invalid_mask]
+
+        if invalid_mask.any():
+            if self.verbose:
+                warnings.warn("Deleting features without "
+                              "observed values: %s" % missing)
+            X = X[:, valid_statistics_indexes]
+
+        # Do actual imputation
+        if sparse.issparse(X) and self.missing_values != 0:
+            mask = _get_mask(X.data, self.missing_values)
+            indexes = np.repeat(np.arange(len(X.indptr) - 1, dtype=np.int),
+                                np.diff(X.indptr))[mask]
+
+            X.data[mask] = astype(valid_statistics[indexes], X.dtype,
+                                  copy=False)
+        else:
+            if sparse.issparse(X):
+                X = X.toarray()
+
+            mask = _get_mask(X, self.missing_values)
+            n_missing = np.sum(mask, axis=self.axis)
+            values = np.repeat(valid_statistics, n_missing)
+
+            coordinates = np.where(mask.transpose())[::-1]
+
+            X[coordinates] = values
 
-        return self
+        return X

tests/test_utils/test_conditionalimputer.py

@@ -1,34 +1,74 @@
 import openml, math, collections
+import numpy as np
 from openml.testing import TestBase
 from openml.utils.preprocessing import ConditionalImputer
 
 class OpenMLTaskTest(TestBase):
 
-    def test_impute_anneal(self):
-        task_id = 2
-
-        task = openml.tasks.get_task(task_id)
-        dataset = task.get_dataset()
-        X, _ = dataset.get_data(target=task.target_name)
-        nominal_indeces = dataset.get_features_by_type('nominal', exclude=[38])
-        clf = ConditionalImputer(strategy="median", strategy_nominal="most_frequent", indeces_nominal=nominal_indeces)
+    def _do_test(self, dataset, X, nominal_indices, clf):
         clf.fit(X)
         X_prime = clf.transform(X)
 
+        # in case of smart guessing nominal attributes, we accept false positives, but no false negatives
+        for column_idx in nominal_indices:
+            if clf.categorical_features_implied is not None:
+                assert column_idx in clf.categorical_features_implied, "False negative with smart nominal detector"
+
         correction = 0
         for idx, value in enumerate(clf.statistics_):
             if math.isnan(value):
                 # imputer can only give nan if all values are unknown
                 correction += 1
-                assert dataset.features[idx].number_missing_values == len(
-                    X), "Imputer calculated nan for usable feature"
+                assert dataset.features[idx].number_missing_values == len(X), "Imputer calculated nan for usable feature"
             else:
                 # check if nominal values get imputed correct
-                if idx in nominal_indeces:
+                if idx in nominal_indices:
                     assert value == math.floor(value) == math.ceil(value), "Wrong impute value for nominal feature"
 
                 corrected_index = idx - correction  # for x prime
                 # check if imputation succeeded
                 counter = collections.Counter(X_prime[:, corrected_index])
                 occurances_after = counter[value]
-                assert occurances_after >= dataset.features[idx].number_missing_values
+                assert occurances_after >= dataset.features[idx].number_missing_values
+
+        return X_prime
+
+    def test_impute_indices(self):
+        task_ids = [2,59]
+
+        for task_id in task_ids:
+            task = openml.tasks.get_task(task_id)
+            dataset = task.get_dataset()
+            X, _ = dataset.get_data(target=task.target_name)
+            nominal_indices = dataset.get_features_by_type('nominal', exclude=[len(dataset.features)-1])
+            clf = ConditionalImputer(strategy="median", strategy_nominal="most_frequent", categorical_features=nominal_indices, verbose=True)
+
+            self._do_test(dataset, X, nominal_indices, clf)
+
+
+    def test_impute_smart(self):
+        task_ids = [2,59]
+
+        for task_id in task_ids:
+            task = openml.tasks.get_task(task_id)
+            dataset = task.get_dataset()
+            X, _ = dataset.get_data(target=task.target_name)
+            nominal_indices = dataset.get_features_by_type('nominal', exclude=[len(dataset.features)-1])
+            clf = ConditionalImputer(strategy="median", strategy_nominal="most_frequent", categorical_features=None, verbose=True)
+
+            self._do_test(dataset, X, nominal_indices, clf)
+
+    def test_impute_with_constant(self):
+        task_ids = [2]
+
+        for task_id in task_ids:
+            task = openml.tasks.get_task(task_id)
+            dataset = task.get_dataset()
+            X, _ = dataset.get_data(target=task.target_name)
+            nominal_indices = dataset.get_features_by_type('nominal', exclude=[len(dataset.features) - 1])
+            clf = ConditionalImputer(strategy="median", strategy_nominal="most_frequent", categorical_features=None,
+                                     verbose=True, empty_attribute_constant=-1)
+
+            X_prime = self._do_test(dataset, X, nominal_indices, clf)
+            assert np.isnan(np.min(X_prime)) == False, 'Result contains nans'
+            assert X_prime.shape == X.shape