made dataset.get_features_of_type aware of features that will be removed

Author: janvanrijn

openml/datasets/dataset.py

@@ -64,7 +64,7 @@ def __init__(self, dataset_id=None, name=None, version=None, description=None,
         self.url = url
         self.default_target_attribute = default_target_attribute
         self.row_id_attribute = row_id_attribute
-        self.ignore_attributes = ignore_attribute
+        self.ignore_attributes = list(ignore_attribute) if ignore_attribute is not None else None # TODO: check
         self.version_label = version_label
         self.citation = citation
         self.tag = tag
@@ -218,10 +218,7 @@ def get_data(self, target=None, target_dtype=int, include_row_id=False,
             if not self.ignore_attributes:
                 pass
             else:
-                if is_string(self.ignore_attributes):
-                    to_exclude.append(self.ignore_attributes)
-                else:
-                    to_exclude.extend(self.ignore_attributes)
+                to_exclude.extend(self.ignore_attributes)
 
         if len(to_exclude) > 0:
             logger.info("Going to remove the following attributes:"
@@ -311,17 +308,29 @@ def retrieve_class_labels(self, target_name='class'):
         else:
             return None
 
-    def get_features_by_type(self, data_type, exclude=None):
+
+    def get_features_by_type(self, data_type, exclude=None, exclude_ignore_attributes=True):
         assert data_type in OpenMLDataFeature.LEGAL_DATA_TYPES, "Illegal feature type requested"
         if exclude is not None:
-            assert type(exclude) is list, "Exclude should be a list of indeces"
+            assert type(exclude) is list, "Exclude should be a list"
+            assert all(isinstance(elem, str) for elem in exclude), "Exclude should be a list of strings"
+        to_exclude = []
+        if exclude is not None:
+            to_exclude.extend(exclude)
+        if exclude_ignore_attributes and self.ignore_attributes is not None:
+            to_exclude.extend(self.ignore_attributes)
+        print(to_exclude)
 
         result = []
+        offset = 0
+        # this function assumes that everything in to_exclude will be 'excluded' from the dataset (hence the offset)
         for idx in self.features:
-            # in many cases we want to exclude, for example, the target feature
-            if exclude is None or idx not in exclude:
+            name = self.features[idx].name
+            if name in to_exclude:
+                offset += 1
+            else:
                 if self.features[idx].data_type == data_type:
-                    result.append(idx)
+                    result.append(idx-offset)
         return result
 
     def publish(self):

tests/test_utils/test_conditionalimputer.py

@@ -34,13 +34,14 @@ def _do_test(self, dataset, X, nominal_indices, clf):
         return X_prime
 
     def test_impute_indices(self):
-        task_ids = [2, 24, 42, 59]
+        task_ids = [2, 24, 41, 42, 59]
 
         for task_id in task_ids:
             task = openml.tasks.get_task(task_id)
             dataset = task.get_dataset()
             X, _ = dataset.get_data(target=task.target_name)
-            nominal_indices = dataset.get_features_by_type('nominal', exclude=[len(dataset.features)-1])
+            nominal_indices = dataset.get_features_by_type('nominal', exclude=[task.target_name])
+            # print("task id %d indices %s" %(task_id, str(nominal_indices)))
             clf = ConditionalImputer(strategy="median",
                                      strategy_nominal="most_frequent",
                                      categorical_features=nominal_indices,
@@ -50,13 +51,14 @@ def test_impute_indices(self):
 
 
     def test_impute_smart(self):
-        task_ids = [2, 24, 42, 59]
+        task_ids = [2, 24, 41, 42, 59]
 
         for task_id in task_ids:
             task = openml.tasks.get_task(task_id)
             dataset = task.get_dataset()
             X, _ = dataset.get_data(target=task.target_name)
-            nominal_indices = dataset.get_features_by_type('nominal', exclude=[len(dataset.features)-1])
+            nominal_indices = dataset.get_features_by_type('nominal', exclude=[task.target_name])
+            # print("task id %d indices %s" %(task_id, str(nominal_indices)))
             clf = ConditionalImputer(strategy="median",
                                      strategy_nominal="most_frequent",
                                      categorical_features=None,
@@ -71,7 +73,7 @@ def test_impute_with_constant(self):
             task = openml.tasks.get_task(task_id)
             dataset = task.get_dataset()
             X, _ = dataset.get_data(target=task.target_name)
-            nominal_indices = dataset.get_features_by_type('nominal', exclude=[len(dataset.features) - 1])
+            nominal_indices = dataset.get_features_by_type('nominal', exclude=[task.target_name])
             clf = ConditionalImputer(strategy="median",
                                      strategy_nominal="most_frequent",
                                      categorical_features=None,