MAINT remove step__ from component identifiers in pipeline

Author: mfeurer

openml/flows/sklearn_converter.py

@@ -1,6 +1,7 @@
 """Convert scikit-learn estimators into an OpenMLFlows and vice versa."""
 
 from collections import OrderedDict
+import copy
 from distutils.version import LooseVersion
 import importlib
 import inspect
@@ -107,6 +108,10 @@ def flow_to_sklearn(o, **kwargs):
                 step_name = value['step_name']
                 key = value['key']
                 component = flow_to_sklearn(kwargs['components'][key])
+                # The component is now added to where it should be used
+                # later. It should not be passed to the constructor of the
+                # main flow object.
+                del kwargs['components'][key]
                 if step_name is None:
                     rval = component
                 else:
@@ -276,14 +281,13 @@ def _extract_information_from_model(model):
                     # component reference as a placeholder to the list of
                     # parameters, which will be replaced by the real component
                     # when deserializing the parameter
-                    sub_component_identifier = k + '__' + identifier
-                    sub_components_explicit.add(sub_component_identifier)
-                    sub_components[sub_component_identifier] = sub_component
+                    sub_components_explicit.add(identifier)
+                    sub_components[identifier] = sub_component
                     component_reference = OrderedDict()
                     component_reference[
                         'oml-python:serialized_object'] = 'component_reference'
                     component_reference['value'] = OrderedDict(
-                        key=sub_component_identifier, step_name=identifier)
+                        key=identifier, step_name=identifier)
                     parameter_value.append(component_reference)
 
             if isinstance(rval, tuple):
@@ -331,25 +335,27 @@ def _deserialize_model(flow, **kwargs):
 
     parameters = flow.parameters
     components = flow.components
-    component_dict = OrderedDict()
     parameter_dict = OrderedDict()
 
-    for name in components:
-        if '__' in name:
-            parameter_name, step = name.split('__')
-            value = components[name]
-            rval = flow_to_sklearn(value)
-            if parameter_name not in component_dict:
-                component_dict[parameter_name] = OrderedDict()
-            component_dict[parameter_name][step] = rval
-        else:
-            value = components[name]
-            rval = flow_to_sklearn(value)
-            parameter_dict[name] = rval
+    # Do a shallow copy of the components dictionary so we can remove the
+    # components from this copy once we added them into the pipeline. This
+    # allows us to not consider them any more when looping over the
+    # components, but keeping the dictionary of components untouched in the
+    # original components dictionary.
+    components_ = copy.copy(components)
 
     for name in parameters:
         value = parameters.get(name)
-        rval = flow_to_sklearn(value, components=components)
+        rval = flow_to_sklearn(value, components=components_)
+        parameter_dict[name] = rval
+
+    for name in components:
+        if name in parameter_dict:
+            continue
+        if name not in components_:
+            continue
+        value = components[name]
+        rval = flow_to_sklearn(value)
         parameter_dict[name] = rval
 
     module_name = model_name.rsplit('.', 1)

tests/test_flows/test_flow.py

@@ -243,12 +243,12 @@ def test_sklearn_to_upload_to_flow(self):
 
         fixture_name = '%ssklearn.model_selection._search.RandomizedSearchCV(' \
                        'estimator=sklearn.pipeline.Pipeline(' \
-                       'steps__ohe=sklearn.preprocessing.data.OneHotEncoder,' \
-                       'steps__scaler=sklearn.preprocessing.data.StandardScaler,' \
-                       'steps__fu=sklearn.pipeline.FeatureUnion(' \
-                       'transformer_list__pca=sklearn.decomposition.truncated_svd.TruncatedSVD,' \
-                       'transformer_list__fs=sklearn.feature_selection.univariate_selection.SelectPercentile),' \
-                       'steps__boosting=sklearn.ensemble.weight_boosting.AdaBoostClassifier(' \
+                       'ohe=sklearn.preprocessing.data.OneHotEncoder,' \
+                       'scaler=sklearn.preprocessing.data.StandardScaler,' \
+                       'fu=sklearn.pipeline.FeatureUnion(' \
+                       'pca=sklearn.decomposition.truncated_svd.TruncatedSVD,' \
+                       'fs=sklearn.feature_selection.univariate_selection.SelectPercentile),' \
+                       'boosting=sklearn.ensemble.weight_boosting.AdaBoostClassifier(' \
                        'base_estimator=sklearn.tree.tree.DecisionTreeClassifier)))' \
                         % sentinel
 

tests/test_flows/test_sklearn.py

@@ -145,8 +145,8 @@ def test_serialize_pipeline(self):
             ('scaler', scaler), ('dummy', dummy)))
 
         fixture_name = 'sklearn.pipeline.Pipeline(' \
-                       'steps__scaler=sklearn.preprocessing.data.StandardScaler,' \
-                       'steps__dummy=sklearn.dummy.DummyClassifier)'
+                       'scaler=sklearn.preprocessing.data.StandardScaler,' \
+                       'dummy=sklearn.dummy.DummyClassifier)'
         fixture_description = 'Automatically created sub-component.'
 
         serialization =  sklearn_to_flow(model)
@@ -162,15 +162,15 @@ def test_serialize_pipeline(self):
         # different sorting. Making a json makes it easier
         self.assertEqual(json.loads(serialization.parameters['steps']),
                          [{'oml-python:serialized_object':
-                               'component_reference', 'value': {'key': 'steps__scaler', 'step_name': 'scaler'}},
+                               'component_reference', 'value': {'key': 'scaler', 'step_name': 'scaler'}},
                           {'oml-python:serialized_object':
-                               'component_reference', 'value': {'key': 'steps__dummy', 'step_name': 'dummy'}}])
+                               'component_reference', 'value': {'key': 'dummy', 'step_name': 'dummy'}}])
 
         # Checking the sub-component
         self.assertEqual(len(serialization.components), 2)
-        self.assertIsInstance(serialization.components['steps__scaler'],
+        self.assertIsInstance(serialization.components['scaler'],
                               OpenMLFlow)
-        self.assertIsInstance(serialization.components['steps__dummy'],
+        self.assertIsInstance(serialization.components['dummy'],
                               OpenMLFlow)
 
         #del serialization.model
@@ -204,8 +204,8 @@ def test_serialize_feature_union(self):
         serialization =  sklearn_to_flow(fu)
         self.assertEqual(serialization.name,
                          'sklearn.pipeline.FeatureUnion('
-                         'transformer_list__ohe=sklearn.preprocessing.data.OneHotEncoder,'
-                         'transformer_list__scaler=sklearn.preprocessing.data.StandardScaler)')
+                         'ohe=sklearn.preprocessing.data.OneHotEncoder,'
+                         'scaler=sklearn.preprocessing.data.StandardScaler)')
         new_model = flow_to_sklearn(serialization)
 
         self.assertEqual(type(new_model), type(fu))
@@ -240,7 +240,7 @@ def test_serialize_feature_union(self):
         serialization = sklearn_to_flow(fu)
         self.assertEqual(serialization.name,
                          'sklearn.pipeline.FeatureUnion('
-                         'transformer_list__ohe=sklearn.preprocessing.data.OneHotEncoder)')
+                         'ohe=sklearn.preprocessing.data.OneHotEncoder)')
         new_model = flow_to_sklearn(serialization)
         self.assertEqual(type(new_model), type(fu))
         self.assertIsNot(new_model, fu)
@@ -256,13 +256,13 @@ def test_serialize_feature_union_switched_names(self):
         self.assertEqual(
             fu1_serialization.name,
             "sklearn.pipeline.FeatureUnion("
-            "transformer_list__ohe=sklearn.preprocessing.data.OneHotEncoder,"
-            "transformer_list__scaler=sklearn.preprocessing.data.StandardScaler)")
+            "ohe=sklearn.preprocessing.data.OneHotEncoder,"
+            "scaler=sklearn.preprocessing.data.StandardScaler)")
         self.assertEqual(
             fu2_serialization.name,
             "sklearn.pipeline.FeatureUnion("
-            "transformer_list__scaler=sklearn.preprocessing.data.OneHotEncoder,"
-            "transformer_list__ohe=sklearn.preprocessing.data.StandardScaler)")
+            "scaler=sklearn.preprocessing.data.OneHotEncoder,"
+            "ohe=sklearn.preprocessing.data.StandardScaler)")
 
     def test_serialize_complex_flow(self):
         ohe = sklearn.preprocessing.OneHotEncoder(categorical_features=[0])
@@ -282,9 +282,9 @@ def test_serialize_complex_flow(self):
 
         fixture_name = 'sklearn.model_selection._search.RandomizedSearchCV(' \
                        'estimator=sklearn.pipeline.Pipeline(' \
-                       'steps__ohe=sklearn.preprocessing.data.OneHotEncoder,' \
-                       'steps__scaler=sklearn.preprocessing.data.StandardScaler,' \
-                       'steps__boosting=sklearn.ensemble.weight_boosting.AdaBoostClassifier(' \
+                       'ohe=sklearn.preprocessing.data.OneHotEncoder,' \
+                       'scaler=sklearn.preprocessing.data.StandardScaler,' \
+                       'boosting=sklearn.ensemble.weight_boosting.AdaBoostClassifier(' \
                        'base_estimator=sklearn.tree.tree.DecisionTreeClassifier)))'
         self.assertEqual(serialized.name, fixture_name)