Merge pull request #200 from openml/FIX/#187

Fix/#187

Author: janvanrijn

openml/flows/flow.py

@@ -334,6 +334,7 @@ def publish(self):
         """
 
         xml_description = self._to_xml()
+
         file_elements = {'description': xml_description}
         return_value = _perform_api_call("flow/", file_elements=file_elements)
         self.flow_id = int(xmltodict.parse(return_value)['oml:upload_flow']['oml:id'])

openml/flows/sklearn_converter.py

@@ -1,6 +1,7 @@
 """Convert scikit-learn estimators into an OpenMLFlows and vice versa."""
 
 from collections import OrderedDict
+import copy
 from distutils.version import LooseVersion
 import importlib
 import inspect
@@ -107,6 +108,10 @@ def flow_to_sklearn(o, **kwargs):
                 step_name = value['step_name']
                 key = value['key']
                 component = flow_to_sklearn(kwargs['components'][key])
+                # The component is now added to where it should be used
+                # later. It should not be passed to the constructor of the
+                # main flow object.
+                del kwargs['components'][key]
                 if step_name is None:
                     rval = component
                 else:
@@ -276,14 +281,13 @@ def _extract_information_from_model(model):
                     # component reference as a placeholder to the list of
                     # parameters, which will be replaced by the real component
                     # when deserializing the parameter
-                    sub_component_identifier = k + '__' + identifier
-                    sub_components_explicit.add(sub_component_identifier)
-                    sub_components[sub_component_identifier] = sub_component
+                    sub_components_explicit.add(identifier)
+                    sub_components[identifier] = sub_component
                     component_reference = OrderedDict()
                     component_reference[
                         'oml-python:serialized_object'] = 'component_reference'
                     component_reference['value'] = OrderedDict(
-                        key=sub_component_identifier, step_name=identifier)
+                        key=identifier, step_name=identifier)
                     parameter_value.append(component_reference)
 
             if isinstance(rval, tuple):
@@ -331,25 +335,27 @@ def _deserialize_model(flow, **kwargs):
 
     parameters = flow.parameters
     components = flow.components
-    component_dict = OrderedDict()
     parameter_dict = OrderedDict()
 
-    for name in components:
-        if '__' in name:
-            parameter_name, step = name.split('__')
-            value = components[name]
-            rval = flow_to_sklearn(value)
-            if parameter_name not in component_dict:
-                component_dict[parameter_name] = OrderedDict()
-            component_dict[parameter_name][step] = rval
-        else:
-            value = components[name]
-            rval = flow_to_sklearn(value)
-            parameter_dict[name] = rval
+    # Do a shallow copy of the components dictionary so we can remove the
+    # components from this copy once we added them into the pipeline. This
+    # allows us to not consider them any more when looping over the
+    # components, but keeping the dictionary of components untouched in the
+    # original components dictionary.
+    components_ = copy.copy(components)
 
     for name in parameters:
         value = parameters.get(name)
-        rval = flow_to_sklearn(value, components=components)
+        rval = flow_to_sklearn(value, components=components_)
+        parameter_dict[name] = rval
+
+    for name in components:
+        if name in parameter_dict:
+            continue
+        if name not in components_:
+            continue
+        value = components[name]
+        rval = flow_to_sklearn(value)
         parameter_dict[name] = rval
 
     module_name = model_name.rsplit('.', 1)

openml/runs/functions.py

@@ -228,7 +228,6 @@ def _create_run_from_xml(xml):
     evaluation_flows = dict()
     for evaluation_dict in run['oml:output_data']['oml:evaluation']:
         key = evaluation_dict['oml:name']
-        flow_id = int(evaluation_dict['oml:flow_id'])
         if 'oml:value' in evaluation_dict:
             value = float(evaluation_dict['oml:value'])
         elif 'oml:array_data' in evaluation_dict:

openml/runs/run.py

@@ -210,12 +210,22 @@ def get_flow_dict(_flow):
             subflow = flow.components[param]
             openml_param_settings += _parse_parameters(python_param_settings[param], subflow)
 
-        # add parameter setting (also the subflow. Just because we can)
-        param_dict = OrderedDict()
-        param_dict['oml:name'] = param
-        param_dict['oml:value'] = str(python_param_settings[param])
-        param_dict['oml:component'] = flow_dict[flow.name]
-        openml_param_settings.append(param_dict)
+        # add parameter setting (in some cases also the subflow. Just because we can)
+        if param in flow.parameters.keys():
+            param_dict = OrderedDict()
+            param_dict['oml:name'] = param
+            param_dict['oml:value'] = str(python_param_settings[param])
+            param_dict['oml:component'] = flow_dict[flow.name]
+            openml_param_settings.append(param_dict)
+        else:
+            if flow.name.startswith("sklearn.pipeline.Pipeline"):
+                # tolerate
+                pass
+            elif flow.name.startswith("sklearn.pipeline.FeatureUnion"):
+                # tolerate
+                pass
+            else:
+                raise ValueError("parameter %s not in flow description of flow %s" %(param,flow.name))
 
     return openml_param_settings
 

tests/test_flows/test_flow.py

@@ -243,12 +243,12 @@ def test_sklearn_to_upload_to_flow(self):
 
         fixture_name = '%ssklearn.model_selection._search.RandomizedSearchCV(' \
                        'estimator=sklearn.pipeline.Pipeline(' \
-                       'steps__ohe=sklearn.preprocessing.data.OneHotEncoder,' \
-                       'steps__scaler=sklearn.preprocessing.data.StandardScaler,' \
-                       'steps__fu=sklearn.pipeline.FeatureUnion(' \
-                       'transformer_list__pca=sklearn.decomposition.truncated_svd.TruncatedSVD,' \
-                       'transformer_list__fs=sklearn.feature_selection.univariate_selection.SelectPercentile),' \
-                       'steps__boosting=sklearn.ensemble.weight_boosting.AdaBoostClassifier(' \
+                       'ohe=sklearn.preprocessing.data.OneHotEncoder,' \
+                       'scaler=sklearn.preprocessing.data.StandardScaler,' \
+                       'fu=sklearn.pipeline.FeatureUnion(' \
+                       'pca=sklearn.decomposition.truncated_svd.TruncatedSVD,' \
+                       'fs=sklearn.feature_selection.univariate_selection.SelectPercentile),' \
+                       'boosting=sklearn.ensemble.weight_boosting.AdaBoostClassifier(' \
                        'base_estimator=sklearn.tree.tree.DecisionTreeClassifier)))' \
                         % sentinel
 

tests/test_flows/test_sklearn.py

@@ -145,8 +145,8 @@ def test_serialize_pipeline(self):
             ('scaler', scaler), ('dummy', dummy)))
 
         fixture_name = 'sklearn.pipeline.Pipeline(' \
-                       'steps__scaler=sklearn.preprocessing.data.StandardScaler,' \
-                       'steps__dummy=sklearn.dummy.DummyClassifier)'
+                       'scaler=sklearn.preprocessing.data.StandardScaler,' \
+                       'dummy=sklearn.dummy.DummyClassifier)'
         fixture_description = 'Automatically created sub-component.'
 
         serialization =  sklearn_to_flow(model)
@@ -162,15 +162,15 @@ def test_serialize_pipeline(self):
         # different sorting. Making a json makes it easier
         self.assertEqual(json.loads(serialization.parameters['steps']),
                          [{'oml-python:serialized_object':
-                               'component_reference', 'value': {'key': 'steps__scaler', 'step_name': 'scaler'}},
+                               'component_reference', 'value': {'key': 'scaler', 'step_name': 'scaler'}},
                           {'oml-python:serialized_object':
-                               'component_reference', 'value': {'key': 'steps__dummy', 'step_name': 'dummy'}}])
+                               'component_reference', 'value': {'key': 'dummy', 'step_name': 'dummy'}}])
 
         # Checking the sub-component
         self.assertEqual(len(serialization.components), 2)
-        self.assertIsInstance(serialization.components['steps__scaler'],
+        self.assertIsInstance(serialization.components['scaler'],
                               OpenMLFlow)
-        self.assertIsInstance(serialization.components['steps__dummy'],
+        self.assertIsInstance(serialization.components['dummy'],
                               OpenMLFlow)
 
         #del serialization.model
@@ -204,8 +204,8 @@ def test_serialize_feature_union(self):
         serialization =  sklearn_to_flow(fu)
         self.assertEqual(serialization.name,
                          'sklearn.pipeline.FeatureUnion('
-                         'transformer_list__ohe=sklearn.preprocessing.data.OneHotEncoder,'
-                         'transformer_list__scaler=sklearn.preprocessing.data.StandardScaler)')
+                         'ohe=sklearn.preprocessing.data.OneHotEncoder,'
+                         'scaler=sklearn.preprocessing.data.StandardScaler)')
         new_model = flow_to_sklearn(serialization)
 
         self.assertEqual(type(new_model), type(fu))
@@ -240,7 +240,7 @@ def test_serialize_feature_union(self):
         serialization = sklearn_to_flow(fu)
         self.assertEqual(serialization.name,
                          'sklearn.pipeline.FeatureUnion('
-                         'transformer_list__ohe=sklearn.preprocessing.data.OneHotEncoder)')
+                         'ohe=sklearn.preprocessing.data.OneHotEncoder)')
         new_model = flow_to_sklearn(serialization)
         self.assertEqual(type(new_model), type(fu))
         self.assertIsNot(new_model, fu)
@@ -256,13 +256,13 @@ def test_serialize_feature_union_switched_names(self):
         self.assertEqual(
             fu1_serialization.name,
             "sklearn.pipeline.FeatureUnion("
-            "transformer_list__ohe=sklearn.preprocessing.data.OneHotEncoder,"
-            "transformer_list__scaler=sklearn.preprocessing.data.StandardScaler)")
+            "ohe=sklearn.preprocessing.data.OneHotEncoder,"
+            "scaler=sklearn.preprocessing.data.StandardScaler)")
         self.assertEqual(
             fu2_serialization.name,
             "sklearn.pipeline.FeatureUnion("
-            "transformer_list__scaler=sklearn.preprocessing.data.OneHotEncoder,"
-            "transformer_list__ohe=sklearn.preprocessing.data.StandardScaler)")
+            "scaler=sklearn.preprocessing.data.OneHotEncoder,"
+            "ohe=sklearn.preprocessing.data.StandardScaler)")
 
     def test_serialize_complex_flow(self):
         ohe = sklearn.preprocessing.OneHotEncoder(categorical_features=[0])
@@ -282,9 +282,9 @@ def test_serialize_complex_flow(self):
 
         fixture_name = 'sklearn.model_selection._search.RandomizedSearchCV(' \
                        'estimator=sklearn.pipeline.Pipeline(' \
-                       'steps__ohe=sklearn.preprocessing.data.OneHotEncoder,' \
-                       'steps__scaler=sklearn.preprocessing.data.StandardScaler,' \
-                       'steps__boosting=sklearn.ensemble.weight_boosting.AdaBoostClassifier(' \
+                       'ohe=sklearn.preprocessing.data.OneHotEncoder,' \
+                       'scaler=sklearn.preprocessing.data.StandardScaler,' \
+                       'boosting=sklearn.ensemble.weight_boosting.AdaBoostClassifier(' \
                        'base_estimator=sklearn.tree.tree.DecisionTreeClassifier)))'
         self.assertEqual(serialized.name, fixture_name)
 

tests/test_runs/test_run_functions.py

@@ -1,9 +1,12 @@
 import sys
 
+from sklearn.dummy import DummyClassifier
+from sklearn.preprocessing import StandardScaler
 from sklearn.linear_model import LogisticRegression, SGDClassifier, LinearRegression
 from sklearn.ensemble import RandomForestClassifier, BaggingClassifier
 from sklearn.svm import SVC
 from sklearn.model_selection import RandomizedSearchCV, GridSearchCV, StratifiedKFold
+from sklearn.pipeline import Pipeline
 import openml
 import openml.exceptions
 from openml.testing import TestBase
@@ -86,6 +89,19 @@ def test_run_optimize_bagging_iris(self):
         run = self._perform_run(task_id, num_instances, grid_search)
         self.assertEqual(len(run.trace_content), num_iterations * num_folds)
 
+    def test_run_pipeline(self):
+        task_id = 10107
+        num_instances = 150
+        num_folds = 10
+        num_iterations = 9  # (num values for C times gamma)
+
+        scaler = StandardScaler(with_mean=False)
+        dummy = DummyClassifier(strategy='prior')
+        model = Pipeline(steps=(('scaler', scaler), ('dummy', dummy)))
+
+        run = self._perform_run(task_id, num_instances, model)
+        self.assertEqual(run.trace_content, None)
+
     def test__run_task_get_arffcontent(self):
         task = openml.tasks.get_task(1939)
         class_labels = task.class_labels
@@ -231,5 +247,5 @@ def test_get_runs_list_by_filters(self):
         runs = openml.runs.list_runs(id=ids, task=tasks, uploader=uploaders_1)
 
     def test_get_runs_list_by_tag(self):
-        runs = openml.runs.list_runs(tag='02-11-16_21.46.39')
-        self.assertEqual(len(runs), 1)
+        runs = openml.runs.list_runs(tag='curves')
+        self.assertGreaterEqual(len(runs), 1)