Add #737 (#772)

* add hyperparameter column to list_evaluations_setups

Author: sahithyaravi

examples/fetch_evaluations_tutorial.py

@@ -16,6 +16,7 @@
 * Sort the obtained results in descending order of the metric
 * Plot a cumulative distribution function for the evaluations
 * Compare the top 10 performing flows based on the evaluation performance
+* Retrieve evaluations with hyperparameter settings
 """
 
 ############################################################################
@@ -147,3 +148,30 @@ def plot_flow_compare(evaluations, top_n=10, metric='predictive_accuracy'):
 flow_names = evals.flow_name.unique()[:top_n]
 for i in range(top_n):
     print((flow_ids[i], flow_names[i]))
+
+#############################################################################
+# Obtaining evaluations with hyperparameter settings
+# ==================================================
+# We'll now obtain the evaluations of a task and a flow with the hyperparameters
+
+# List evaluations in descending order based on predictive_accuracy with
+# hyperparameters
+evals_setups = openml.evaluations.list_evaluations_setups(function='predictive_accuracy', task=[31],
+                                                          size=100, sort_order='desc')
+
+""
+print(evals_setups.head())
+
+""
+# Return evaluations for flow_id in descending order based on predictive_accuracy
+# with hyperparameters. parameters_in_separate_columns returns parameters in
+# separate columns
+evals_setups = openml.evaluations.list_evaluations_setups(function='predictive_accuracy',
+                                                          flow=[6767],
+                                                          size=100,
+                                                          parameters_in_separate_columns=True)
+
+""
+print(evals_setups.head(10))
+
+""

openml/evaluations/functions.py

@@ -254,7 +254,8 @@ def list_evaluations_setups(
         tag: Optional[str] = None,
         per_fold: Optional[bool] = None,
         sort_order: Optional[str] = None,
-        output_format: str = 'dataframe'
+        output_format: str = 'dataframe',
+        parameters_in_separate_columns: bool = False
 ) -> Union[Dict, pd.DataFrame]:
     """
     List all run-evaluation pairs matching all of the given filters
@@ -287,12 +288,19 @@ def list_evaluations_setups(
         The parameter decides the format of the output.
         - If 'dict' the output is a dict of dict
         - If 'dataframe' the output is a pandas DataFrame
+    parameters_in_separate_columns: bool, optional (default= False)
+        Returns hyperparameters in separate columns if set to True.
+        Valid only for a single flow
 
 
     Returns
     -------
     dict or dataframe with hyperparameter settings as a list of tuples.
     """
+    if parameters_in_separate_columns and (flow is None or len(flow) != 1):
+        raise ValueError("Can set parameters_in_separate_columns to true "
+                         "only for single flow_id")
+
     # List evaluations
     evals = list_evaluations(function=function, offset=offset, size=size, id=id, task=task,
                              setup=setup, flow=flow, uploader=uploader, tag=tag,
@@ -315,14 +323,18 @@ def list_evaluations_setups(
         # Convert parameters of setup into list of tuples of (hyperparameter, value)
         for parameter_dict in setups['parameters']:
             if parameter_dict is not None:
-                parameters.append([tuple([param['parameter_name'], param['value']])
-                                   for param in parameter_dict.values()])
+                parameters.append({param['full_name']: param['value']
+                                   for param in parameter_dict.values()})
             else:
-                parameters.append([])
+                parameters.append({})
         setups['parameters'] = parameters
         # Merge setups with evaluations
         df = pd.merge(evals, setups, on='setup_id', how='left')
 
+    if parameters_in_separate_columns:
+        df = pd.concat([df.drop('parameters', axis=1),
+                        df['parameters'].apply(pd.Series)], axis=1)
+
     if output_format == 'dataframe':
         return df
     else:

tests/test_evaluations/test_evaluation_functions.py

@@ -27,8 +27,11 @@ def _check_list_evaluation_setups(self, size, **kwargs):
         # Check if the hyper-parameter column is as accurate and flow_id
         for index, row in evals_setups.iterrows():
             params = openml.runs.get_run(row['run_id']).parameter_settings
-            hyper_params = [tuple([param['oml:name'], param['oml:value']]) for param in params]
-            self.assertTrue(sorted(row['parameters']) == sorted(hyper_params))
+            list1 = [param['oml:value'] for param in params]
+            list2 = list(row['parameters'].values())
+            # check if all values are equal
+            self.assertSequenceEqual(sorted(list1), sorted(list2))
+        return evals_setups
 
     def test_evaluation_list_filter_task(self):
         openml.config.server = self.production_server
@@ -171,7 +174,17 @@ def test_list_evaluations_setups_filter_flow(self):
         openml.config.server = self.production_server
         flow_id = [405]
         size = 100
-        self._check_list_evaluation_setups(size, flow=flow_id)
+        evals = self._check_list_evaluation_setups(size, flow=flow_id)
+        # check if parameters in separate columns works
+        evals_cols = openml.evaluations.list_evaluations_setups("predictive_accuracy",
+                                                                flow=flow_id, size=size,
+                                                                sort_order='desc',
+                                                                output_format='dataframe',
+                                                                parameters_in_separate_columns=True
+                                                                )
+        columns = (list(evals_cols.columns))
+        keys = (list(evals['parameters'].values[0].keys()))
+        self.assertTrue(all(elem in columns for elem in keys))
 
     def test_list_evaluations_setups_filter_task(self):
         openml.config.server = self.production_server