review comments

Author: sahithyaravi

openml/evaluations/functions.py

@@ -268,27 +268,23 @@ def list_evaluations_setups(
         the number of runs to skip, starting from the first
     size : int, optional
         the maximum number of runs to show
-
-    id : list, optional
-
-    task : list, optional
-
-    setup: list, optional
-
-    flow : list, optional
-
-    uploader : list, optional
-
+    id : list[int], optional
+        the list of evaluation ID's
+    task : list[int], optional
+        the list of task ID's
+    setup: list[int], optional
+        the list of setup ID's
+    flow : list[int], optional
+        the list of flow ID's
+    uploader : list[int], optional
+        the list of uploader ID's
     tag : str, optional
-
+        filter evaluation based on given tag
     per_fold : bool, optional
-
     sort_order : str, optional
        order of sorting evaluations, ascending ("asc") or descending ("desc")
-
-    output_format: str, optional (default='object')
+    output_format: str, optional (default='dataframe')
         The parameter decides the format of the output.
-
         - If 'dict' the output is a dict of dict
         - If 'dataframe' the output is a pandas DataFrame
 

tests/test_evaluations/test_evaluation_functions.py

@@ -6,6 +6,31 @@
 class TestEvaluationFunctions(TestBase):
     _multiprocess_can_split_ = True
 
+    def _check_list_evaluation_setups(self, size, **kwargs):
+        evals_setups = openml.evaluations.list_evaluations_setups("predictive_accuracy",
+                                                                  **kwargs, size=size,
+                                                                  sort_order='desc',
+                                                                  output_format='dataframe')
+        evals = openml.evaluations.list_evaluations("predictive_accuracy",
+                                                    **kwargs,  size=size,
+                                                    sort_order='desc',
+                                                    output_format='dataframe')
+
+        # Check if list is non-empty
+        self.assertGreater(len(evals_setups), 0)
+        # Check if output from sort is sorted in the right order
+        self.assertSequenceEqual(sorted(evals_setups['value'].tolist(), reverse=True)
+                                 , evals_setups['value'].tolist())
+
+        # Check if output and order of list_evaluations is preserved
+        self.assertSequenceEqual(evals_setups['run_id'].tolist(), evals['run_id'].tolist())
+        # Check if the hyper-parameter column is as accurate and flow_id
+        for index, row in evals_setups.iterrows():
+            params = openml.runs.get_run(row['run_id']).parameter_settings
+            hyper_params = [tuple([param['oml:name'], param['oml:value']]) for param in params]
+            self.assertTrue(sorted(row['parameters']) == sorted(hyper_params))
+
+
     def test_evaluation_list_filter_task(self):
         openml.config.server = self.production_server
 
@@ -145,54 +170,12 @@ def test_list_evaluation_measures(self):
 
     def test_list_evaluations_setups_filter_flow(self):
         openml.config.server = self.production_server
-        flow_id = 405
+        flow_id = [405]
         size = 100
-        evals_setups = openml.evaluations.list_evaluations_setups("predictive_accuracy",
-                                                                  flow=[flow_id], size=size,
-                                                                  sort_order='desc',
-                                                                  output_format='dataframe')
-        evals = openml.evaluations.list_evaluations("predictive_accuracy",
-                                                    flow=[flow_id], size=size,
-                                                    sort_order='desc',
-                                                    output_format='dataframe')
-
-        # Check if list is non-empty
-        self.assertGreater(len(evals_setups), 0)
-        # Check if output from sort is sorted in the right order
-        self.assertTrue(sorted(list(evals_setups['value'].values), reverse=True)
-                        == list(evals_setups['value'].values))
-        # Check if output and order of list_evaluations is preserved
-        self.assertTrue((evals_setups['run_id'].values == evals['run_id'].values).all())
-        # Check if the hyper-parameter column is as accurate and flow_id
-        for index, row in evals_setups.iterrows():
-            params = openml.runs.get_run(row['run_id']).parameter_settings
-            hyper_params = [tuple([param['oml:name'], param['oml:value']]) for param in params]
-            self.assertTrue(sorted(row['parameters']) == sorted(hyper_params))
-            self.assertEqual(row['flow_id'], flow_id)
+        self._check_list_evaluation_setups(size, flow=flow_id)
 
     def test_list_evaluations_setups_filter_task(self):
         openml.config.server = self.production_server
-        task_id = 6
+        task_id = [6]
         size = 100
-        evals_setups = openml.evaluations.list_evaluations_setups("predictive_accuracy",
-                                                                  task=[task_id], size=size,
-                                                                  sort_order='desc',
-                                                                  output_format='dataframe')
-        evals = openml.evaluations.list_evaluations("predictive_accuracy",
-                                                    task=[task_id], size=size,
-                                                    sort_order='desc',
-                                                    output_format='dataframe')
-
-        # Check if list is non-empty
-        self.assertGreater(len(evals_setups), 0)
-        # Check if output from sort is sorted in the right order
-        self.assertTrue(sorted(list(evals_setups['value'].values), reverse=True)
-                        == list(evals_setups['value'].values))
-        # Check if output and order of list_evaluations is preserved
-        self.assertTrue((evals_setups['run_id'].values == evals['run_id'].values).all())
-        # Check if the hyper-parameter column is as accurate and task_id
-        for index, row in evals_setups.iterrows():
-            params = openml.runs.get_run(row['run_id']).parameter_settings
-            hyper_params = [tuple([param['oml:name'], param['oml:value']]) for param in params]
-            self.assertTrue(sorted(row['parameters']) == sorted(hyper_params))
-            self.assertEqual(row['task_id'], task_id)
+        self._check_list_evaluation_setups(size, task=task_id)

tests/test_flows/test_flow_functions.py

@@ -286,11 +286,6 @@ def test_get_flow_reinstantiate_model_no_extension(self):
     @unittest.skipIf(LooseVersion(sklearn.__version__) == "0.19.1",
                      reason="Target flow is from sklearn 0.19.1")
     def test_get_flow_reinstantiate_model_wrong_version(self):
-        openml.config.server = self.production_server
-        # 20 is scikit-learn ==0.20.0
-        # I can't find a != 0.20 permanent flow on the test server.
-        self.assertRaises(ValueError, openml.flows.get_flow, flow_id=7238, reinstantiate=True)
-
         # Note that CI does not test against 0.19.1.
         openml.config.server = self.production_server
         _, sklearn_major, _ = LooseVersion(sklearn.__version__).version[:3]