Merge branch 'origin/MAIN/improve_coverage' into develop

Author: mfeurer

openml/runs/run.py

@@ -23,8 +23,8 @@ class OpenMLRun(object):
     FIXME
 
     """
-    def __init__(self, task_id, flow_id, setup_string, dataset_id, files=None,
-                 setup_id=None, tags=None, uploader=None, uploader_name=None,
+    def __init__(self, task_id, flow_id, dataset_id, setup_string=None, 
+                 files=None, setup_id=None, tags=None, uploader=None, uploader_name=None,
                  evaluations=None, detailed_evaluations=None,
                  data_content=None, model=None, task_type=None,
                  task_evaluation_measure=None, flow_name=None,
@@ -49,15 +49,20 @@ def __init__(self, task_id, flow_id, setup_string, dataset_id, files=None,
         self.flow = flow
         self.run_id = run_id
 
-    def _generate_arff(self):
-        """Generates an arff for upload to server.
+    def _generate_arff_dict(self):
+        """Generates the arff dictionary for upload to the server.
+
+        Assumes that the run has been executed.
 
         Returns
         -------
-        arf_dict : dictionary
-            Dictionary representation of an ARFF data format containing
-            predictions and confidences.
+        arf_dict : dict
+            Dictionary representation of the ARFF file that will be uploaded.
+            Contains predictions and information about the run environment.
         """
+        if self.data_content is None:
+            raise ValueError('Run has not been executed.')
+
         run_environment = (_get_version_information() +
                            [time.strftime("%c")] + ['Created by run_task()'])
         task = get_task(self.task_id)
@@ -85,7 +90,7 @@ def publish(self):
         -------
         self : OpenMLRun
         """
-        predictions = arff.dumps(self._generate_arff())
+        predictions = arff.dumps(self._generate_arff_dict())
         description_xml = self._create_description_xml()
         file_elements = {'predictions': ("predictions.csv", predictions),
                          'description': ("description.xml", description_xml)}
@@ -159,11 +164,19 @@ def run_task(task, model):
                          'only works for tasks with class labels.')
     setup_string = _create_setup_string(model)
 
-    run = OpenMLRun(task_id=task.task_id, flow_id=flow_id,
-                    setup_string=setup_string, dataset_id=dataset.dataset_id,
-                    task=task, flow=flow)
+    run = OpenMLRun(task.task_id, flow_id, setup_string, dataset.id)
+    run.data_content = _run_task_get_arffcontent(model, task, class_labels)
+
+    # The model will not be uploaded at the moment, but used to get the
+    # hyperparameter values when uploading the run
+    X, Y = task.get_X_and_y()
+    run.model = model.fit(X, Y)
+    return run
+
 
-    train_times = []
+def _run_task_get_arffcontent(model, task, class_labels):
+    X, Y = task.get_X_and_y()
+    arff_datacontent = []
 
     rep_no = 0
     # TODO use different iterator to only provide a single iterator (less
@@ -177,26 +190,21 @@ def run_task(task, model):
             testX = X[test_indices]
             testY = Y[test_indices]
 
-            start_time = time.time()
             model.fit(trainX, trainY)
             ProbaY = model.predict_proba(testX)
             PredY = model.predict(testX)
-            end_time = time.time()
-
-            train_times.append(end_time - start_time)
 
             for i in range(0, len(test_indices)):
-                arff_line = [rep_no, fold_no, test_indices[i],
-                             class_labels[PredY[i]], class_labels[testY[i]]]
-                arff_line[3:3] = ProbaY[i]
+                arff_line = [rep_no, fold_no, test_indices[i]]
+                arff_line.extend(ProbaY[i])
+                arff_line.append(class_labels[PredY[i]])
+                arff_line.append(class_labels[testY[i]])
                 arff_datacontent.append(arff_line)
 
             fold_no = fold_no + 1
         rep_no = rep_no + 1
 
-    run.data_content = arff_datacontent
-    run.model = model.fit(X, Y)
-    return run
+    return arff_datacontent
 
 
 def _to_dict(taskid, flow_id, setup_string, parameter_settings, tags):

tests/runs/test_runs.py

@@ -1,4 +1,4 @@
-from sklearn.linear_model import LogisticRegression
+from sklearn.linear_model import LogisticRegression, SGDClassifier
 import openml
 from openml.testing import TestBase
 
@@ -12,6 +12,36 @@ def test_run_iris(self):
         self.assertEqual(run_, run)
         self.assertIsInstance(run.dataset_id, int)
 
+    def test__run_task_get_arffcontent(self):
+        task = openml.tasks.get_task(1939)
+        class_labels = task.class_labels
+
+        clf = SGDClassifier(loss='hinge', random_state=1)
+        self.assertRaisesRegexp(AttributeError,
+                                "probability estimates are not available for loss='hinge'",
+                                openml.runs.run._run_task_get_arffcontent,
+                                clf, task, class_labels)
+
+        clf = SGDClassifier(loss='log', random_state=1)
+        arff_datacontent = openml.runs.run._run_task_get_arffcontent(
+            clf, task, class_labels)
+        self.assertIsInstance(arff_datacontent, list)
+        # 10 times 10 fold CV of 150 samples
+        self.assertEqual(len(arff_datacontent), 1500)
+        for arff_line in arff_datacontent:
+            self.assertEqual(len(arff_line), 8)
+            self.assertGreaterEqual(arff_line[0], 0)
+            self.assertLessEqual(arff_line[0], 9)
+            self.assertGreaterEqual(arff_line[1], 0)
+            self.assertLessEqual(arff_line[1], 9)
+            self.assertGreaterEqual(arff_line[2], 0)
+            self.assertLessEqual(arff_line[2], 149)
+            self.assertAlmostEqual(sum(arff_line[3:6]), 1.0)
+            self.assertIn(arff_line[6], ['Iris-setosa', 'Iris-versicolor',
+                                         'Iris-virginica'])
+            self.assertIn(arff_line[7], ['Iris-setosa', 'Iris-versicolor',
+                                         'Iris-virginica'])
+
     def test_get_run(self):
         run = openml.runs.get_run(473350)
         self.assertEqual(run.dataset_id, 1167)