MAINT improve test coverage for run.py

Author: mfeurer

openml/runs/run.py

@@ -23,9 +23,9 @@ class OpenMLRun(object):
     FIXME
 
     """
-    def __init__(self, task_id, flow_id, setup_string, dataset_id, files=None,
-                 setup_id=None, tags=None, run_id=None, uploader=None,
-                 uploader_name=None, evaluations=None,
+    def __init__(self, task_id, flow_id, dataset_id, setup_string=None,
+                 files=None, setup_id=None, tags=None, run_id=None,
+                 uploader=None, uploader_name=None, evaluations=None,
                  detailed_evaluations=None, data_content=None,
                  model=None, task_type=None, task_evaluation_measure=None,
                  flow_name=None, parameter_settings=None, predictions_url=None):
@@ -47,8 +47,8 @@ def __init__(self, task_id, flow_id, setup_string, dataset_id, files=None,
         self.data_content = data_content
         self.model = model
 
-    def _generate_arff(self):
-        """Generates an arff for upload to server.
+    def _generate_arff_header_dict(self):
+        """Generates the arff header dictionary for upload to the server.
 
         Returns
         -------
@@ -78,7 +78,7 @@ def publish(self):
 
         Uploads the results of a run to OpenML.
         """
-        predictions = arff.dumps(self._generate_arff())
+        predictions = arff.dumps(self._generate_arff_header_dict())
         description_xml = self._create_description_xml()
         file_elements = {'predictions': ("predictions.csv", predictions),
                          'description': ("description.xml", description_xml)}
@@ -152,8 +152,18 @@ def run_task(task, model):
     setup_string = _create_setup_string(model)
 
     run = OpenMLRun(task.task_id, flow_id, setup_string, dataset.id)
+    run.data_content = _run_task_get_arffcontent(model, task, class_labels)
 
-    train_times = []
+    # The model will not be uploaded at the moment, but used to get the
+    # hyperparameter values when uploading the run
+    X, Y = task.get_X_and_y()
+    run.model = model.fit(X, Y)
+    return run
+
+
+def _run_task_get_arffcontent(model, task, class_labels):
+    X, Y = task.get_X_and_y()
+    arff_datacontent = []
 
     rep_no = 0
     # TODO use different iterator to only provide a single iterator (less
@@ -167,26 +177,21 @@ def run_task(task, model):
             testX = X[test_indices]
             testY = Y[test_indices]
 
-            start_time = time.time()
             model.fit(trainX, trainY)
             ProbaY = model.predict_proba(testX)
             PredY = model.predict(testX)
-            end_time = time.time()
-
-            train_times.append(end_time - start_time)
 
             for i in range(0, len(test_indices)):
-                arff_line = [rep_no, fold_no, test_indices[i],
-                             class_labels[PredY[i]], class_labels[testY[i]]]
-                arff_line[3:3] = ProbaY[i]
+                arff_line = [rep_no, fold_no, test_indices[i]]
+                arff_line.extend(ProbaY[i])
+                arff_line.append(class_labels[PredY[i]])
+                arff_line.append(class_labels[testY[i]])
                 arff_datacontent.append(arff_line)
 
             fold_no = fold_no + 1
         rep_no = rep_no + 1
 
-    run.data_content = arff_datacontent
-    run.model = model.fit(X, Y)
-    return run
+    return arff_datacontent
 
 
 def _to_dict(taskid, flow_id, setup_string, parameter_settings, tags):

tests/runs/test_runs.py

@@ -1,4 +1,4 @@
-from sklearn.linear_model import LogisticRegression
+from sklearn.linear_model import LogisticRegression, SGDClassifier
 import openml
 from openml.testing import TestBase
 
@@ -12,6 +12,36 @@ def test_run_iris(self):
         self.assertEqual(return_code, 200)
         # self.assertTrue("This is a read-only account" in return_value)
 
+    def test__run_task_get_arffcontent(self):
+        task = openml.tasks.get_task(1939)
+        class_labels = task.class_labels
+
+        clf = SGDClassifier(loss='hinge', random_state=1)
+        self.assertRaisesRegex(AttributeError,
+                               "probability estimates are not available for loss='hinge'",
+                               openml.runs.run._run_task_get_arffcontent,
+                               clf, task, class_labels)
+
+        clf = SGDClassifier(loss='log', random_state=1)
+        arff_datacontent = openml.runs.run._run_task_get_arffcontent(
+            clf, task, class_labels)
+        self.assertIsInstance(arff_datacontent, list)
+        # 10 times 10 fold CV of 150 samples
+        self.assertEqual(len(arff_datacontent), 1500)
+        for arff_line in arff_datacontent:
+            self.assertEqual(len(arff_line), 8)
+            self.assertGreaterEqual(arff_line[0], 0)
+            self.assertLessEqual(arff_line[0], 9)
+            self.assertGreaterEqual(arff_line[1], 0)
+            self.assertLessEqual(arff_line[1], 9)
+            self.assertGreaterEqual(arff_line[2], 0)
+            self.assertLessEqual(arff_line[2], 149)
+            self.assertAlmostEqual(sum(arff_line[3:6]), 1.0)
+            self.assertIn(arff_line[6], ['Iris-setosa', 'Iris-versicolor',
+                                         'Iris-virginica'])
+            self.assertIn(arff_line[7], ['Iris-setosa', 'Iris-versicolor',
+                                         'Iris-virginica'])
+
     def test_get_run(self):
         run = openml.runs.get_run(473350)
         self.assertEqual(run.dataset_id, 1167)