Incompatible task / flows are uploaded with error message

Author: janvanrijn

openml/runs/functions.py

@@ -50,7 +50,11 @@ def run_task(task, model):
 
     # execute the run
     run = OpenMLRun(task_id=task.task_id, flow_id=None, dataset_id=dataset.dataset_id, model=model)
-    run.data_content, run.trace_content = _run_task_get_arffcontent(model, task, class_labels)
+
+    try:
+        run.data_content, run.trace_content = _run_task_get_arffcontent(model, task, class_labels)
+    except AttributeError as message:
+        run.error_message = str(message)
 
     # now generate the flow
     flow = sklearn_to_flow(model)

openml/runs/run.py

@@ -43,6 +43,7 @@ def __init__(self, task_id, flow_id, dataset_id, setup_string=None,
         self.detailed_evaluations = detailed_evaluations
         self.data_content = data_content
         self.trace_content = trace_content
+        self.error_message = None
         self.task = task
         self.flow = flow
         self.run_id = run_id
@@ -136,12 +137,13 @@ def publish(self):
         if self.flow_id is None:
             raise PyOpenMLError("OpenMLRun obj does not contain a flow id. (Should have been added while executing the task.) ");
 
-
-        predictions = arff.dumps(self._generate_arff_dict())
         description_xml = self._create_description_xml()
+        file_elements = {'description': ("description.xml", description_xml)}
+
+        if self.error_message is None:
+            predictions = arff.dumps(self._generate_arff_dict())
+            file_elements['predictions'] = ("predictions.arff", predictions)
 
-        file_elements = {'predictions': ("predictions.arff", predictions),
-                         'description': ("description.xml", description_xml)}
         if self.trace_content is not None:
             trace_arff = arff.dumps(self._generate_trace_arff_dict(self.model))
             file_elements['trace'] = ("trace.arff", trace_arff)
@@ -175,6 +177,7 @@ def _create_description_xml(self):
         description = _to_dict(taskid=self.task_id, flow_id=self.flow_id,
                                setup_string=_create_setup_string(self.model),
                                parameter_settings=openml_param_settings,
+                               error_message=self.error_message,
                                tags=tags)
         description_xml = xmltodict.unparse(description, pretty=True)
         return description_xml
@@ -256,7 +259,7 @@ def _get_version_information():
     return [python_version, sklearn_version, numpy_version, scipy_version]
 
 
-def _to_dict(taskid, flow_id, setup_string, parameter_settings, tags):
+def _to_dict(taskid, flow_id, setup_string, error_message, parameter_settings, tags):
     """ Creates a dictionary corresponding to the desired xml desired by openML
 
     Parameters
@@ -280,6 +283,8 @@ def _to_dict(taskid, flow_id, setup_string, parameter_settings, tags):
     description['oml:run']['@xmlns:oml'] = 'http://openml.org/openml'
     description['oml:run']['oml:task_id'] = taskid
     description['oml:run']['oml:flow_id'] = flow_id
+    if error_message is not None:
+        description['oml:run']['oml:error_message'] = error_message
     description['oml:run']['oml:parameter_setting'] = parameter_settings
     description['oml:run']['oml:tag'] = tags  # Tags describing the run
     # description['oml:run']['oml:output_data'] = 0;

tests/test_runs/test_run_functions.py

@@ -35,7 +35,11 @@ def test_run_regression_on_classif_task(self):
 
         clf = LinearRegression()
         task = openml.tasks.get_task(task_id)
-        self.assertRaises(AttributeError, openml.runs.run_task, task=task, model=clf)
+        run = openml.runs.run_task(task=task, model=clf)
+        run.publish()
+
+        # TODO: download and check whether it really contains the error message
+        #downloaded_run = openml.runs.get_run(run.run_id)
 
     @mock.patch('openml.flows.sklearn_to_flow')
     def test_check_erronous_sklearn_flow_fails(self, sklearn_to_flow_mock):