Base functionalities for uploading runs and optimization traces

Author: janvanrijn

openml/flows/__init__.py

@@ -1,4 +1,4 @@
 from .flow import OpenMLFlow, create_flow_from_model
-from .functions import get_flow
+from .functions import get_flow, get_flow_dict
 
-__all__ = ['OpenMLFlow', 'create_flow_from_model', 'get_flow']
+__all__ = ['OpenMLFlow', 'create_flow_from_model', 'get_flow', 'get_flow_dict']

openml/flows/flow.py

@@ -394,19 +394,16 @@ def _ensure_flow_exists(self):
         # TODO add numpy and scipy version!
 
         if int(flow_id) == -1:
-            return_code, response_xml = self.publish()
+            flow = self.publish()
 
-            response_dict = xmltodict.parse(response_xml)
-            flow_id = response_dict['oml:upload_flow']['oml:id']
-            return int(flow_id)
+            return int(flow.flow_id)
 
         return int(flow_id)
 
     def _get_name(self):
         """Helper function. Can be mocked for testing."""
         return self.name
 
-
 def create_flow_from_model(model, converter, description=None):
     """Use a converter to create an OpenMLFlow from model.
 

openml/flows/functions.py

@@ -1,6 +1,7 @@
 import xmltodict
 
 from openml._api_calls import _perform_api_call
+from openml.exceptions import PyOpenMLError
 from . import OpenMLFlow
 from ..util import URLError
 
@@ -34,3 +35,18 @@ def get_flow(flow_id, converter=None):
         flow.model = model
 
     return flow
+
+
+def get_flow_dict(flow):
+    """Returns a dictionary with keys flow name and values flow id.
+
+        Parameters
+        ----------
+        flow : OpenMLFlow
+        """
+    if flow.flow_id is None:
+        raise PyOpenMLError("Can only invoke function 'get_flow_map' on a server downloaded flow. ")
+    flow_map = {flow.name : flow.flow_id}
+    for subflow in flow.components:
+        flow_map.update(get_flow_dict(flow.components[subflow]))
+    return flow_map

openml/runs/functions.py

@@ -2,6 +2,7 @@
 import io
 import os
 import xmltodict
+from sklearn.grid_search import BaseSearchCV
 
 from .. import config
 from ..flows import create_flow_from_model
@@ -57,19 +58,16 @@ def run_task(task, model):
                          'only works for tasks with class labels.')
 
     run = OpenMLRun(task_id=task.task_id, flow_id=flow_id,
-                    dataset_id=dataset.dataset_id)
-    run.data_content = _run_task_get_arffcontent(model, task, class_labels)
+                    dataset_id=dataset.dataset_id, model=model)
+    run.data_content, run.trace_content = _run_task_get_arffcontent(model, task, class_labels)
 
-    # The model will not be uploaded at the moment, but used to get the
-    # hyperparameter values when uploading the run
-    X, Y = task.get_X_and_y()
-    run.model = model.fit(X, Y)
     return run
 
 
 def _run_task_get_arffcontent(model, task, class_labels):
     X, Y = task.get_X_and_y()
     arff_datacontent = []
+    arff_tracecontent = []
 
     rep_no = 0
     # TODO use different iterator to only provide a single iterator (less
@@ -84,6 +82,16 @@ def _run_task_get_arffcontent(model, task, class_labels):
             testY = Y[test_indices]
 
             model.fit(trainX, trainY)
+            if isinstance(model, BaseSearchCV):
+                for itt_no in range(0, len(model.grid_scores_)):
+                    current = model.grid_scores_[itt_no]
+                    # we use the string values for True and False, as it is defined in this way by the OpenML server
+                    selected = 'false'
+                    if current.parameters == model.best_params_:
+                       selected = 'true'
+                    arff_line = [rep_no, fold_no, itt_no, current.parameters, current.mean_validation_score, selected]
+                    arff_tracecontent.append(arff_line)
+
             ProbaY = model.predict_proba(testX)
             PredY = model.predict(testX)
 
@@ -97,7 +105,10 @@ def _run_task_get_arffcontent(model, task, class_labels):
             fold_no = fold_no + 1
         rep_no = rep_no + 1
 
-    return arff_datacontent
+    if not isinstance(model, BaseSearchCV):
+        arff_tracecontent = None
+
+    return arff_datacontent, arff_tracecontent
 
 
 def get_runs(run_ids):

openml/runs/run.py

@@ -4,10 +4,12 @@
 
 import arff
 import xmltodict
+from sklearn.base import BaseEstimator
 
+import openml
 from ..tasks import get_task
 from .._api_calls import _perform_api_call
-
+from ..exceptions import PyOpenMLError
 
 class OpenMLRun(object):
     """OpenML Run: result of running a model on an openml dataset.
@@ -17,10 +19,10 @@ class OpenMLRun(object):
     FIXME
 
     """
-    def __init__(self, task_id, flow_id, dataset_id, setup_string=None, 
+    def __init__(self, task_id, flow_id, dataset_id, setup_string=None,
                  files=None, setup_id=None, tags=None, uploader=None, uploader_name=None,
                  evaluations=None, detailed_evaluations=None,
-                 data_content=None, model=None, task_type=None,
+                 data_content=None, trace_content=None, model=None, task_type=None,
                  task_evaluation_measure=None, flow_name=None,
                  parameter_settings=None, predictions_url=None, task=None,
                  flow=None, run_id=None):
@@ -39,12 +41,14 @@ def __init__(self, task_id, flow_id, dataset_id, setup_string=None,
         self.evaluations = evaluations
         self.detailed_evaluations = detailed_evaluations
         self.data_content = data_content
+        self.trace_content = trace_content
         self.task = task
         self.flow = flow
         self.run_id = run_id
+        self.model = model
 
     def _generate_arff_dict(self):
-        """Generates the arff dictionary for upload to the server.
+        """Generates the arff dictionary for uploading predictions to the server.
 
         Assumes that the run has been executed.
 
@@ -74,6 +78,20 @@ def _generate_arff_dict(self):
         arff_dict['relation'] = 'openml_task_' + str(task.task_id) + '_predictions'
         return arff_dict
 
+    def _generate_trace_arff_dict(self):
+        if self.trace_content is None:
+            raise ValueError('No trace content avaiable. (This should never happen.)')
+        arff_dict = {}
+        arff_dict['attributes'] = [('repeat', 'NUMERIC'),
+                                   ('fold', 'NUMERIC'),
+                                   ('iteration', 'NUMERIC'),
+                                   ('setup_string', 'STRING'),
+                                   ('evaluation', 'NUMERIC'),
+                                   ('selected', ['true', 'false'])]
+        arff_dict['data'] = self.trace_content
+        arff_dict['relation'] = 'openml_task_' + str(self.task_id) + '_predictions'
+        return arff_dict
+
     def publish(self):
         """Publish a run to the OpenML server.
 
@@ -84,10 +102,18 @@ def publish(self):
         -------
         self : OpenMLRun
         """
+        if self.model is None:
+            raise PyOpenMLError("OpenMLRun obj does not contain a model. (This should never happen.) ");
+
         predictions = arff.dumps(self._generate_arff_dict())
         description_xml = self._create_description_xml()
-        file_elements = {'predictions': ("predictions.csv", predictions),
+
+        file_elements = {'predictions': ("predictions.arff", predictions),
                          'description': ("description.xml", description_xml)}
+        if self.trace_content is not None:
+            trace_arff = arff.dumps(self._generate_trace_arff_dict())
+            file_elements['trace'] = ("trace.arff", trace_arff)
+
         return_code, return_value = _perform_api_call(
             "/run/", file_elements=file_elements)
         run_id = int(xmltodict.parse(return_value)['oml:upload_run']['oml:run_id'])
@@ -104,7 +130,11 @@ def _create_description_xml(self):
         """
         run_environment = _get_version_information()
 
-        parameter_settings = self.model.get_params()
+        # TODO: don't we have flow object in data structure? Use this one
+        downloaded_flow = openml.flows.get_flow(self.flow_id)
+
+        openml_param_settings = _parse_parameters(self.model, downloaded_flow)
+
         # as a tag, it must be of the form ([a-zA-Z0-9_\-\.])+
         # so we format time from 'mm/dd/yy hh:mm:ss' to 'mm-dd-yy_hh.mm.ss'
         well_formatted_time = time.strftime("%c").replace(
@@ -113,11 +143,33 @@ def _create_description_xml(self):
             [self.model.__module__ + "." + self.model.__class__.__name__]
         description = _to_dict(taskid=self.task_id, flow_id=self.flow_id,
                                setup_string=_create_setup_string(self.model),
-                               parameter_settings=parameter_settings,
+                               parameter_settings=openml_param_settings,
                                tags=tags)
         description_xml = xmltodict.unparse(description, pretty=True)
         return description_xml
 
+def _parse_parameters(model, flow):
+    python_param_settings = model.get_params()
+    openml_param_settings = []
+    flow_dict = openml.flows.get_flow_dict(flow)
+
+    for param in python_param_settings:
+        if "__" in param:
+            # parameter of subflow. will be handled later
+            continue
+        if isinstance(python_param_settings[param], BaseEstimator):
+            # extract parameters of the subflow individually
+            subflow = flow.components[param]
+
+        # add parameter setting (also the subflow. Just because we can)
+        param_dict = OrderedDict()
+        param_dict['oml:name'] = param;
+        param_dict['oml:value'] = str(python_param_settings[param]);
+        param_dict['oml:component'] = flow_dict[flow.name];
+        openml_param_settings.append(param_dict)
+
+    return openml_param_settings
+
 ################################################################################
 # Functions which cannot be in runs/functions due to circular imports
 
@@ -169,15 +221,7 @@ def _to_dict(taskid, flow_id, setup_string, parameter_settings, tags):
     description['oml:run']['@xmlns:oml'] = 'http://openml.org/openml'
     description['oml:run']['oml:task_id'] = taskid
     description['oml:run']['oml:flow_id'] = flow_id
-
-    params = []
-    for k, v in parameter_settings.items():
-        param_dict = OrderedDict()
-        param_dict['oml:name'] = k
-        param_dict['oml:value'] = ('None' if v is None else v)
-        params.append(param_dict)
-
-    description['oml:run']['oml:parameter_setting'] = params
+    description['oml:run']['oml:parameter_setting'] = parameter_settings
     description['oml:run']['oml:tag'] = tags  # Tags describing the run
     # description['oml:run']['oml:output_data'] = 0;
     # all data that was output of this run, which can be evaluation scores

tests/runs/test_run_functions.py

@@ -1,4 +1,6 @@
 from sklearn.linear_model import LogisticRegression, SGDClassifier
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.grid_search import RandomizedSearchCV
 import openml
 from openml.testing import TestBase
 
@@ -12,6 +14,24 @@ def test_run_iris(self):
         self.assertEqual(run_, run)
         self.assertIsInstance(run.dataset_id, int)
 
+    def test_run_optimize_randomforest_iris(self):
+        task = openml.tasks.get_task(10107)
+        clf = RandomForestClassifier(n_estimators=5)
+
+        param_dist = {"max_depth": [3, None],
+                      "max_features": [1,2,3,4],
+                      "min_samples_split": [1,2,3,4,5,6,7,8,9,10],
+                      "min_samples_leaf": [1,2,3,4,5,6,7,8,9,10],
+                      "bootstrap": [True, False],
+                      "criterion": ["gini", "entropy"]}
+        random_search = RandomizedSearchCV(clf, param_distributions=param_dist,n_iter=20)
+
+
+        run = openml.runs.run_task(task, random_search)
+        run_ = run.publish()
+        self.assertEqual(run_, run)
+        self.assertIsInstance(run.dataset_id, int)
+
     def test__run_task_get_arffcontent(self):
         task = openml.tasks.get_task(1939)
         class_labels = task.class_labels