initial commit

Author: janvanrijn

openml/evaluations/__init__.py

@@ -0,0 +1,2 @@
+from .evaluation import OpenMLEvaluation
+from .functions import list_evaluations

openml/evaluations/evaluation.py

@@ -0,0 +1,16 @@
+
+class OpenMLEvaluation(object):
+
+    def __init__(self, run_id, task_id, setup_id, flow_id, flow_name,
+                 data_name, function, upload_time, value, array_data):
+        self.run_id = run_id
+        self.task_id = task_id
+        self.setup_id = setup_id
+        self.flow_id = flow_id
+        self.flow_name = flow_name
+        self.data_name = data_name
+        self.function = function
+        self.upload_time = upload_time
+        self.value = value
+        self.array_data = array_data
+

openml/evaluations/functions.py

@@ -0,0 +1,34 @@
+import xmltodict
+
+from .._api_calls import _perform_api_call
+from ..evaluations import OpenMLEvaluation
+
+def list_evaluations(function, task_id):
+    """Helper function to parse API calls which are lists of runs"""
+
+    xml_string = _perform_api_call("evaluation/list/funtion/%s/task_id/%d" %(function, task_id))
+
+    evals_dict = xmltodict.parse(xml_string)
+    # Minimalistic check if the XML is useful
+    if 'oml:evaluations' not in evals_dict:
+        raise ValueError('Error in return XML, does not contain "oml:evaluations": %s'
+                         % str(evals_dict))
+
+    if isinstance(evals_dict['oml:evaluations']['oml:evaluation'], list):
+        evals_list = evals_dict['oml:evaluations']['oml:evaluation']
+    elif isinstance(evals_dict['oml:evaluations']['oml:evaluation'], dict):
+        evals_list = [evals_dict['oml:runs']['oml:run']]
+    else:
+        raise TypeError()
+
+    evals = dict()
+    for eval_ in evals_list:
+        run_id = int(eval_['oml:run_id'])
+        evaluation = OpenMLEvaluation(int(eval_['oml:run_id']), int(eval_['task_id']),
+                                      int(eval_['oml:setup_id']), int(eval_['oml:flow_id']),
+                                      eval_['oml:flow_name'], eval_['oml:data_name'],
+                                      eval_['oml:function'], eval_['oml:upload_time'],
+                                      float(eval_['oml:value']), eval_['oml:array_data'])
+        evals[run_id] = evaluation
+    return evaluation
+

tests/test_evaluations/test_evaluation_functions.py

@@ -0,0 +1,14 @@
+import unittest
+import openml
+import openml.evaluations
+from openml.testing import TestBase
+
+class TestEvaluationFunctions(TestBase):
+
+    def test_evaluation_list(self):
+        openml.config.server = self.production_server
+
+        res = openml.evaluations.list_evaluations("predictive_accuracy", 59)
+
+        self.assertGreater(len(res), 100)
+