Merge branch 'develop' into joaquinvanschoren-patch-1

Author: janvanrijn

examples/OpenMLDemo.ipynb

@@ -22,11 +22,13 @@
 from . import runs
 from . import flows
 from . import setups
+from . import study
+from . import evaluations
 from .runs import OpenMLRun
 from .tasks import OpenMLTask, OpenMLSplit
 from .flows import OpenMLFlow
 
-__version__ = "0.4.0dev"
+from .__version__ import __version__
 
 
 def populate_cache(task_ids=None, dataset_ids=None, flow_ids=None,
@@ -66,5 +68,6 @@ def populate_cache(task_ids=None, dataset_ids=None, flow_ids=None,
 
 
 __all__ = ['OpenMLDataset', 'OpenMLDataFeature', 'OpenMLRun',
-           'OpenMLSplit', 'datasets', 'OpenMLTask', 'OpenMLFlow',
+           'OpenMLSplit', 'OpenMLEvaluation', 'OpenMLSetup',
+           'OpenMLTask', 'OpenMLFlow', 'datasets', 'evaluations',
            'config', 'runs', 'flows', 'tasks', 'setups']

examples/OpenML_Tutorial.ipynb

@@ -0,0 +1,4 @@
+"""Version information."""
+
+# The following line *must* be the last in the module, exactly as formatted:
+__version__ = "0.5.0dev"

examples/PyOpenML.ipynb

@@ -117,6 +117,7 @@ def _read_url(url, data=None):
         warnings.warn('Received uncompressed content from OpenML for %s.' % url)
     return response.text
 
+
 def _parse_server_exception(response):
     # OpenML has a sopisticated error system
     # where information about failures is provided. try to parse this

openml/__init__.py

@@ -39,7 +39,7 @@ def __init__(self, dataset_id=None, name=None, version=None, description=None,
                  row_id_attribute=None, ignore_attribute=None,
                  version_label=None, citation=None, tag=None, visibility=None,
                  original_data_url=None, paper_url=None, update_comment=None,
-                 md5_checksum=None, data_file=None, features=None):
+                 md5_checksum=None, data_file=None, features=None, qualities=None):
         # Attributes received by querying the RESTful API
         self.dataset_id = int(dataset_id) if dataset_id is not None else None
         self.name = name
@@ -74,6 +74,7 @@ def __init__(self, dataset_id=None, name=None, version=None, description=None,
         self.md5_cheksum = md5_checksum
         self.data_file = data_file
         self.features = None
+        self.qualities = None
 
         if features is not None:
             self.features = {}
@@ -87,6 +88,12 @@ def __init__(self, dataset_id=None, name=None, version=None, description=None,
                     raise ValueError('Data features not provided in right order')
                 self.features[feature.index] = feature
 
+        if qualities is not None:
+            self.qualities = {}
+            for idx, xmlquality in enumerate(qualities['oml:quality']):
+                name = xmlquality['oml:name']
+                value = xmlquality['oml:value']
+                self.qualities[name] = value
 
         if data_file is not None:
             if self._data_features_supported():

openml/__version__.py

@@ -75,7 +75,8 @@ def _get_cached_dataset(dataset_id):
     description = _get_cached_dataset_description(dataset_id)
     arff_file = _get_cached_dataset_arff(dataset_id)
     features = _get_cached_dataset_features(dataset_id)
-    dataset = _create_dataset_from_description(description, features, arff_file)
+    qualities = _get_cached_dataset_qualities(dataset_id)
+    dataset = _create_dataset_from_description(description, features, qualities, arff_file)
 
     return dataset
 
@@ -107,6 +108,19 @@ def _get_cached_dataset_features(dataset_id):
                                    "cached" % dataset_id)
 
 
+def _get_cached_dataset_qualities(dataset_id):
+    cache_dir = config.get_cache_directory()
+    did_cache_dir = os.path.join(cache_dir, "datasets", str(dataset_id))
+    qualities_file = os.path.join(did_cache_dir, "qualities.xml")
+    try:
+        with io.open(qualities_file, encoding='utf8') as fh:
+            qualities_xml = fh.read()
+            return xmltodict.parse(qualities_xml)["oml:data_qualities"]
+    except (IOError, OSError):
+        raise OpenMLCacheException("Dataset qualities for dataset id %d not "
+                                   "cached" % dataset_id)
+
+
 def _get_cached_dataset_arff(dataset_id):
     cache_dir = config.get_cache_directory()
     did_cache_dir = os.path.join(cache_dir, "datasets", str(dataset_id))
@@ -272,7 +286,7 @@ def get_dataset(dataset_id):
         _remove_dataset_cache_dir(did_cache_dir)
         raise e
 
-    dataset = _create_dataset_from_description(description, features, arff_file)
+    dataset = _create_dataset_from_description(description, features, qualities, arff_file)
     return dataset
 
 
@@ -470,7 +484,7 @@ def _remove_dataset_cache_dir(did_cache_dir):
                              'Please do this manually!' % did_cache_dir)
 
 
-def _create_dataset_from_description(description, features, arff_file):
+def _create_dataset_from_description(description, features, qualities, arff_file):
     """Create a dataset object from a description dict.
 
     Parameters
@@ -510,5 +524,6 @@ def _create_dataset_from_description(description, features, arff_file):
         description.get("oml:update_comment"),
         description.get("oml:md5_checksum"),
         data_file=arff_file,
-        features=features)
+        features=features,
+        qualities=qualities)
     return dataset

openml/_api_calls.py

@@ -0,0 +1,2 @@
+from .evaluation import OpenMLEvaluation
+from .functions import list_evaluations

openml/datasets/dataset.py

@@ -0,0 +1,40 @@
+
+class OpenMLEvaluation(object):
+    '''
+    Contains all meta-information about a run / evaluation combination,
+    according to the evaluation/list function
+
+    Parameters
+       ----------
+        run_id : int
+        task_id : int
+        setup_id : int
+        flow_id : int
+        flow_name : str
+        data_id : int
+        data_name : str
+            the name of the dataset
+        function : str
+            the evaluation function of this item (e.g., accuracy)
+        upload_time : str
+            the time of evaluation
+        value : float
+            the value of this evaluation
+        array_data : str
+            list of information per class (e.g., in case of precision, auroc, recall)
+    '''
+    def __init__(self, run_id, task_id, setup_id, flow_id, flow_name,
+                 data_id, data_name, function, upload_time, value,
+                 array_data=None):
+        self.run_id = run_id
+        self.task_id = task_id
+        self.setup_id = setup_id
+        self.flow_id = flow_id
+        self.flow_name = flow_name
+        self.data_id = data_id
+        self.data_name = data_name
+        self.function = function
+        self.upload_time = upload_time
+        self.value = value
+        self.array_data = array_data
+