Merge pull request #353 from openml/fix_345

Cast data qualities to float

Author: amueller

openml/datasets/dataset.py

@@ -88,12 +88,7 @@ def __init__(self, dataset_id=None, name=None, version=None, description=None,
                     raise ValueError('Data features not provided in right order')
                 self.features[feature.index] = feature
 
-        if qualities is not None:
-            self.qualities = {}
-            for idx, xmlquality in enumerate(qualities['oml:quality']):
-                name = xmlquality['oml:name']
-                value = xmlquality['oml:value']
-                self.qualities[name] = value
+        self.qualities = _check_qualities(qualities)
 
         if data_file is not None:
             if self._data_features_supported():
@@ -426,3 +421,21 @@ def _data_features_supported(self):
                     return False
             return True
         return True
+
+
+
+def _check_qualities(qualities):
+    if qualities is not None:
+        qualities_ = {}
+        for xmlquality in qualities:
+            name = xmlquality['oml:name']
+            if xmlquality['oml:value'] is None:
+                value = float('NaN')
+            elif xmlquality['oml:value'] == 'null':
+                value = float('NaN')
+            else:
+                value = float(xmlquality['oml:value'])
+            qualities_[name] = value
+        return qualities_
+    else:
+        return None

openml/datasets/functions.py

@@ -116,7 +116,7 @@ def _get_cached_dataset_qualities(dataset_id):
     try:
         with io.open(qualities_file, encoding='utf8') as fh:
             qualities_xml = fh.read()
-            return xmltodict.parse(qualities_xml)["oml:data_qualities"]
+            return xmltodict.parse(qualities_xml)["oml:data_qualities"]['oml:quality']
     except (IOError, OSError):
         raise OpenMLCacheException("Dataset qualities for dataset id %d not "
                                    "cached" % dataset_id)
@@ -451,7 +451,7 @@ def _get_dataset_qualities(did_cache_dir, dataset_id):
         with io.open(qualities_file, "w", encoding='utf8') as fh:
             fh.write(qualities_xml)
 
-    qualities = xmltodict.parse(qualities_xml, force_list=('oml:quality',))['oml:data_qualities']
+    qualities = xmltodict.parse(qualities_xml, force_list=('oml:quality',))['oml:data_qualities']['oml:quality']
 
     return qualities
 

tests/test_datasets/test_dataset.py

@@ -182,3 +182,18 @@ def test_get_sparse_dataset_rowid_and_ignore_and_target(self):
         self.assertEqual(len(categorical), 19998)
         self.assertListEqual(categorical, [False] * 19998)
         self.assertEqual(y.shape, (600, ))
+
+
+class OpenMLDatasetQualityTest(TestBase):
+    def test__check_qualities(self):
+        qualities = [{'oml:name': 'a', 'oml:value': '0.5'}]
+        qualities = openml.datasets.dataset._check_qualities(qualities)
+        self.assertEqual(qualities['a'], 0.5)
+
+        qualities = [{'oml:name': 'a', 'oml:value': 'null'}]
+        qualities = openml.datasets.dataset._check_qualities(qualities)
+        self.assertNotEqual(qualities['a'], qualities['a'])
+
+        qualities = [{'oml:name': 'a', 'oml:value': None}]
+        qualities = openml.datasets.dataset._check_qualities(qualities)
+        self.assertNotEqual(qualities['a'], qualities['a'])

tests/test_datasets/test_dataset_functions.py

@@ -74,7 +74,7 @@ def test__get_cached_dataset(self, ):
         self.assertIsInstance(dataset, OpenMLDataset)
         self.assertTrue(len(dataset.features) > 0)
         self.assertTrue(len(dataset.features) == len(features['oml:feature']))
-        self.assertTrue(len(dataset.qualities) == len(qualities['oml:quality']))
+        self.assertTrue(len(dataset.qualities) == len(qualities))
 
     def test_get_cached_dataset_description(self):
         openml.config.set_cache_directory(self.static_cache_dir)
@@ -210,7 +210,7 @@ def test__get_dataset_features(self):
     def test__get_dataset_qualities(self):
         # Only a smoke check
         qualities = _get_dataset_qualities(self.workdir, 2)
-        self.assertIsInstance(qualities, dict)
+        self.assertIsInstance(qualities, list)
 
     def test_deletion_of_cache_dir(self):
         # Simple removal