make get_dataset_qualities, get_dataset_description, get_dataset_features private, document dataset functions

Author: amueller

openml/datasets/__init__.py

@@ -1,11 +1,7 @@
 from .functions import (list_datasets, list_datasets_by_tag,
-                        check_datasets_active, get_datasets, get_dataset,
-                        _get_dataset_description,
-                        _get_dataset_features, _get_dataset_qualities)
+                        check_datasets_active, get_datasets, get_dataset)
 from .dataset import OpenMLDataset
 
 __all__ = ['check_datasets_active', 'get_dataset', 'get_datasets',
-           'get_datasets_arf', '_get_dataset_features',
-           '_get_dataset_qualities', 'OpenMLDataset', 'list_datasets',
-           'list_datasets_by_tag',
-           '_get_dataset_description', 'list_datasets']
+           'OpenMLDataset', 'list_datasets', 'list_datasets_by_tag',
+           'list_datasets']

openml/datasets/dataset.py

@@ -110,6 +110,17 @@ def __eq__(self, other):
             return False
 
     def _get_arff(self):
+        """Read ARFF file and return decoded arff.
+
+        Reads the file referenced in self.data_file.
+
+        Returns
+        -------
+        arff_string :
+            Decoded arff.
+
+        """
+
         # TODO: add a partial read method which only returns the attribute
         # headers of the corresponding .arff file!
 
@@ -226,7 +237,7 @@ def get_data(self, target=None, target_dtype=int, include_row_id=False,
         else:
             return rval
 
-    def retrieve_class_labels(self):
+    def _retrieve_class_labels(self):
         """Reads the datasets arff to determine the class-labels, and returns those.
         If the task has no class labels (for example a regression problem) it returns None."""
         # TODO improve performance, currently reads the whole file
@@ -248,16 +259,28 @@ def publish(self):
 
         Returns
         -------
+        return_code : int
+            Return code from server
+
+        return_value : string
+            xml return from server
         """
-        data = {'description': self.to_xml()}
+        data = {'description': self._to_xml()}
         if self.data_file is not None:
             return_code, return_value = _perform_api_call(
                 "/data/", data=data, file_dictionary={'dataset': self.data_file})
         else:
             return_code, return_value = _perform_api_call("/data/", data=data)
         return return_code, return_value
 
-    def to_xml(self):
+    def _to_xml(self):
+        """Serialize object to xml for upload
+
+        Returns
+        -------
+        xml_dataset : string
+            XML description of the data.
+        """
         xml_dataset = ('<oml:data_set_description '
                        'xmlns:oml="http://openml.org/openml">')
         props = ['id', 'name', 'version', 'description', 'format', 'creator',

openml/datasets/functions.py

@@ -110,7 +110,7 @@ def list_datasets():
 
     Returns
     -------
-    list
+    datasets : list of dicts
         A list of all datasets. Every dataset is represented by a
         dictionary containing the following information: dataset id,
         and status. If qualities are calculated for the dataset, some of
@@ -124,7 +124,7 @@ def list_datasets_by_tag(tag):
 
     Returns
     -------
-    list
+    datasets : list of dicts
         A list of all datasets having the given tag. Every dataset is
         represented by a dictionary containing the following information:
         dataset id, and status. If qualities are calculated for the dataset,
@@ -174,7 +174,7 @@ def check_datasets_active(dids):
         A list of integers representing dataset ids.
 
     Returns
-    -------
+    active : dict of int to boolean
     dict
         A dictionary with items {did: active}, where active is a boolean. It
         is set to True if the dataset is active.
@@ -202,7 +202,7 @@ def get_datasets(dids):
 
     Returns
     -------
-    list
+    datasets : list of datasets
         A list of dataset objects.
 
     Notes
@@ -282,6 +282,24 @@ def _get_dataset_description(did):
 
 
 def _get_dataset_arff(did, description=None):
+    """Load dataset arff (from cache or download).
+
+    Tries to load did from cache. If that fails, uses
+    ``description`` (fetched if none) to download arff.
+
+    Parameters
+    ----------
+    did : int
+        Dataset ID
+
+    description : dictionary
+        Dataset description dict.
+
+    Returns
+    -------
+    output_filename : string
+        Location of arff file.
+    """
     did_cache_dir = _create_dataset_cache_directory(did)
     output_file = os.path.join(did_cache_dir, "dataset.arff")
 
@@ -308,6 +326,21 @@ def _get_dataset_arff(did, description=None):
 
 
 def _get_dataset_features(did):
+    """API call to get dataset features (cached)
+
+    Features are feature descriptions for each column.
+    (name, index, categorical, ...)
+
+    Parameters
+    ----------
+    did : int
+        Dataset ID
+
+    Returns
+    -------
+    features : dict
+        Dictionary containing dataset feature descriptions.
+    """
     did_cache_dir = _create_dataset_cache_directory(did)
     features_file = os.path.join(did_cache_dir, "features.xml")
 
@@ -338,6 +371,20 @@ def _get_dataset_features(did):
 
 
 def _get_dataset_qualities(did):
+    """API call to get dataset qualities (cached)
+
+    Features are metafeatures (number of features, number of classes, ...)
+
+    Parameters
+    ----------
+    did : int
+        Dataset ID
+
+    Returns
+    -------
+    qualities : dict
+        Dictionary containing dataset qualities.
+    """
     # Dataset qualities are subject to change and must be fetched every time
     did_cache_dir = _create_dataset_cache_directory(did)
     qualities_file = os.path.join(did_cache_dir, "qualities.xml")
@@ -362,6 +409,7 @@ def _get_dataset_qualities(did):
 
 
 def _create_dataset_cache_directory(did):
+    """Create a dataset cache directory"""
     dataset_cache_dir = os.path.join(config.get_cache_directory(), "datasets", str(did))
     try:
         os.makedirs(dataset_cache_dir)
@@ -372,6 +420,7 @@ def _create_dataset_cache_directory(did):
 
 
 def _remove_dataset_chache_dir(did):
+    """Remove the dataset cache directory"""
     dataset_cache_dir = os.path.join(config.get_cache_directory(), "datasets", str(did))
     try:
         os.rmdir(dataset_cache_dir)
@@ -381,6 +430,20 @@ def _remove_dataset_chache_dir(did):
 
 
 def _create_dataset_from_description(description, arff_file):
+    """Create a dataset object from a description dict.
+
+    Parameters
+    ----------
+    description : dict
+        Description of a dataset in xmlish dict.
+    arff_file : string
+        Path of dataset arff file.
+
+    Returns
+    -------
+    dataset : dataset object
+        Dataset object from dict and arff.
+    """
     dataset = OpenMLDataset(
         description["oml:id"],
         description["oml:name"],

tests/test_datasets.py

@@ -12,7 +12,11 @@
 from openml.util import is_string
 from openml.testing import TestBase
 
-from openml.datasets.functions import _get_cached_dataset, _get_cached_datasets
+from openml.datasets.functions import (_get_cached_dataset,
+                                       _get_cached_datasets,
+                                       _get_dataset_description,
+                                       _get_dataset_features,
+                                       _get_dataset_qualities)
 
 
 class TestOpenMLDataset(TestBase):
@@ -112,17 +116,17 @@ def test_download_rowid(self):
     def test__get_dataset_description(self):
         # Only a smoke test, I don't know exactly how to test the URL
         # retrieval and "caching"
-        description = openml.datasets._get_dataset_description(2)
+        description = _get_dataset_description(2)
         self.assertIsInstance(description, dict)
 
     def test__get_dataset_features(self):
         # Only a smoke check
-        features = openml.datasets._get_dataset_features(2)
+        features = _get_dataset_features(2)
         self.assertIsInstance(features, dict)
 
     def test__get_dataset_qualities(self):
         # Only a smoke check
-        qualities = openml.datasets._get_dataset_qualities(2)
+        qualities = _get_dataset_qualities(2)
         self.assertIsInstance(qualities, dict)
 
     def test_publish_dataset(self):