Merge pull request #263 from openml/develop

Dump development into master

Author: mfeurer

.travis.yml

@@ -15,9 +15,11 @@ env:
   - TEST_DIR=/tmp/test_dir/
   - MODULE=openml
   matrix:
-  - DISTRIB="conda" PYTHON_VERSION="2.7" NUMPY_VERSION="1.11" SCIPY_VERSION="0.17.0" CYTHON_VERSION="0.21" SKLEARN_VERSION="0.18"
-  - DISTRIB="conda" PYTHON_VERSION="3.4" NUMPY_VERSION="1.11" SCIPY_VERSION="0.17.0" CYTHON_VERSION="0.23.4" SKLEARN_VERSION="0.18"
-  - DISTRIB="conda" PYTHON_VERSION="3.5" COVERAGE="true" NUMPY_VERSION="1.11" SCIPY_VERSION="0.17.0" CYTHON_VERSION="0.23.4" SKLEARN_VERSION="0.18"
+  - DISTRIB="conda" PYTHON_VERSION="2.7" NUMPY_VERSION="1.11" SCIPY_VERSION="0.17.0" CYTHON_VERSION="0.21" SKLEARN_VERSION="0.18.1"
+  - DISTRIB="conda" PYTHON_VERSION="3.4" NUMPY_VERSION="1.11" SCIPY_VERSION="0.17.0" CYTHON_VERSION="0.23.4" SKLEARN_VERSION="0.18.1"
+  - DISTRIB="conda" PYTHON_VERSION="3.5" NUMPY_VERSION="1.11" SCIPY_VERSION="0.17.0" CYTHON_VERSION="0.23.4" SKLEARN_VERSION="0.18.1"
+  - DISTRIB="conda" PYTHON_VERSION="3.6" COVERAGE="true" NUMPY_VERSION="1.12.1" SCIPY_VERSION="0.19.0" CYTHON_VERSION="0.25.2" SKLEARN_VERSION="0.18.1"
+
 install: source ci_scripts/install.sh
 script: bash ci_scripts/test.sh
 after_success: source ci_scripts/success.sh

ci_scripts/install.sh

@@ -31,7 +31,7 @@ source activate testenv
 
 pip install matplotlib jupyter notebook nbconvert nbformat jupyter_client ipython ipykernel
 if [[ "$COVERAGE" == "true" ]]; then
-    pip install coverage coveralls
+    pip install codecov
 fi
 
 python --version

ci_scripts/success.sh

@@ -9,5 +9,5 @@ if [[ "$COVERAGE" == "true" ]]; then
     # very reliable but we don't want travis to report a failure
     # in the github UI just because the coverage report failed to
     # be published.
-    coveralls || echo "Coveralls upload failed"
+    codecov || echo "Codecov upload failed"
 fi

openml/__init__.py

@@ -16,16 +16,55 @@
 """
 from . import config
 
-from .datasets import OpenMLDataset
+from .datasets import OpenMLDataset, OpenMLDataFeature
 from . import datasets
+from . import tasks
 from . import runs
 from . import flows
+from . import setups
 from .runs import OpenMLRun
 from .tasks import OpenMLTask, OpenMLSplit
 from .flows import OpenMLFlow
 
+__version__ = "0.4.0dev"
 
-__version__ = "0.2.1"
 
-__all__ = ['OpenMLDataset', 'OpenMLRun', 'OpenMLSplit', 'datasets',
-           'OpenMLTask', 'OpenMLFlow', 'config', 'runs', 'flows']
+def populate_cache(task_ids=None, dataset_ids=None, flow_ids=None,
+                   run_ids=None):
+    """
+    Populate a cache for offline and parallel usage of the OpenML connector.
+
+    Parameters
+    ----------
+    task_ids : iterable
+
+    dataset_ids : iterable
+
+    flow_ids : iterable
+
+    run_ids : iterable
+
+    Returns
+    -------
+    None
+    """
+    if task_ids is not None:
+        for task_id in task_ids:
+            tasks.functions.get_task(task_id)
+
+    if dataset_ids is not None:
+        for dataset_id in dataset_ids:
+            datasets.functions.get_dataset(dataset_id)
+
+    if flow_ids is not None:
+        for flow_id in flow_ids:
+            flows.functions.get_flow(flow_id)
+
+    if run_ids is not None:
+        for run_id in run_ids:
+            runs.functions.get_run(run_id)
+
+
+__all__ = ['OpenMLDataset', 'OpenMLDataFeature', 'OpenMLRun',
+           'OpenMLSplit', 'datasets', 'OpenMLTask', 'OpenMLFlow',
+           'config', 'runs', 'flows', 'tasks', 'setups']

openml/_api_calls.py

@@ -1,11 +1,13 @@
 import io
 import os
 import requests
-import arff
 import warnings
 
+import arff
+import xmltodict
+
 from . import config
-from .exceptions import OpenMLServerError
+from .exceptions import OpenMLServerError, OpenMLServerException
 
 
 def _perform_api_call(call, data=None, file_dictionary=None,
@@ -50,6 +52,18 @@ def _read_url(self, url, add_authentication=False, data=None, filePath=None):
     return _read_url(url, data)
 
 
+def _file_id_to_url(file_id, filename=None):
+    '''
+     Presents the URL how to download a given file id
+     filename is optional
+    '''
+    openml_url = config.server.split('/api/')
+    url = openml_url[0] + '/data/download/%s' %file_id
+    if filename is not None:
+        url += '/' + filename
+    return url
+
+
 def _read_url_files(url, data=None, file_dictionary=None, file_elements=None):
     """do a post request to url with data, file content of
     file_dictionary and sending file_elements as files"""
@@ -80,7 +94,7 @@ def _read_url_files(url, data=None, file_dictionary=None, file_elements=None):
     # 'gzip,deflate'
     response = requests.post(url, data=data, files=file_elements)
     if response.status_code != 200:
-        raise OpenMLServerError(('Status code: %d\n' % response.status_code) + response.text)
+        raise _parse_server_exception(response)
     if 'Content-Encoding' not in response.headers or \
             response.headers['Content-Encoding'] != 'gzip':
         warnings.warn('Received uncompressed content from OpenML for %s.' % url)
@@ -97,8 +111,25 @@ def _read_url(url, data=None):
     response = requests.post(url, data=data)
 
     if response.status_code != 200:
-        raise OpenMLServerError(('Status code: %d\n' % response.status_code) + response.text)
+        raise _parse_server_exception(response)
     if 'Content-Encoding' not in response.headers or \
             response.headers['Content-Encoding'] != 'gzip':
         warnings.warn('Received uncompressed content from OpenML for %s.' % url)
     return response.text
+
+def _parse_server_exception(response):
+    # OpenML has a sopisticated error system
+    # where information about failures is provided. try to parse this
+    try:
+        server_exception = xmltodict.parse(response.text)
+    except:
+        raise OpenMLServerError(('Unexpected server error. Please '
+                                 'contact the developers!\nStatus code: '
+                                 '%d\n' % response.status_code) + response.text)
+
+    code = int(server_exception['oml:error']['oml:code'])
+    message = server_exception['oml:error']['oml:message']
+    additional = None
+    if 'oml:additional_information' in server_exception['oml:error']:
+        additional = server_exception['oml:error']['oml:additional_information']
+    return OpenMLServerException(code, message, additional)

openml/config.py

@@ -1,9 +1,12 @@
 """
 Stores module level information like the API key, cache directory and the server.
 """
-import os
-import sys
 import logging
+import os
+
+from six import StringIO
+from six.moves import configparser
+
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(
@@ -15,12 +18,7 @@
 cachedir = ""
 
 
-if sys.version_info[0] < 3:
-    import ConfigParser as configparser
-    from StringIO import StringIO
-else:
-    import configparser
-    from io import StringIO
+
 
 
 def _setup():
@@ -36,6 +34,7 @@ def _setup():
     """
     global apikey
     global server
+    global avoid_duplicate_runs
     # read config file, create cache directory
     try:
         os.mkdir(os.path.expanduser('~/.openml'))
@@ -46,6 +45,7 @@ def _setup():
     apikey = config.get('FAKE_SECTION', 'apikey')
     server = config.get('FAKE_SECTION', 'server')
     cache_dir = config.get('FAKE_SECTION', 'cachedir')
+    avoid_duplicate_runs = config.getboolean('FAKE_SECTION', 'avoid_duplicate_runs')
     set_cache_directory(cache_dir)
 
 
@@ -84,7 +84,8 @@ def _parse_config():
     defaults = {'apikey': apikey,
                 'server': server,
                 'verbosity': 0,
-                'cachedir': os.path.expanduser('~/.openml/cache')}
+                'cachedir': os.path.expanduser('~/.openml/cache'),
+                'avoid_duplicate_runs': 'True'}
 
     config_file = os.path.expanduser('~/.openml/config')
     config = configparser.RawConfigParser(defaults=defaults)

openml/datasets/__init__.py

@@ -1,6 +1,7 @@
 from .functions import (list_datasets, check_datasets_active,
                         get_datasets, get_dataset)
 from .dataset import OpenMLDataset
+from .data_feature import OpenMLDataFeature
 
 __all__ = ['check_datasets_active', 'get_dataset', 'get_datasets',
-           'OpenMLDataset', 'list_datasets']
+           'OpenMLDataset', 'OpenMLDataFeature', 'list_datasets']

openml/datasets/data_feature.py

@@ -0,0 +1,36 @@
+
+class OpenMLDataFeature(object):
+    """Data Feature (a.k.a. Attribute) object.
+
+       Parameters
+       ----------
+       index : int
+            The index of this feature
+        name : str
+            Name of the feature
+        data_type : str
+            can be nominal, numeric, string, date (corresponds to arff)
+        nominal_values : list(str)
+            list of the possible values, in case of nominal attribute
+        number_missing_values : int
+       """
+    LEGAL_DATA_TYPES = ['nominal', 'numeric', 'string', 'date']
+
+    def __init__(self, index, name, data_type, nominal_values, number_missing_values):
+        if type(index) != int:
+            raise ValueError('Index is of wrong datatype')
+        if data_type not in self.LEGAL_DATA_TYPES:
+            raise ValueError('data type should be in %s, found: %s' %(str(self.LEGAL_DATA_TYPES),data_type))
+        if nominal_values is not None and type(nominal_values) != list:
+            raise ValueError('Nominal_values is of wrong datatype')
+        if type(number_missing_values) != int:
+            raise ValueError('number_missing_values is of wrong datatype')
+
+        self.index = index
+        self.name = str(name)
+        self.data_type = str(data_type)
+        self.nominal_values = nominal_values
+        self.number_missing_values = number_missing_values
+
+    def __str__(self):
+        return "[%d - %s (%s)]" %(self.index, self.name, self.data_type)