Merge pull request #202 from openml/develop

Merge develop into master (on popular demand @ hackathon)

Author: janvanrijn

.travis.yml

@@ -15,9 +15,9 @@ env:
   - TEST_DIR=/tmp/test_dir/
   - MODULE=openml
   matrix:
-  - DISTRIB="conda" PYTHON_VERSION="2.7" NUMPY_VERSION="1.10.4" SCIPY_VERSION="0.17.0" CYTHON_VERSION="0.21"
-  - DISTRIB="conda" PYTHON_VERSION="3.4" NUMPY_VERSION="1.10.4" SCIPY_VERSION="0.17.0" CYTHON_VERSION="0.23.4"
-  - DISTRIB="conda" PYTHON_VERSION="3.5" COVERAGE="true" NUMPY_VERSION="1.10.4" SCIPY_VERSION="0.17.0" CYTHON_VERSION="0.23.4"
+  - DISTRIB="conda" PYTHON_VERSION="2.7" NUMPY_VERSION="1.11" SCIPY_VERSION="0.17.0" CYTHON_VERSION="0.21" SKLEARN_VERSION="0.18"
+  - DISTRIB="conda" PYTHON_VERSION="3.4" NUMPY_VERSION="1.11" SCIPY_VERSION="0.17.0" CYTHON_VERSION="0.23.4" SKLEARN_VERSION="0.18"
+  - DISTRIB="conda" PYTHON_VERSION="3.5" COVERAGE="true" NUMPY_VERSION="1.11" SCIPY_VERSION="0.17.0" CYTHON_VERSION="0.23.4" SKLEARN_VERSION="0.18"
 install: source ci_scripts/install.sh
 script: bash ci_scripts/test.sh
 after_success: source ci_scripts/success.sh

README.md

@@ -1,9 +1,16 @@
-[![Build Status](https://travis-ci.org/openml/openml-python.svg)](https://travis-ci.org/openml/openml-python)
-[![Code Health](https://landscape.io/github/openml/openml-python/master/landscape.svg)](https://landscape.io/github/openml/openml-python/master)
-[![Stories in Ready](https://badge.waffle.io/openml/openml-python.png?label=ready&title=Ready)](https://waffle.io/openml/openml-python)
-[![Coverage Status](https://coveralls.io/repos/github/openml/openml-python/badge.svg?branch=develop)](https://coveralls.io/github/openml/openml-python?branch=develop)
-
 A python interface for [OpenML](http://openml.org). You can find the documentation on the [openml-python website](https://openml.github.io/openml-python).
 
 Please commit to the right branches following the gitflow pattern:
 http://nvie.com/posts/a-successful-git-branching-model/
+
+Master branch:
+
+[![Build Status](https://travis-ci.org/openml/openml-python.svg?branch=master)](https://travis-ci.org/openml/openml-python)
+[![Code Health](https://landscape.io/github/openml/openml-python/master/landscape.svg)](https://landscape.io/github/openml/openml-python/master)
+[![Coverage Status](https://coveralls.io/repos/github/openml/openml-python/badge.svg?branch=master)](https://coveralls.io/github/openml/openml-python?branch=master)
+
+Development branch:
+
+[![Build Status](https://travis-ci.org/openml/openml-python.svg?branch=develop)](https://travis-ci.org/openml/openml-python)
+[![Code Health](https://landscape.io/github/openml/openml-python/master/landscape.svg)](https://landscape.io/github/openml/openml-python/master)
+[![Coverage Status](https://coveralls.io/repos/github/openml/openml-python/badge.svg?branch=develop)](https://coveralls.io/github/openml/openml-python?branch=develop)

ci_scripts/install.sh

@@ -13,10 +13,10 @@ ls -l
 echo
 if [[ ! -f miniconda.sh ]]
    then
-   wget http://repo.continuum.io/miniconda/Miniconda-3.6.0-Linux-x86_64.sh \
+   wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \
        -O miniconda.sh
    fi
-chmod +x miniconda.sh && ./miniconda.sh -b
+chmod +x miniconda.sh && ./miniconda.sh -b -p $HOME/miniconda
 cd ..
 export PATH=/home/travis/miniconda/bin:$PATH
 conda update --yes conda
@@ -25,11 +25,11 @@ popd
 # Configure the conda environment and put it in the path using the
 # provided versions
 conda create -n testenv --yes python=$PYTHON_VERSION pip nose \
-   numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION cython=$CYTHON_VERSION matplotlib scikit-learn nbconvert nbformat jupyter_client ipython jupyter notebook ipykernel pandas
+   numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION cython=$CYTHON_VERSION  \
+   scikit-learn=$SKLEARN_VERSION pandas
 source activate testenv
-ipython kernel install
-
 
+pip install matplotlib jupyter notebook nbconvert nbformat jupyter_client ipython ipykernel
 if [[ "$COVERAGE" == "true" ]]; then
     pip install coverage coveralls
 fi

doc/progress.rst

@@ -6,6 +6,16 @@
 Progress
 ========
 
+Changelog
+=========
+
+0.3.0
+~~~~~
+
+* Add this changelog (Matthias Feurer)
+* 2nd example notebook PyOpenML.ipynb (Joaquin Vanschoren)
+* Pagination support for list datasets and list tasks
+
 API calls
 =========
 

examples/OpenMLDemo.ipynb

@@ -136,7 +136,7 @@
    "source": [
     "datasets = openml.datasets.list_datasets()\n",
     "\n",
-    "data = pd.DataFrame(datasets)\n",
+    "data = pd.DataFrame(datasets).transpose()\n",
     "print(\"First 10 of %s datasets...\" % len(datasets))\n",
     "print(data[:10][['did','name','NumberOfInstances','NumberOfFeatures']])"
    ]
@@ -569,7 +569,7 @@
    "source": [
     "task_list = openml.tasks.list_tasks()\n",
     "\n",
-    "tasks = pd.DataFrame(task_list)\n",
+    "tasks = pd.DataFrame(task_list).transpose()\n",
     "print(\"First 5 of %s tasks:\" % len(tasks))\n",
     "print(tasks[:5][['tid','did','name','task_type','estimation_procedure']])"
    ]
@@ -688,14 +688,14 @@
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 3
+    "version": 3.0
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.5.1"
+   "version": "3.4.3"
   }
  },
  "nbformat": 4,

openml/__init__.py

@@ -1,7 +1,7 @@
 """
 The OpenML module implements a python interface to
-`OpenML <http://openml.org>`_, a collaborative platform for machine learning.
-OpenML can be used to
+`OpenML <https://www.openml.org>`_, a collaborative platform for machine
+learning. OpenML can be used to
 
 * store, download and analyze datasets
 * make experiments and their results (e.g. models, predictions)
@@ -10,7 +10,7 @@
   meta studies
 
 In particular, this module implemts a python interface for the
-`OpenML REST API <http://openml.org/guide#!rest_services>`_
+`OpenML REST API <https://www.openml.org/guide#!rest_services>`_
 (`REST on wikipedia
 <http://en.wikipedia.org/wiki/Representational_state_transfer>`_).
 """
@@ -19,12 +19,13 @@
 from .datasets import OpenMLDataset
 from . import datasets
 from . import runs
+from . import flows
 from .runs import OpenMLRun
 from .tasks import OpenMLTask, OpenMLSplit
 from .flows import OpenMLFlow
 
 
 __version__ = "0.2.1"
 
-__all__ = ['OpenMLDataset', 'OpenMLRun', 'OpenMLSplit',
-           'datasets', 'OpenMLTask', 'OpenMLFlow', 'config', 'runs']
+__all__ = ['OpenMLDataset', 'OpenMLRun', 'OpenMLSplit', 'datasets',
+           'OpenMLTask', 'OpenMLFlow', 'config', 'runs', 'flows']

openml/_api_calls.py

@@ -8,7 +8,7 @@
 from .exceptions import OpenMLServerError
 
 
-def _perform_api_call(call, file_dictionary=None,
+def _perform_api_call(call, data=None, file_dictionary=None,
                       file_elements=None, add_authentication=True):
     """
     Perform an API call at the OpenML server.
@@ -19,6 +19,8 @@ def _read_url(self, url, add_authentication=False, data=None, filePath=None):
     ----------
     call : str
         The API call. For example data/list
+    data : dict
+        Dictionary with post-request payload.
     file_dictionary : dict
         Mapping of {filename: path} of files which should be uploaded to the
         server.
@@ -39,17 +41,20 @@ def _read_url(self, url, add_authentication=False, data=None, filePath=None):
     if not url.endswith("/"):
         url += "/"
     url += call
+
+    url = url.replace('=', '%3d')
+
     if file_dictionary is not None or file_elements is not None:
-        return _read_url_files(url, file_dictionary=file_dictionary,
+        return _read_url_files(url, data=data, file_dictionary=file_dictionary,
                                file_elements=file_elements)
-    return _read_url(url)
+    return _read_url(url, data)
 
 
-def _read_url_files(url, file_dictionary=None, file_elements=None):
-    """do a post request to url with data None, file content of
+def _read_url_files(url, data=None, file_dictionary=None, file_elements=None):
+    """do a post request to url with data, file content of
     file_dictionary and sending file_elements as files"""
 
-    data = {}
+    data = {} if data is None else data
     data['api_key'] = config.apikey
     if file_elements is None:
         file_elements = {}
@@ -75,24 +80,25 @@ def _read_url_files(url, file_dictionary=None, file_elements=None):
     # 'gzip,deflate'
     response = requests.post(url, data=data, files=file_elements)
     if response.status_code != 200:
-        raise OpenMLServerError(response.text)
+        raise OpenMLServerError(('Status code: %d\n' % response.status_code) + response.text)
     if 'Content-Encoding' not in response.headers or \
             response.headers['Content-Encoding'] != 'gzip':
         warnings.warn('Received uncompressed content from OpenML for %s.' % url)
-    return response.status_code, response.text
+    return response.text
 
 
-def _read_url(url):
+def _read_url(url, data=None):
 
-    data = {}
+    data = {} if data is None else data
     data['api_key'] = config.apikey
 
     # Using requests.post sets header 'Accept-encoding' automatically to
     # 'gzip,deflate'
     response = requests.post(url, data=data)
+
     if response.status_code != 200:
-        raise OpenMLServerError(response.text)
+        raise OpenMLServerError(('Status code: %d\n' % response.status_code) + response.text)
     if 'Content-Encoding' not in response.headers or \
             response.headers['Content-Encoding'] != 'gzip':
         warnings.warn('Received uncompressed content from OpenML for %s.' % url)
-    return response.status_code, response.text
+    return response.text

openml/config.py

@@ -1,6 +1,5 @@
 """
-Stores module level information like the API key, cache director, private
-directory and the server.
+Stores module level information like the API key, cache directory and the server.
 """
 import os
 import sys
@@ -11,10 +10,9 @@
     format='[%(levelname)s] [%(asctime)s:%(name)s] %('
            'message)s', datefmt='%H:%M:%S')
 
-server = "http://www.openml.org/api/v1/xml"
+server = "https://www.openml.org/api/v1/xml"
 apikey = ""
 cachedir = ""
-privatedir = ""
 
 
 if sys.version_info[0] < 3:
@@ -47,57 +45,35 @@ def _setup():
     config = _parse_config()
     apikey = config.get('FAKE_SECTION', 'apikey')
     server = config.get('FAKE_SECTION', 'server')
-    private_dir = config.get('FAKE_SECTION', 'private_directory')
     cache_dir = config.get('FAKE_SECTION', 'cachedir')
-    set_cache_directory(cache_dir, private_dir)
+    set_cache_directory(cache_dir)
 
 
-def set_cache_directory(cachedir, privatedir=None):
+def set_cache_directory(cachedir):
     """Set module-wide cache directory.
 
     Sets the cache directory into which to download datasets, tasks etc.
-    Also sets the private directory for storing local datasets.
 
     Parameters
     ----------
     cachedir : string
         Path to use as cache directory.
 
-    privatedir : string
-        Path containing private datasets, tasks, etc.
-
     See also
     --------
     get_cache_directory
-    get_private_directory
     """
-    if privatedir is None:
-        privatedir = cachedir
 
     global _cachedir
-    global _privatedir
     _cachedir = cachedir
-    _privatedir = privatedir
 
     # Set up the cache directories
     dataset_cache_dir = os.path.join(cachedir, "datasets")
     task_cache_dir = os.path.join(cachedir, "tasks")
     run_cache_dir = os.path.join(cachedir, 'runs')
 
-    # Set up the private directory
-    _private_directory_datasets = os.path.join(
-        privatedir, "datasets")
-    _private_directory_tasks = os.path.join(
-        privatedir, "tasks")
-    _private_directory_runs = os.path.join(
-        privatedir, "runs")
-
-    for dir_ in [cachedir, dataset_cache_dir,
-                 task_cache_dir, run_cache_dir,
-                 privatedir,
-                 _private_directory_datasets,
-                 _private_directory_tasks,
-                 _private_directory_runs]:
+
+    for dir_ in [cachedir, dataset_cache_dir, task_cache_dir, run_cache_dir]:
         if not os.path.exists(dir_) and not os.path.isdir(dir_):
             os.mkdir(dir_)
 
@@ -108,8 +84,7 @@ def _parse_config():
     defaults = {'apikey': apikey,
                 'server': server,
                 'verbosity': 0,
-                'cachedir': os.path.expanduser('~/.openml/cache'),
-                'private_directory': os.path.expanduser('~/.openml/private')}
+                'cachedir': os.path.expanduser('~/.openml/cache')}
 
     config_file = os.path.expanduser('~/.openml/config')
     config = configparser.RawConfigParser(defaults=defaults)
@@ -147,26 +122,10 @@ def get_cache_directory():
     See also
     --------
     set_cache_directory
-    get_private_directory
     """
     return _cachedir
 
 
-def get_private_directory():
-    """Get the current private directory.
-
-    Returns
-    -------
-    privatecir : string
-        The current private directory.
-
-    See also
-    --------
-    set_cache_directory
-    get_cache_directory
-    """
-    return _privatedir
-
-__all__ = ["set_cache_directory", 'get_cache_directory', 'get_private_directory']
+__all__ = ["set_cache_directory", 'get_cache_directory']
 
 _setup()

openml/datasets/__init__.py

@@ -1,7 +1,6 @@
-from .functions import (list_datasets, list_datasets_by_tag,
-                        check_datasets_active, get_datasets, get_dataset)
+from .functions import (list_datasets, check_datasets_active,
+                        get_datasets, get_dataset)
 from .dataset import OpenMLDataset
 
 __all__ = ['check_datasets_active', 'get_dataset', 'get_datasets',
-           'OpenMLDataset', 'list_datasets', 'list_datasets_by_tag',
-           'list_datasets']
+           'OpenMLDataset', 'list_datasets']