Extension interface (#647)

* draft extensions interface

* Change to new advised style of defining abstract base class.

* incorporate @pgijbers' feedback

* incorporate Jan's comments

* (hopefully) make the tests run again

* make more tests work again

* fix more tests?

* Move all files for the sklearn converter to a single location

* fix tests

* TST fix function call

* slight reorganization of the files

* TST fix wrong path

* TST fix wrong path

* MAINT add type hints to all methods touched in this PR

* factor a lot of extension functions to new file

* fix a few broken tests

* rename test files to reflect previous refactor

* fix unit tests

* fix unit tests

* add extension plugin mechanism

* pep8 & mypy

* save docstring progress

* fix?

* finish docstrings & simplify interface

* add extension interface to documentation

* PEP8 & doc building

* Address comments by Jan and Pieter

* progress dump

* tests, pep8, shuffle functions and tests around

Author: mfeurer

ci_scripts/flake8_diff.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
 
 flake8 --ignore E402,W503 --show-source --max-line-length 100 $options
+mypy openml --ignore-missing-imports --follow-imports skip

ci_scripts/install.sh

@@ -40,7 +40,7 @@ if [[ "$COVERAGE" == "true" ]]; then
     pip install codecov pytest-cov
 fi
 if [[ "$RUN_FLAKE8" == "true" ]]; then
-    pip install flake8
+    pip install flake8 mypy
 fi
 
 python --version

doc/api.rst

@@ -20,6 +20,32 @@ Top-level Classes
    OpenMLFlow
    OpenMLEvaluation
 
+.. _api_extensions:
+
+Extensions
+----------
+
+.. currentmodule:: openml.extensions
+
+.. autosummary::
+   :toctree: generated/
+   :template: class.rst
+
+   Extension
+   sklearn.SklearnExtension
+
+.. currentmodule:: openml.extensions
+
+.. autosummary::
+   :toctree: generated/
+   :template: function.rst
+
+    register_extension
+    get_extension_by_model
+    get_extension_by_flow
+
+Modules
+-------
 
 :mod:`openml.datasets`: Dataset Functions
 -----------------------------------------
@@ -55,10 +81,8 @@ Top-level Classes
    :template: function.rst
 
     flow_exists
-    flow_to_sklearn
     get_flow
     list_flows
-    sklearn_to_flow
 
 :mod:`openml.runs`: Run Functions
 ----------------------------------
@@ -112,5 +136,3 @@ Top-level Classes
     get_tasks
     list_tasks
 
-
- 

doc/contributing.rst

@@ -106,17 +106,13 @@ From within the directory of the cloned package, execute:
 
     pytest tests/
 
-.. _extending:
-
-Executing a specific test can be done by specifying the module, test case, and test.  
+Executing a specific test can be done by specifying the module, test case, and test.
 To obtain a hierarchical list of all tests, run
 
 .. code:: bash
 
     pytest --collect-only
 
-.. _extending:
-
 .. code:: bash
 
     <Module 'tests/test_datasets/test_dataset.py'>
@@ -129,33 +125,26 @@ To obtain a hierarchical list of all tests, run
         <TestCaseFunction 'test_get_data_with_target'>
       <UnitTestCase 'OpenMLDatasetTestOnTestServer'>
         <TestCaseFunction 'test_tagging'>
-        
-.. _extending:
+
 
 To run a specific module, add the module name, for instance:
 
 .. code:: bash
 
     pytest tests/test_datasets/test_dataset.py
 
-.. _extending:
-
 To run a specific unit test case, add the test case name, for instance:
 
 .. code:: bash
 
     pytest tests/test_datasets/test_dataset.py::OpenMLDatasetTest
 
-.. _extending:
-
 To run a specific unit test, add the test name, for instance:
 
 .. code:: bash
 
     pytest tests/test_datasets/test_dataset.py::OpenMLDatasetTest::test_get_data
 
-.. _extending:
-
 Happy testing!
 
 

doc/usage.rst

@@ -116,6 +116,13 @@ obtained on. Learn how to share your datasets in the following tutorial:
 
 * `Upload a dataset <examples/create_upload_tutorial.html>`_
 
+~~~~~~~~~~~~~~~~~~~~~~~
+Extending OpenML-Python
+~~~~~~~~~~~~~~~~~~~~~~~
+
+OpenML-Python provides an extension interface to connect other machine learning libraries than
+scikit-learn to OpenML. Please check the :ref:`api_extensions` and use the
+scikit-learn extension in :class:`openml.extensions.sklearn.SklearnExtension` as a starting point.
 
 ~~~~~~~~~~~~~~~
 Advanced topics

examples/flows_and_runs_tutorial.py

@@ -49,11 +49,8 @@
 # Build any classifier or pipeline
 clf = tree.ExtraTreeClassifier()
 
-# Create a flow
-flow = openml.flows.sklearn_to_flow(clf)
-
 # Run the flow
-run = openml.runs.run_flow_on_task(flow, task)
+run = openml.runs.run_model_on_task(clf, task)
 
 # pprint(vars(run), depth=2)
 
@@ -85,9 +82,8 @@
     ('OneHotEncoder', preprocessing.OneHotEncoder(sparse=False, handle_unknown='ignore')),
     ('Classifier', ensemble.RandomForestClassifier())
 ])
-flow = openml.flows.sklearn_to_flow(pipe)
 
-run = openml.runs.run_flow_on_task(flow, task, avoid_duplicate_runs=False)
+run = openml.runs.run_model_on_task(pipe, task, avoid_duplicate_runs=False)
 myrun = run.publish()
 print("Uploaded to http://test.openml.org/r/" + str(myrun.run_id))
 
@@ -118,6 +114,22 @@
 # Publishing the run will automatically upload the related flow if
 # it does not yet exist on the server.
 
+############################################################################
+# Alternatively, one can also directly run flows.
+
+# Get a task
+task = openml.tasks.get_task(403)
+
+# Build any classifier or pipeline
+clf = tree.ExtraTreeClassifier()
+
+# Obtain the scikit-learn extension interface to convert the classifier
+# into a flow object.
+extension = openml.extensions.get_extension_by_model(clf)
+flow = extension.model_to_flow(clf)
+
+run = openml.runs.run_flow_on_task(flow, task)
+
 ############################################################################
 # Challenge
 # ^^^^^^^^^
@@ -142,8 +154,7 @@
     task = openml.tasks.get_task(task_id)
     data = openml.datasets.get_dataset(task.dataset_id)
     clf = neighbors.KNeighborsClassifier(n_neighbors=5)
-    flow = openml.flows.sklearn_to_flow(clf)
 
-    run = openml.runs.run_flow_on_task(flow, task, avoid_duplicate_runs=False)
+    run = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False)
     myrun = run.publish()
     print("kNN on %s: http://test.openml.org/r/%d" % (data.name, myrun.run_id))

examples/introduction_tutorial.py

@@ -77,8 +77,7 @@
 task = openml.tasks.get_task(403)
 data = openml.datasets.get_dataset(task.dataset_id)
 clf = neighbors.KNeighborsClassifier(n_neighbors=5)
-flow = openml.flows.sklearn_to_flow(clf)
-run = openml.runs.run_flow_on_task(flow, task, avoid_duplicate_runs=False)
+run = openml.runs.run_model_on_task(clf, task, avoid_duplicate_runs=False)
 # Publish the experiment on OpenML (optional, requires an API key).
 # For this tutorial, our configuration publishes to the test server
 # as to not pollute the main server.

openml/__init__.py

@@ -14,23 +14,36 @@
 (`REST on wikipedia
 <http://en.wikipedia.org/wiki/Representational_state_transfer>`_).
 """
-from . import config
 
+from . import _api_calls
+from . import config
 from .datasets import OpenMLDataset, OpenMLDataFeature
 from . import datasets
+from . import evaluations
+from .evaluations import OpenMLEvaluation
+from . import extensions
+from . import exceptions
 from . import tasks
+from .tasks import (
+    OpenMLTask,
+    OpenMLSplit,
+    OpenMLSupervisedTask,
+    OpenMLClassificationTask,
+    OpenMLRegressionTask,
+    OpenMLClusteringTask,
+    OpenMLLearningCurveTask,
+)
 from . import runs
-from . import flows
-from . import setups
-from . import evaluations
-
 from .runs import OpenMLRun
-from .tasks import OpenMLTask, OpenMLSplit
+from . import flows
 from .flows import OpenMLFlow
-from .evaluations import OpenMLEvaluation
+from . import setups
+from . import study
 from .study import OpenMLStudy
+from . import utils
+
 
-from .__version__ import __version__  # noqa: F401
+from .__version__ import __version__
 
 
 def populate_cache(task_ids=None, dataset_ids=None, flow_ids=None,
@@ -69,7 +82,35 @@ def populate_cache(task_ids=None, dataset_ids=None, flow_ids=None,
             runs.functions.get_run(run_id)
 
 
-__all__ = ['OpenMLDataset', 'OpenMLDataFeature', 'OpenMLRun',
-           'OpenMLSplit', 'OpenMLEvaluation', 'OpenMLSetup',
-           'OpenMLTask', 'OpenMLFlow', 'OpenMLStudy', 'datasets',
-           'evaluations', 'config', 'runs', 'flows', 'tasks', 'setups']
+__all__ = [
+    'OpenMLDataset',
+    'OpenMLDataFeature',
+    'OpenMLRun',
+    'OpenMLSplit',
+    'OpenMLEvaluation',
+    'OpenMLSetup',
+    'OpenMLTask',
+    'OpenMLSupervisedTask',
+    'OpenMLClusteringTask',
+    'OpenMLLearningCurveTask',
+    'OpenMLRegressionTask',
+    'OpenMLClassificationTask',
+    'OpenMLFlow',
+    'OpenMLStudy',
+    'datasets',
+    'evaluations',
+    'exceptions',
+    'extensions',
+    'config',
+    'runs',
+    'flows',
+    'tasks',
+    'setups',
+    'study',
+    'utils',
+    '_api_calls',
+    '__version__',
+]
+
+# Load the scikit-learn extension by default
+import openml.extensions.sklearn  # noqa: F401

openml/config.py

@@ -28,13 +28,14 @@
 
 # Default values are actually added here in the _setup() function which is
 # called at the end of this module
-server = ""
-apikey = ""
+server = _defaults['server']
+apikey = _defaults['apikey']
 # The current cache directory (without the server name)
-cache_directory = ""
+cache_directory = _defaults['cachedir']
+avoid_duplicate_runs = True if _defaults['avoid_duplicate_runs'] == 'True' else False
 
 # Number of retries if the connection breaks
-connection_n_retries = 2
+connection_n_retries = _defaults['connection_n_retries']
 
 
 def _setup():

openml/datasets/functions.py

@@ -511,8 +511,9 @@ def create_dataset(name, description, creator, contributor,
         specified, the index of the dataframe will be used as the
         ``row_id_attribute``. If the name of the index is ``None``, it will
         be discarded.
+
         .. versionadded: 0.8
-           Inference of ``row_id_attribute`` from a dataframe.
+            Inference of ``row_id_attribute`` from a dataframe.
     original_data_url : str, optional
         For derived data, the url to the original dataset.
     paper_url : str, optional