Merge remote-tracking branch 'origin/develop' into add_#737

# Conflicts:
#	doc/progress.rst
#	tests/test_flows/test_flow_functions.py

Author: sahithyaravi

.travis.yml

@@ -15,10 +15,11 @@ env:
   - TEST_DIR=/tmp/test_dir/
   - MODULE=openml
   matrix:
-  - DISTRIB="conda" PYTHON_VERSION="3.5" SKLEARN_VERSION="0.20.0"
-  - DISTRIB="conda" PYTHON_VERSION="3.6" SKLEARN_VERSION="0.20.0"
-  - DISTRIB="conda" PYTHON_VERSION="3.7" SKLEARN_VERSION="0.20.0" RUN_FLAKE8="true" SKIP_TESTS="true"
-  - DISTRIB="conda" PYTHON_VERSION="3.7" SKLEARN_VERSION="0.20.0" COVERAGE="true" DOCPUSH="true"
+  - DISTRIB="conda" PYTHON_VERSION="3.5" SKLEARN_VERSION="0.21.2"
+  - DISTRIB="conda" PYTHON_VERSION="3.6" SKLEARN_VERSION="0.21.2"
+  - DISTRIB="conda" PYTHON_VERSION="3.7" SKLEARN_VERSION="0.21.2" RUN_FLAKE8="true" SKIP_TESTS="true"
+  - DISTRIB="conda" PYTHON_VERSION="3.7" SKLEARN_VERSION="0.21.2" COVERAGE="true" DOCPUSH="true"
+  - DISTRIB="conda" PYTHON_VERSION="3.7" SKLEARN_VERSION="0.20.2"
   # Checks for older scikit-learn versions (which also don't nicely work with
   # Python3.7)
   - DISTRIB="conda" PYTHON_VERSION="3.6" SKLEARN_VERSION="0.19.2"

CONTRIBUTING.md

@@ -81,6 +81,10 @@ following rules before you submit a pull request:
    Drafts often benefit from the inclusion of a
    [task list](https://github.com/blog/1375-task-lists-in-gfm-issues-pulls-comments)
    in the PR description.
+   
+- Add [unit tests](https://github.com/openml/openml-python/tree/develop/tests) and [examples](https://github.com/openml/openml-python/tree/develop/examples) for any new functionality being introduced. 
+    - If an unit test contains an upload to the test server, please ensure that it is followed by a file collection for deletion, to prevent the test server from bulking up. For example, `TestBase._mark_entity_for_removal('data', dataset.dataset_id)`, `TestBase._mark_entity_for_removal('flow', (flow.flow_id, flow.name))`.
+    - Please ensure that the example is run on the test server by beginning with the call to `openml.config.start_using_configuration_for_example()`.      
 
 -  All tests pass when running `pytest`. On
    Unix-like systems, check with (from the toplevel source folder):

LICENSE

@@ -1,6 +1,6 @@
 BSD 3-Clause License
 
-Copyright (c) 2014-2018, Matthias Feurer, Jan van Rijn, Andreas Müller, 
+Copyright (c) 2014-2019, Matthias Feurer, Jan van Rijn, Andreas Müller, 
 Joaquin Vanschoren and others.
 All rights reserved.
 

PULL_REQUEST_TEMPLATE.md

@@ -9,6 +9,8 @@ Please make sure that:
 * for any new function or class added, please add it to doc/api.rst
     * the list of classes and functions should be alphabetical 
 * for any new functionality, consider adding a relevant example
+* add unit tests for new functionalities
+    * collect files uploaded to test server using _mark_entity_for_removal()
 -->
 
 #### Reference Issue

doc/conf.py

@@ -15,6 +15,7 @@
 import os
 import sys
 import sphinx_bootstrap_theme
+import time
 import openml
 
 # If extensions (or modules to document with autodoc) are in another directory,
@@ -65,7 +66,7 @@
 # General information about the project.
 project = u'OpenML'
 copyright = (
-    u'2014-2019, the OpenML-Python team.'
+    u'2014-{}, the OpenML-Python team.'.format(time.strftime("%Y,%m,%d,%H,%M,%S").split(',')[0])
 )
 
 # The version info for the project you're documenting, acts as replacement for

doc/index.rst

@@ -21,16 +21,12 @@ Example
 .. code:: python
 
     import openml
-    from sklearn import preprocessing, tree, pipeline
-
-    # Set the OpenML API Key which is required to upload your runs.
-    # You can get your own API by signing up to OpenML.org.
-    openml.config.apikey = 'ABC'
+    from sklearn import impute, tree, pipeline
 
     # Define a scikit-learn classifier or pipeline
     clf = pipeline.Pipeline(
         steps=[
-            ('imputer', preprocessing.Imputer()),
+            ('imputer', impute.SimpleImputer()),
             ('estimator', tree.DecisionTreeClassifier())
         ]
     )
@@ -39,10 +35,13 @@ Example
     task = openml.tasks.get_task(31)
     # Run the scikit-learn model on the task.
     run = openml.runs.run_model_on_task(clf, task)
-    # Publish the experiment on OpenML (optional, requires an API key).
+    # Publish the experiment on OpenML (optional, requires an API key.
+    # You can get your own API key by signing up to OpenML.org)
     run.publish()
     print('View the run online: %s/run/%d' % (openml.config.server, run.run_id))
 
+You can find more examples in our `examples gallery <examples/index.html>`_.
+
 ----------------------------
 How to get OpenML for python
 ----------------------------

doc/progress.rst

@@ -9,14 +9,18 @@ Changelog
 0.10.0
 ~~~~~~
 * ADD #737: Add list_evaluations_setups to return hyperparameters along with list of evaluations.
-* ADD #722: Automatic reinstantiation of flow in `run_model_on_task`. Clearer errors if that's not possible.
+* FIX #261: Test server is cleared of all files uploaded during unit testing.
+* FIX #447: All files created by unit tests no longer persist in local.
 * FIX #608: Fixing dataset_id referenced before assignment error in get_run function.
-* ADD #715: `list_evaluations` now has an option to sort evaluations by score (value).
+* FIX #447: All files created by unit tests are deleted after the completion of all unit tests.
 * FIX #589: Fixing a bug that did not successfully upload the columns to ignore when creating and publishing a dataset.
+* FIX #608: Fixing dataset_id referenced before assignment error in get_run function.
 * DOC #639: More descriptive documention for function to convert array format.
 * ADD #687: Adds a function to retrieve the list of evaluation measures available.
 * ADD #695: A function to retrieve all the data quality measures available.
-* FIX #447: All files created by unit tests are deleted after the completion of all unit tests.
+* ADD #412: Add a function to trim flow names for scikit-learn flows.
+* ADD #715: `list_evaluations` now has an option to sort evaluations by score (value).
+* ADD #722: Automatic reinstantiation of flow in `run_model_on_task`. Clearer errors if that's not possible.
 * MAINT #726: Update examples to remove deprecation warnings from scikit-learn
 
 0.9.0

examples/fetch_evaluations_tutorial.py

@@ -20,7 +20,6 @@
 
 ############################################################################
 import openml
-from pprint import pprint
 
 ############################################################################
 # Listing evaluations
@@ -37,7 +36,7 @@
                                             output_format='dataframe')
 
 # Querying the returned results for precision above 0.98
-pprint(evals[evals.value > 0.98])
+print(evals[evals.value > 0.98])
 
 #############################################################################
 # Viewing a sample task
@@ -47,7 +46,7 @@
 # We will start by displaying a simple *supervised classification* task:
 task_id = 167140        # https://www.openml.org/t/167140
 task = openml.tasks.get_task(task_id)
-pprint(vars(task))
+print(task)
 
 #############################################################################
 # Obtaining all the evaluations for the task
@@ -60,11 +59,11 @@
 evals = openml.evaluations.list_evaluations(function=metric, task=[task_id],
                                             output_format='dataframe')
 # Displaying the first 10 rows
-pprint(evals.head(n=10))
+print(evals.head(n=10))
 # Sorting the evaluations in decreasing order of the metric chosen
 evals = evals.sort_values(by='value', ascending=False)
 print("\nDisplaying head of sorted dataframe: ")
-pprint(evals.head())
+print(evals.head())
 
 #############################################################################
 # Obtaining CDF of metric for chosen task
@@ -147,4 +146,4 @@ def plot_flow_compare(evaluations, top_n=10, metric='predictive_accuracy'):
 flow_ids = evals.flow_id.unique()[:top_n]
 flow_names = evals.flow_name.unique()[:top_n]
 for i in range(top_n):
-    pprint((flow_ids[i], flow_names[i]))
+    print((flow_ids[i], flow_names[i]))

examples/flows_and_runs_tutorial.py

@@ -6,7 +6,6 @@
 """
 
 import openml
-from pprint import pprint
 from sklearn import compose, ensemble, impute, neighbors, preprocessing, pipeline, tree
 
 ############################################################################
@@ -58,7 +57,7 @@
 # Run the flow
 run = openml.runs.run_model_on_task(clf, task)
 
-# pprint(vars(run), depth=2)
+print(run)
 
 ############################################################################
 # Share the run on the OpenML server
@@ -75,17 +74,37 @@
 # We can now also inspect the flow object which was automatically created:
 
 flow = openml.flows.get_flow(run.flow_id)
-pprint(vars(flow), depth=1)
+print(flow)
 
 ############################################################################
 # It also works with pipelines
 # ############################
 #
 # When you need to handle 'dirty' data, build pipelines to model then automatically.
-task = openml.tasks.get_task(115)
+task = openml.tasks.get_task(1)
+features = task.get_dataset().features
+nominal_feature_indices = [
+    i for i in range(len(features))
+    if features[i].name != task.target_name and features[i].data_type == 'nominal'
+]
 pipe = pipeline.Pipeline(steps=[
-    ('Imputer', impute.SimpleImputer(strategy='median')),
-    ('OneHotEncoder', preprocessing.OneHotEncoder(sparse=False, handle_unknown='ignore')),
+    (
+        'Preprocessing',
+        compose.ColumnTransformer([
+            ('Nominal', pipeline.Pipeline(
+                [
+                    ('Imputer', impute.SimpleImputer(strategy='most_frequent')),
+                    (
+                        'Encoder',
+                        preprocessing.OneHotEncoder(
+                            sparse=False, handle_unknown='ignore',
+                        )
+                    ),
+                ]),
+                nominal_feature_indices,
+             ),
+        ]),
+    ),
     ('Classifier', ensemble.RandomForestClassifier(n_estimators=10))
 ])
 

examples/introduction_tutorial.py

@@ -1,6 +1,6 @@
 """
 Introduction
-===================
+============
 
 An introduction to OpenML, followed up by a simple example.
 """
@@ -15,6 +15,8 @@
 # * Works seamlessly with scikit-learn and other libraries
 # * Large scale benchmarking, compare to state of the art
 #
+
+############################################################################
 # Installation
 # ^^^^^^^^^^^^
 # Installation is done via ``pip``:
@@ -26,6 +28,8 @@
 # For further information, please check out the installation guide at
 # https://openml.github.io/openml-python/master/contributing.html#installation
 #
+
+############################################################################
 # Authentication
 # ^^^^^^^^^^^^^^
 #
@@ -49,6 +53,7 @@
 # .. warning:: This example uploads data. For that reason, this example
 #   connects to the test server instead. This prevents the live server from
 #   crowding with example datasets, tasks, studies, and so on.
+
 ############################################################################
 import openml
 from sklearn import neighbors