[MRG] EHN: Add support for pandas DataFrame and SparseDataFrame when loading (#548)

* EHN: add support for DataFrame when loading dataset

* MAINT: add pandas as dependency

* FIX: typo in setup

* TST: add unit test for checking pandas and numpy

* FIX: back-compatibility defaulting on float 32

* PEP8

* FIX: transform y to integer if a category for back-compat

* PEP8

* DOC: add example

* TST: remove useless tests

* iter

* iter

* iter

* EHN: partially address mfeurer comments

* FIX: append column and concat

* simplify

* FIX: add back missing test files

* CLEAN: remove new useless pkl

* FIX: revert backward compatibility

* PEP8

* PEP8

* fix

* TST: ensure behavior of ignore_attribute

* TST: add test for SparseDataFrame

* raise FutureWarning and avoid warning in testing

* EHN: interpret propely the boolean type

* FIX typo

* PEP8

* MAINT: show slowest tests

* FIX: avoid reallocation in a loop with pandas

* fix typo

* fixes

Author: glemaitre

ci_scripts/test.sh

@@ -22,7 +22,7 @@ run_tests() {
         PYTEST_ARGS=''
     fi
 
-    pytest -n 4 --timeout=600 --timeout-method=thread -sv --ignore='test_OpenMLDemo.py' $PYTEST_ARGS $test_dir
+    pytest -n 4 --duration=20 --timeout=600 --timeout-method=thread -sv --ignore='test_OpenMLDemo.py' $PYTEST_ARGS $test_dir
 }
 
 if [[ "$RUN_FLAKE8" == "true" ]]; then

examples/datasets_tutorial.py

@@ -55,16 +55,28 @@
 ############################################################################
 # Get the actual data.
 #
-# Returned as numpy array, with meta-info
-# (e.g. target feature, feature names, ...)
+# The dataset can be returned in 2 possible formats: as a NumPy array, a SciPy
+# sparse matrix, or as a Pandas DataFrame (or SparseDataFrame). The format is
+# controlled with the parameter ``dataset_format`` which can be either 'array'
+# (default) or 'dataframe'. Let's first build our dataset from a NumPy array
+# and manually create a dataframe.
 X, y, attribute_names = dataset.get_data(
+    dataset_format='array',
     target=dataset.default_target_attribute,
     return_attribute_names=True,
 )
 eeg = pd.DataFrame(X, columns=attribute_names)
 eeg['class'] = y
 print(eeg[:10])
 
+############################################################################
+# Instead of manually creating the dataframe, you can already request a
+# dataframe with the correct dtypes.
+X, y = dataset.get_data(target=dataset.default_target_attribute,
+                        dataset_format='dataframe')
+print(X.head())
+print(X.info())
+
 ############################################################################
 # Exercise 2
 # **********

examples/flows_and_runs_tutorial.py

@@ -17,6 +17,7 @@
 
 dataset = openml.datasets.get_dataset(68)
 X, y = dataset.get_data(
+    dataset_format='array',
     target=dataset.default_target_attribute
 )
 clf = neighbors.KNeighborsClassifier(n_neighbors=1)
@@ -28,6 +29,7 @@
 # * e.g. categorical features -> do feature encoding
 dataset = openml.datasets.get_dataset(17)
 X, y, categorical = dataset.get_data(
+    dataset_format='array',
     target=dataset.default_target_attribute,
     return_categorical_indicator=True,
 )