Merge pull request #399 from openml/fix373

fixes #373 + unit test

Author: janvanrijn

openml/runs/functions.py

@@ -594,11 +594,16 @@ def _extract_arfftrace_attributes(model):
     for key in model.cv_results_:
         if key.startswith('param_'):
             # supported types should include all types, including bool, int float
-            supported_types = (bool, int, float, six.string_types)
-            if all(isinstance(i, supported_types) or i is None for i in model.cv_results_[key]):
-                type = 'STRING'
-            else:
-                raise TypeError('Unsupported param type in param grid')
+            supported_basic_types = (bool, int, float, six.string_types)
+            for param_value in model.cv_results_[key]:
+                if isinstance(param_value, supported_basic_types) or param_value is None:
+                    # basic string values
+                    type = 'STRING'
+                elif isinstance(param_value, list) and all(isinstance(i, int) for i in param_value):
+                    # list of integers
+                    type = 'STRING'
+                else:
+                    raise TypeError('Unsupported param type in param grid: %s' %key)
 
             # we renamed the attribute param to parameter, as this is a required
             # OpenML convention

tests/test_runs/test_run_functions.py

@@ -27,6 +27,7 @@
 from sklearn.feature_selection import VarianceThreshold
 from sklearn.linear_model import LogisticRegression, SGDClassifier, \
     LinearRegression
+from sklearn.neural_network import MLPClassifier
 from sklearn.ensemble import RandomForestClassifier, BaggingClassifier
 from sklearn.svm import SVC, LinearSVC
 from sklearn.model_selection import RandomizedSearchCV, GridSearchCV, \
@@ -617,18 +618,21 @@ def test__get_seeded_model_raises(self):
             self.assertRaises(ValueError, _get_seeded_model, model=clf, seed=42)
 
     def test__extract_arfftrace(self):
-        param_grid = {"max_depth": [3, None],
-                      "max_features": [1, 2, 3, 4],
-                      "bootstrap": [True, False],
-                      "criterion": ["gini", "entropy"]}
+        param_grid = {"hidden_layer_sizes": [[5, 5], [10, 10], [20, 20]],
+                      "activation" : ['identity', 'logistic', 'tanh', 'relu'],
+                      "learning_rate_init": [0.1, 0.01, 0.001, 0.0001],
+                      "max_iter": [10, 20, 40, 80]}
         num_iters = 10
         task = openml.tasks.get_task(20)
-        clf = RandomizedSearchCV(RandomForestClassifier(), param_grid, num_iters)
+        clf = RandomizedSearchCV(MLPClassifier(), param_grid, num_iters)
         # just run the task
         train, _ = task.get_train_test_split_indices(0, 0)
         X, y = task.get_X_and_y()
         clf.fit(X[train], y[train])
 
+        # check num layers of MLP
+        self.assertIn(clf.best_estimator_.hidden_layer_sizes, param_grid['hidden_layer_sizes'])
+
         trace_attribute_list = _extract_arfftrace_attributes(clf)
         trace_list = _extract_arfftrace(clf, 0, 0)
         self.assertIsInstance(trace_attribute_list, list)
@@ -662,7 +666,6 @@ def test__extract_arfftrace(self):
                     else: # att_type = real
                         self.assertIsInstance(trace_list[line_idx][att_idx], float)
 
-
         self.assertEqual(set(param_grid.keys()), optimized_params)
 
     def test__prediction_to_row(self):

tests/test_setups/test_setup_functions.py

@@ -138,7 +138,7 @@ def test_setup_list_filter_flow(self):
             self.assertEquals(setups[setup_id].flow_id, flow_id)
 
     def test_list_setups_empty(self):
-        setups = openml.setups.list_setups(setup=[-1])
+        setups = openml.setups.list_setups(setup=[0])
         if len(setups) > 0:
             raise ValueError('UnitTest Outdated, got somehow results')