Merge pull request #256 from openml/random_search_online

add additional flag to unit test argument

Author: mfeurer

openml/runs/functions.py

@@ -258,6 +258,18 @@ def _get_seeded_model(model, seed=None):
             a seed
     '''
 
+    def _seed_current_object(current_value):
+        if isinstance(current_value, int):  # acceptable behaviour
+            return False
+        elif isinstance(current_value, np.random.RandomState):
+            raise ValueError(
+                'Models initialized with a RandomState object are not supported. Please seed with an integer. ')
+        elif current_value is not None:
+            raise ValueError(
+                'Models should be seeded with int or None (this should never happen). ')
+        else:
+            return True
+
     rs = np.random.RandomState(seed)
     model_params = model.get_params()
     random_states = {}
@@ -268,18 +280,22 @@ def _get_seeded_model(model, seed=None):
             # this way we guarantee that if a different set of subflows is seeded,
             # the same number of the random generator is used
             newValue = rs.randint(0, 2**16)
-            if currentValue is None:
+            if _seed_current_object(currentValue):
                 random_states[param_name] = newValue
-            elif isinstance(currentValue, int):
-                # acceptable behaviour
-                pass
-            elif isinstance(currentValue, np.random.RandomState):
-                raise ValueError('Models initialized with a RandomState object are not supported. Please seed with an integer. ')
-            else:
-                raise ValueError('Models should be seeded with int or None (this should never happen). ')
-            model.set_params(**random_states)
-    return model
 
+        # Also seed CV objects!
+        elif isinstance(model_params[param_name],
+                        sklearn.model_selection.BaseCrossValidator):
+            if not hasattr(model_params[param_name], 'random_state'):
+                continue
+
+            currentValue = model_params[param_name].random_state
+            newValue = rs.randint(0, 2 ** 16)
+            if _seed_current_object(currentValue):
+                model_params[param_name].random_state = newValue
+
+    model.set_params(**random_states)
+    return model
 
 
 def _prediction_to_row(rep_no, fold_no, row_id, correct_label, predicted_label,

tests/test_runs/test_run_functions.py

@@ -264,14 +264,14 @@ def test_run_and_upload(self):
              "min_samples_leaf": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
              "bootstrap": [True, False],
              "criterion": ["gini", "entropy"]},
-            cv=StratifiedKFold(n_splits=2, random_state=1),
+            cv=StratifiedKFold(n_splits=2, shuffle=True),
             n_iter=num_iterations)
 
         clfs.append(randomsearch)
         # The random states for the RandomizedSearchCV is set after the
         # random state of the RandomForestClassifier is set, therefore,
         # it has a different value than the other examples before
-        random_state_fixtures.append('33003')
+        random_state_fixtures.append('12172')
 
         for clf, rsv in zip(clfs, random_state_fixtures):
             run = self._perform_run(task_id, num_test_instances, clf,
@@ -291,6 +291,30 @@ def test_run_and_upload(self):
             self._check_detailed_evaluations(run.detailed_evaluations, 1, num_folds)
             pass
 
+    def test_initialize_cv_from_run(self):
+        randomsearch = RandomizedSearchCV(
+            RandomForestClassifier(n_estimators=5),
+            {"max_depth": [3, None],
+             "max_features": [1, 2, 3, 4],
+             "min_samples_split": [2, 3, 4, 5, 6, 7, 8, 9, 10],
+             "min_samples_leaf": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+             "bootstrap": [True, False],
+             "criterion": ["gini", "entropy"]},
+            cv=StratifiedKFold(n_splits=2, shuffle=True),
+            n_iter=2)
+
+        task = openml.tasks.get_task(11)
+        run = openml.runs.run_model_on_task(task, randomsearch,
+                                            avoid_duplicate_runs=False, seed=1)
+        run_ = run.publish()
+        run = openml.runs.get_run(run_.run_id)
+
+        modelR = openml.runs.initialize_model_from_run(run.run_id)
+        modelS = openml.setups.initialize_model(run.setup_id)
+
+        self.assertEquals(modelS.cv.random_state, 62501)
+        self.assertEqual(modelR.cv.random_state, 62501)
+
     def test_initialize_model_from_run(self):
         clf = sklearn.pipeline.Pipeline(steps=[('Imputer', Imputer(strategy='median')),
                                                ('VarianceThreshold', VarianceThreshold(threshold=0.05)),
@@ -392,11 +416,11 @@ def test__get_seeded_model(self):
                                 "bootstrap": [True, False],
                                 "criterion": ["gini", "entropy"],
                                 "random_state" : [-1, 0, 1, 2]},
-                               ),
+                               cv=StratifiedKFold(n_splits=2, shuffle=True)),
             DummyClassifier()
         ]
 
-        for clf in randomized_clfs:
+        for idx, clf in enumerate(randomized_clfs):
             const_probe = 42
             all_params = clf.get_params()
             params = [key for key in all_params if key.endswith('random_state')]
@@ -417,6 +441,9 @@ def test__get_seeded_model(self):
                 self.assertIsInstance(new_params[param], int)
                 self.assertIsNotNone(new_params[param])
 
+            if idx == 1:
+                self.assertEqual(clf.cv.random_state, 56422)
+
     def test__get_seeded_model_raises(self):
         # the _get_seeded_model should raise exception if random_state is anything else than an int
         randomized_clfs = [