Mark unit test as flaky (#770)

* add flaky decorator and install flaky
* remove reruns flag

Author: mfeurer

setup.py

@@ -48,7 +48,8 @@
                          'pytest-xdist',
                          'pytest-timeout',
                          'nbformat',
-                         'oslo.concurrency'
+                         'oslo.concurrency',
+                         'flaky',
                      ],
                      'examples': [
                          'matplotlib',

tests/test_runs/test_run.py

@@ -134,7 +134,7 @@ def test_to_from_filesystem_vanilla(self):
         TestBase.logger.info("collected from {}: {}".format(__file__.split('/')[-1],
                                                             run_prime.run_id))
 
-    @pytest.mark.flaky(reruns=3)
+    @pytest.mark.flaky()
     def test_to_from_filesystem_search(self):
 
         model = Pipeline([

tests/test_runs/test_run_functions.py

@@ -7,6 +7,7 @@
 import unittest.mock
 
 import numpy as np
+import pytest
 
 import openml
 import openml.exceptions
@@ -826,6 +827,7 @@ def test_initialize_model_from_run(self):
         self.assertEqual(flowS.components['VarianceThreshold'].
                          parameters['threshold'], '0.05')
 
+    @pytest.mark.flaky()
     def test_get_run_trace(self):
         # get_run_trace is already tested implicitly in test_run_and_publish
         # this test is a bit additional.