fix CI

Author: mfeurer

ci_scripts/install.sh

@@ -28,9 +28,9 @@ conda create -n testenv --yes python=$PYTHON_VERSION pip
 source activate testenv
 pip install nose numpy scipy cython scikit-learn==$SKLEARN_VERSION oslo.concurrency
 
-if [[ "EXAMPLES" == "true" ]]; then
-    pip install matplotlib jupyter notebook nbconvert nbformat jupyter_client ipython \
-        ipykernel pandas seaborn
+if [[ "$EXAMPLES" == "true" ]]; then
+    pip install matplotlib jupyter notebook nbconvert nbformat jupyter_client \
+        ipython ipykernel pandas seaborn
 fi
 
 if [[ "$COVERAGE" == "true" ]]; then

ci_scripts/test.sh

@@ -9,8 +9,10 @@ test_dir=$cwd/tests
 
 cd $TEST_DIR
 
-if [[ "$COVERAGE" == "true" ]]; then
-    nosetests --processes=4 --process-timeout=600 -sv --with-coverage --cover-package=$MODULE $test_dir
+if [[ "$EXAMPLES" == "true" ]]; then
+    nosetests -sv $test_dir/test_examples/
+elif [[ "$COVERAGE" == "true" ]]; then
+    nosetests --processes=4 --process-timeout=600 -sv --ignore-files="test_OpenMLDemo\.py" --with-coverage --cover-package=$MODULE $test_dir
 else
-    nosetests --processes=4 --process-timeout=600 -sv $test_dir
+    nosetests --processes=4 --process-timeout=600 -sv --ignore-files="test_OpenMLDemo\.py" $test_dir
 fi

openml/config.py

@@ -13,6 +13,7 @@
     format='[%(levelname)s] [%(asctime)s:%(name)s] %('
            'message)s', datefmt='%H:%M:%S')
 
+config_file = os.path.expanduser('~/.openml/config')
 server = "https://www.openml.org/api/v1/xml"
 apikey = ""
 cachedir = ""
@@ -70,7 +71,9 @@ def set_cache_directory(cachedir):
     run_cache_dir = os.path.join(cachedir, 'runs')
     lock_dir = os.path.join(cachedir, 'locks')
 
-    for dir_ in [cachedir, dataset_cache_dir, task_cache_dir, run_cache_dir]:
+    for dir_ in [
+        cachedir, dataset_cache_dir, task_cache_dir, run_cache_dir, lock_dir,
+    ]:
         if not os.path.exists(dir_) and not os.path.isdir(dir_):
             os.mkdir(dir_)
 
@@ -84,7 +87,6 @@ def _parse_config():
                 'cachedir': os.path.expanduser('~/.openml/cache'),
                 'avoid_duplicate_runs': 'True'}
 
-    config_file = os.path.expanduser('~/.openml/config')
     config = configparser.RawConfigParser(defaults=defaults)
 
     if not os.path.exists(config_file):

openml/testing.py

@@ -5,6 +5,7 @@
 import time
 import unittest
 
+from oslo_concurrency import lockutils
 import six
 
 import openml
@@ -56,6 +57,13 @@ def setUp(self):
 
         openml.config.set_cache_directory(self.workdir)
 
+        # If we're on travis, we save the api key in the config file to allow
+        # the notebook tests to read them.
+        if os.environ.get('TRAVIS'):
+            with lockutils.external_lock('config', lock_path=self.workdir):
+                with open(openml.config.config_file, 'w') as fh:
+                    fh.write('apikey = %s' % openml.config.apikey)
+
     def tearDown(self):
         os.chdir(self.cwd)
         shutil.rmtree(self.workdir)

tests/test_examples/test_OpenMLDemo.py

@@ -1,15 +1,18 @@
 import os
-import unittest
 import shutil
 import sys
 
 import nbformat
 from nbconvert.preprocessors import ExecutePreprocessor
 from nbconvert.preprocessors.execute import CellExecutionError
 
+from openml.testing import TestBase
 
-class OpenMLDemoTest(unittest.TestCase):
+
+class OpenMLDemoTest(TestBase):
     def setUp(self):
+        super(OpenMLDemoTest, self).setUp()
+
         python_version = sys.version_info[0]
         self.kernel_name = 'python%d' % python_version
         self.this_file_directory = os.path.dirname(__file__)
@@ -37,7 +40,7 @@ def _test_notebook(self, notebook_name):
             nb = nbformat.read(f, as_version=4)
             nb.metadata.get('kernelspec', {})['name'] = self.kernel_name
             ep = ExecutePreprocessor(kernel_name=self.kernel_name)
-            ep.timeout = 60
+            ep.timeout = 180
 
             try:
                 ep.preprocess(nb, {'metadata': {'path': self.this_file_directory}})

tests/test_runs/test_run_functions.py

@@ -756,22 +756,22 @@ def test__create_trace_from_arff(self):
     def test_get_run(self):
         # this run is not available on test
         openml.config.server = self.production_server
-        run = openml.runs.get_run(473344)
-        self.assertEqual(run.dataset_id, 1167)
-        self.assertEqual(run.evaluations['f_measure'], 0.624668)
-        for i, value in [(0, 0.66233),
-                         (1, 0.639286),
-                         (2, 0.567143),
-                         (3, 0.745833),
-                         (4, 0.599638),
-                         (5, 0.588801),
-                         (6, 0.527976),
-                         (7, 0.666365),
-                         (8, 0.56759),
-                         (9, 0.64621)]:
+        run = openml.runs.get_run(473351)
+        self.assertEqual(run.dataset_id, 357)
+        self.assertEqual(run.evaluations['f_measure'], 0.841225)
+        for i, value in [(0, 0.840918),
+                         (1, 0.839458),
+                         (2, 0.839613),
+                         (3, 0.842571),
+                         (4, 0.839567),
+                         (5, 0.840922),
+                         (6, 0.840985),
+                         (7, 0.847129),
+                         (8, 0.84218),
+                         (9, 0.844014)]:
             self.assertEqual(run.fold_evaluations['f_measure'][0][i], value)
         assert('weka' in run.tags)
-        assert('stacking' in run.tags)
+        assert('weka_3.7.12' in run.tags)
 
     def _check_run(self, run):
         self.assertIsInstance(run, dict)