Prefer lazy loading in unit tests (#655)

* Prefer lazy loading for all unit tests that don't explicitly need the arff file.

* Skip test for which API is currently not working.

Author: PGijsbers

tests/test_datasets/test_dataset.py

@@ -20,12 +20,12 @@ def setUp(self):
 
         # Load dataset id 2 - dataset 2 is interesting because it contains
         # missing values, categorical features etc.
-        self.dataset = openml.datasets.get_dataset(2)
+        self.dataset = openml.datasets.get_dataset(2, download_data=False)
         # titanic as missing values, categories, and string
-        self.titanic = openml.datasets.get_dataset(40945)
+        self.titanic = openml.datasets.get_dataset(40945, download_data=False)
         # these datasets have some boolean features
-        self.pc4 = openml.datasets.get_dataset(1049)
-        self.jm1 = openml.datasets.get_dataset(1053)
+        self.pc4 = openml.datasets.get_dataset(1049, download_data=False)
+        self.jm1 = openml.datasets.get_dataset(1053, download_data=False)
 
     def test_get_data_future_warning(self):
         warn_msg = 'will change from "array" to "dataframe"'
@@ -197,7 +197,7 @@ class OpenMLDatasetTestOnTestServer(TestBase):
     def setUp(self):
         super(OpenMLDatasetTestOnTestServer, self).setUp()
         # longley, really small dataset
-        self.dataset = openml.datasets.get_dataset(125)
+        self.dataset = openml.datasets.get_dataset(125, download_data=False)
 
     def test_tagging(self):
         tag = "testing_tag_{}_{}".format(self.id(), time())
@@ -219,7 +219,7 @@ def setUp(self):
         super(OpenMLDatasetTestSparse, self).setUp()
         openml.config.server = self.production_server
 
-        self.sparse_dataset = openml.datasets.get_dataset(4136)
+        self.sparse_dataset = openml.datasets.get_dataset(4136, download_data=False)
 
     def test_get_sparse_dataset_with_target(self):
         X, y = self.sparse_dataset.get_data(

tests/test_datasets/test_dataset_functions.py

@@ -271,6 +271,7 @@ def test_get_datasets_lazy(self):
             openml.config.get_cache_directory(), "datasets", "2", "dataset.arff")))
 
     def test_get_dataset(self):
+        # This is the only non-lazy load to ensure default behaviour works.
         dataset = openml.datasets.get_dataset(1)
         self.assertEqual(type(dataset), OpenMLDataset)
         self.assertEqual(dataset.name, 'anneal')
@@ -313,7 +314,7 @@ def test_get_dataset_lazy(self):
 
         # Issue324 Properly handle private datasets when trying to access them
         openml.config.server = self.production_server
-        self.assertRaises(OpenMLPrivateDatasetError, openml.datasets.get_dataset, 45)
+        self.assertRaises(OpenMLPrivateDatasetError, openml.datasets.get_dataset, 45, False)
 
     def test_get_dataset_lazy_all_functions(self):
         """ Test that all expected functionality is available without downloading the dataset. """
@@ -343,14 +344,14 @@ def test_get_dataset_lazy_all_functions(self):
             openml.config.get_cache_directory(), "datasets", "1", "dataset.arff")))
 
     def test_get_dataset_sparse(self):
-        dataset = openml.datasets.get_dataset(102)
+        dataset = openml.datasets.get_dataset(102, download_data=False)
         X = dataset.get_data(dataset_format='array')
         self.assertIsInstance(X, scipy.sparse.csr_matrix)
 
     def test_download_rowid(self):
         # Smoke test which checks that the dataset has the row-id set correctly
         did = 44
-        dataset = openml.datasets.get_dataset(did)
+        dataset = openml.datasets.get_dataset(did, download_data=False)
         self.assertEqual(dataset.row_id_attribute, 'Counter')
 
     def test__get_dataset_description(self):
@@ -416,7 +417,7 @@ def test_deletion_of_cache_dir_faulty_download(self, patch):
         self.assertEqual(len(os.listdir(datasets_cache_dir)), 0)
 
     def test_publish_dataset(self):
-
+        # lazy loading not possible as we need the arff-file.
         openml.datasets.get_dataset(3)
         file_path = os.path.join(openml.config.get_cache_directory(),
                                  "datasets", "3", "dataset.arff")
@@ -434,9 +435,9 @@ def test_publish_dataset(self):
 
     def test__retrieve_class_labels(self):
         openml.config.cache_directory = self.static_cache_dir
-        labels = openml.datasets.get_dataset(2).retrieve_class_labels()
+        labels = openml.datasets.get_dataset(2, download_data=False).retrieve_class_labels()
         self.assertEqual(labels, ['1', '2', '3', '4', '5', 'U'])
-        labels = openml.datasets.get_dataset(2).retrieve_class_labels(
+        labels = openml.datasets.get_dataset(2, download_data=False).retrieve_class_labels(
             target_name='product-type')
         self.assertEqual(labels, ['C', 'H', 'G'])
 
@@ -761,9 +762,8 @@ def test_create_invalid_dataset(self):
         )
 
     def test_get_online_dataset_arff(self):
-
-        # Australian dataset
-        dataset_id = 100
+        dataset_id = 100  # Australian
+        # lazy loading not used as arff file is checked.
         dataset = openml.datasets.get_dataset(dataset_id)
         decoder = arff.ArffDecoder()
         # check if the arff from the dataset is
@@ -785,7 +785,7 @@ def test_get_online_dataset_format(self):
 
         # Phoneme dataset
         dataset_id = 77
-        dataset = openml.datasets.get_dataset(dataset_id)
+        dataset = openml.datasets.get_dataset(dataset_id, download_data=False)
 
         self.assertEqual(
             (dataset.format).lower(),

tests/test_runs/test_run_functions.py

@@ -1434,6 +1434,7 @@ def test_get_runs_list_by_filters(self):
 
         runs = openml.runs.list_runs(id=ids, task=tasks, uploader=uploaders_1)
 
+    @unittest.skip("API currently broken: https://github.com/openml/OpenML/issues/948")
     def test_get_runs_list_by_tag(self):
         # TODO: comes from live, no such lists on test
         openml.config.server = self.production_server