Support sparse matrices, (needs support in liac-arff)

Author: mfeurer

openml/entities/dataset.py

@@ -16,6 +16,7 @@
 logger = logging.getLogger(__name__)
 
 import numpy as np
+import scipy.sparse
 
 from ..util import is_string
 
@@ -68,8 +69,17 @@ def __init__(self, id, name, version, description, format, creator,
             categorical = [False if type(type_) != list else True
                                 for name, type_ in data['attributes']]
             attribute_names = [name for name, type_ in data['attributes']]
-            # X = pd.DataFrame(data=data['data'], columns=attribute_names)
-            X = np.array(data['data'], dtype=np.float32)
+
+            if isinstance(data['data'], tuple):
+                X = data['data']
+                X_shape = (max(X[1]) + 1, max(X[2]) + 1)
+                X = scipy.sparse.coo_matrix(
+                    (X[0], (X[1], X[2])), shape=X_shape, dtype=np.float32)
+                X = X.tocsr()
+            elif isinstance(data['data'], list):
+                X = np.array(data['data'], dtype=np.float32)
+            else:
+                raise Exception()
 
             with open(self.data_pickle_file, "w") as fh:
                 pickle.dump((X, categorical, attribute_names), fh, -1)
@@ -128,7 +138,7 @@ def get_dataset(self, target=None, include_row_id=False,
                 data, categorical, attribute_names = pickle.load(fh)
 
         to_exclude = []
-        if include_row_id == False:
+        if include_row_id is False:
             if not self.row_id_attribute:
                 pass
             else:
@@ -137,7 +147,7 @@ def get_dataset(self, target=None, include_row_id=False,
                 else:
                     to_exclude.extend(self.row_id_attribute)
 
-        if include_ignore_attributes == False:
+        if include_ignore_attributes is False:
             if not self.ignore_attributes:
                 pass
             else:
@@ -179,6 +189,10 @@ def get_dataset(self, target=None, include_row_id=False,
                 import sys
                 sys.stdout.flush()
                 raise e
+
+            if scipy.sparse.issparse(y):
+                y = np.asarray(y.todense()).astype(np.int32).flatten()
+
             rval.append(x)
             rval.append(y)
 

tests/entities/test_dataset.py

@@ -3,7 +3,7 @@
 import os
 
 import numpy as np
-import pandas as pd
+import scipy.sparse
 
 from openml.entities.dataset import OpenMLDataset
 from openml.util import is_string
@@ -17,7 +17,7 @@ def setUp(self):
             "files", "datasets", "2", "dataset.arff")
         self.pickle_filename = os.path.join(self.directory, "..",
             "files", "datasets", "2", "dataset.pkl")
-        self.dataset = OpenMLDataset(1, "anneal", 1, "Lorem ipsum.",
+        self.dataset = OpenMLDataset(1, "anneal", 2, "Lorem ipsum.",
                                      "arff", None, None, None,
                                      "2014-04-06 23:19:24", None, "Public",
                                      "http://openml.liacs.nl/files/download/2/dataset_2_anneal.ORIG.arff",
@@ -26,8 +26,20 @@ def setUp(self):
                                      "939966a711925e333bf4aaadeaa71135",
                                      data_file=self.arff_filename)
 
+        self.sparse_arff_filename = os.path.join(self.directory, "..",
+            "files", "datasets", "-1", "dataset.arff")
+        self.sparse_pickle_filename = os.path.join(self.directory, "..",
+            "files", "datasets", "-1", "dataset.pkl")
+        self.sparse_dataset = OpenMLDataset(-1, "dexter", -1, "Lorem ipsum.",
+                                            "arff", None, None, None,
+                                            None, None, "Public",
+                                            "http://www.cs.ubc.ca/labs/beta/Projects/autoweka/datasets/dexter.zip",
+                                            "class", None, None, None, None,
+                                            None, None, None, None, None, None,
+                                            data_file=self.sparse_arff_filename)
+
     def tearDown(self):
-        for file_ in [self.pickle_filename]:
+        for file_ in [self.pickle_filename, self.sparse_pickle_filename]:
             os.remove(file_)
 
     ############################################################################
@@ -37,7 +49,7 @@ def tearDown(self):
     def test_get_arff(self):
         rval = self.dataset.get_arff()
         self.assertIsInstance(rval, tuple)
-        self.assertIsInstance(rval[0], pd.DataFrame)
+        self.assertIsInstance(rval[0], np.ndarray)
         self.assertTrue(hasattr(rval[1], '__dict__'))
         self.assertEqual(rval[0].shape, (898, ))
 
@@ -56,8 +68,23 @@ def test_get_dataset(self):
         self.assertEqual(len(attribute_names), 39)
         self.assertTrue(all([is_string(att) for att in attribute_names]))
 
+    def test_get_sparse_dataset(self):
+        rval = self.sparse_dataset.get_dataset()
+        self.assertIsInstance(rval, scipy.sparse.spmatrix)
+        self.assertEqual(rval.dtype, np.float32)
+        self.assertEqual((2, 20001), rval.shape)
+        rval, categorical = self.sparse_dataset.get_dataset(
+            return_categorical_indicator=True)
+        self.assertEqual(len(categorical), 20001)
+        self.assertTrue(all([isinstance(cat, bool) for cat in categorical]))
+        rval, attribute_names = self.sparse_dataset.get_dataset(
+            return_attribute_names=True)
+        self.assertEqual(len(attribute_names), 20001)
+        self.assertTrue(all([is_string(att) for att in attribute_names]))
+
     def test_get_dataset_with_target(self):
         X, y = self.dataset.get_dataset(target="class")
+        self.assertIsInstance(X, np.ndarray)
         self.assertEqual(X.dtype, np.float32)
         self.assertEqual(y.dtype, np.int32)
         self.assertEqual(X.shape, (898, 38))
@@ -67,6 +94,19 @@ def test_get_dataset_with_target(self):
         self.assertNotIn("class", attribute_names)
         self.assertEqual(y.shape, (898, ))
 
+    def test_get_sparse_dataset_with_target(self):
+        X, y = self.sparse_dataset.get_dataset(target="class")
+        self.assertIsInstance(X, scipy.sparse.spmatrix)
+        self.assertEqual(X.dtype, np.float32)
+        self.assertIsInstance(y, np.ndarray)
+        self.assertEqual(y.dtype, np.int32)
+        self.assertEqual(X.shape, (2, 20000))
+        X, y, attribute_names = self.sparse_dataset.get_dataset(
+            target="class", return_attribute_names=True)
+        self.assertEqual(len(attribute_names), 20000)
+        self.assertNotIn("class", attribute_names)
+        self.assertEqual(y.shape, (2, ))
+
     def test_get_dataset_with_rowid(self):
         self.dataset.row_id_attribute = "condition"
         rval, categorical = self.dataset.get_dataset(
@@ -85,6 +125,26 @@ def test_get_dataset_with_rowid(self):
         #self.dataset.row_id_attribute = rowid
         #rval = self.dataset.get_pandas(include_row_id=False)
 
+    def test_get_sparse_dataset_with_rowid(self):
+        self.sparse_dataset.row_id_attribute = "a_0"
+        rval, categorical = self.sparse_dataset.get_dataset(
+            include_row_id=True, return_categorical_indicator=True)
+        self.assertIsInstance(rval, scipy.sparse.spmatrix)
+        self.assertEqual(rval.dtype, np.float32)
+        self.assertEqual(rval.shape, (2, 20001))
+        self.assertEqual(len(categorical), 20001)
+        rval, categorical = self.sparse_dataset.get_dataset(
+            include_row_id=False, return_categorical_indicator=True)
+        self.assertIsInstance(rval, scipy.sparse.spmatrix)
+        self.assertEqual(rval.dtype, np.float32)
+        self.assertEqual(rval.shape, (2, 20000))
+        self.assertEqual(len(categorical), 20000)
+
+        # TODO this is not yet supported!
+        # rowid = ["condition", "formability"]
+        #self.dataset.row_id_attribute = rowid
+        #rval = self.dataset.get_pandas(include_row_id=False)
+
     def test_get_dataset_with_ignore_attributes(self):
         self.dataset.ignore_attributes = "condition"
         rval = self.dataset.get_dataset(include_ignore_attributes=True)
@@ -101,12 +161,21 @@ def test_get_dataset_with_ignore_attributes(self):
         self.assertEqual(len(categorical), 38)
         # TODO test multiple ignore attributes!
 
-    def test_get_dataset_rowid_and_ignore(self):
-        self.dataset.ignore_attributes = "condition"
-        self.dataset.row_id_attribute = "condition"
-        rval = self.dataset.get_dataset(include_ignore_attributes=False,
-                                        include_row_id=False)
+    def test_get_sparse_dataset_with_ignore_attributes(self):
+        self.sparse_dataset.ignore_attributes = "a_0"
+        rval = self.sparse_dataset.get_dataset(include_ignore_attributes=True)
+        self.assertEqual(rval.dtype, np.float32)
+        self.assertEqual(rval.shape, (2, 20001))
+        rval, categorical = self.sparse_dataset.get_dataset(
+            include_ignore_attributes=True, return_categorical_indicator=True)
+        self.assertEqual(len(categorical), 20001)
+        rval = self.sparse_dataset.get_dataset(include_ignore_attributes=False)
         self.assertEqual(rval.dtype, np.float32)
+        self.assertEqual(rval.shape, (2, 20000))
+        rval, categorical = self.sparse_dataset.get_dataset(
+            include_ignore_attributes=False, return_categorical_indicator=True)
+        self.assertEqual(len(categorical), 20000)
+        # TODO test multiple ignore attributes!
 
     def test_get_dataset_rowid_and_ignore_and_target(self):
         self.dataset.ignore_attributes = "condition"
@@ -121,4 +190,18 @@ def test_get_dataset_rowid_and_ignore_and_target(self):
         self.assertEqual(len(categorical), 36)
         self.assertListEqual(categorical, [True]*3 + [False] + [True]*2 + [
             False] + [True]*23 + [False]*3 + [True]*3)
-        self.assertEqual(y.shape, (898, ))
+        self.assertEqual(y.shape, (898, ))
+
+    def test_get_sparse_dataset_rowid_and_ignore_and_target(self):
+        self.sparse_dataset.ignore_attributes = "a_0"
+        self.sparse_dataset.row_id_attribute = "a_1"
+        X, y = self.sparse_dataset.get_dataset(target="class",
+            include_row_id=False, include_ignore_attributes=False)
+        self.assertEqual(X.dtype, np.float32)
+        self.assertEqual(y.dtype, np.int32)
+        self.assertEqual(X.shape, (2, 19998))
+        X, y, categorical = self.sparse_dataset.get_dataset(
+            target="class", return_categorical_indicator=True)
+        self.assertEqual(len(categorical), 19998)
+        self.assertListEqual(categorical, [False] * 19998)
+        self.assertEqual(y.shape, (2, ))