TST add more tests

Author: TomDLT

.github/workflows/run_tests.yml

@@ -24,6 +24,11 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-
 
+    - uses: actions/cache@v3
+      with:
+        path: ~/.voxelwise_tutorials_data/shortclips
+        key: ${{ runner.os }}-shortclips
+
     - name: Install dependencies
       run: |
         pip install -U setuptools

voxelwise_tutorials/tests/test_mappers.py

@@ -6,33 +6,35 @@
 import os
 
 import numpy as np
+import cortex
+from cortex.testing_utils import has_installed
 import matplotlib.pyplot as plt
 
+from voxelwise_tutorials.io import load_hdf5_array
 from voxelwise_tutorials.io import load_hdf5_sparse_array
 from voxelwise_tutorials.viz import plot_flatmap_from_mapper
 from voxelwise_tutorials.viz import plot_2d_flatmap_from_mapper
 
 from voxelwise_tutorials.io import get_data_home
 from voxelwise_tutorials.io import download_datalad
 
-dataset_directory = get_data_home(dataset="shortclips")
-subject_id = "S01"
+subject = "S01"
+directory = get_data_home(dataset="shortclips")
+file_name = os.path.join("mappers", f'{subject}_mappers.hdf')
+mapper_file = os.path.join(directory, file_name)
 
 # download mapper if not already present
-download_datalad("mappers/S01_mappers.hdf", destination=dataset_directory,
+download_datalad(file_name, destination=directory,
                  source="https://gin.g-node.org/gallantlab/shortclips")
 
+# Change to save = True to save the figures locally and check the results
+save_fig = False
 
-def test_flatmap_mappers():
 
-    # Change to save = True to save the figures locally and check the results
-    save_fig = False
+def test_flatmap_mappers():
 
     ##################
     # create fake data
-    mapper_file = os.path.join(dataset_directory, "mappers",
-                               '{}_mappers.hdf'.format(subject_id))
-
     voxel_to_flatmap = load_hdf5_sparse_array(mapper_file, 'voxel_to_flatmap')
     voxels = np.linspace(0, 1, voxel_to_flatmap.shape[1])
 
@@ -42,7 +44,7 @@ def test_flatmap_mappers():
                                   ax=None)
     fig = ax.figure
     if save_fig:
-        fig.savefig(f'{subject_id}.png')
+        fig.savefig(f'test.png')
     plt.close(fig)
 
 
@@ -53,9 +55,6 @@ def test_plot_2d_flatmap_from_mapper():
 
     ##################
     # create fake data
-    mapper_file = os.path.join(dataset_directory, "mappers",
-                               '{}_mappers.hdf'.format(subject_id))
-
     voxel_to_flatmap = load_hdf5_sparse_array(mapper_file, 'voxel_to_flatmap')
     phase = np.linspace(0, 2 * np.pi, voxel_to_flatmap.shape[1])
     sin = np.sin(phase)
@@ -67,5 +66,46 @@ def test_plot_2d_flatmap_from_mapper():
                                      vmin=-1, vmax=1, vmin2=-1, vmax2=1)
     fig = ax.figure
     if save_fig:
-        fig.savefig(f'{subject_id}.png')
+        fig.savefig(f'test_2d.png')
+    plt.close(fig)
+
+
+def test_roi_masks_shape():
+    all_mappers = load_hdf5_array(mapper_file, key=None)
+
+    n_pixels, n_voxels = all_mappers['voxel_to_flatmap_shape']
+    n_vertices, n_voxels_ = all_mappers['voxel_to_fsaverage_shape']
+    assert n_voxels_ == n_voxels
+
+    for key, val in all_mappers.items():
+        if 'roi_mask_' in key:
+            assert val.shape == (n_voxels, )
+
+
+def test_fsaverage_mappers():
+
+    # Change to save = True to save the figures locally and check the results
+    save_fig = False
+
+    ##################
+    # create fake data
+    voxel_to_fsaverage = load_hdf5_sparse_array(mapper_file,
+                                                'voxel_to_fsaverage')
+    voxels = np.linspace(0, 1, voxel_to_fsaverage.shape[1])
+
+    ##################
+    # download fsaverage subject
+    if not hasattr(cortex.db, "fsaverage"):
+        cortex.utils.download_subject(subject_id="fsaverage",
+                                      pycortex_store=cortex.db.filestore)
+        cortex.db.reload_subjects()  # force filestore reload
+
+    #############################
+    # plot with fsaverage mappers
+    projected = voxel_to_fsaverage @ voxels
+    vertex = cortex.Vertex(projected, 'fsaverage', vmin=0, vmax=0.3,
+                           cmap='inferno', alpha=0.7, with_curvature=True)
+    fig = cortex.quickshow(vertex, with_rois=has_installed("inkscape"))
+    if save_fig:
+        fig.savefig(f'test_fsaverage.png')
     plt.close(fig)

voxelwise_tutorials/tests/test_model.py

@@ -0,0 +1,99 @@
+import os
+
+import pytest
+import numpy as np
+
+from sklearn.model_selection import check_cv
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import StandardScaler
+
+from himalaya.backend import set_backend
+from himalaya.kernel_ridge import KernelRidgeCV
+
+from voxelwise_tutorials.delayer import Delayer
+from voxelwise_tutorials.io import load_hdf5_array
+from voxelwise_tutorials.io import get_data_home
+from voxelwise_tutorials.io import download_datalad
+from voxelwise_tutorials.utils import explainable_variance
+from voxelwise_tutorials.utils import generate_leave_one_run_out
+
+# use "cupy" or "torch_cuda" for faster computation with GPU
+backend = set_backend("numpy", on_error="warn")
+
+# Download the dataset
+subject = "S01"
+feature_spaces = ["motion_energy", "wordnet"]
+directory = get_data_home(dataset="shortclips")
+for file_name in [
+        "features/motion_energy.hdf",
+        "features/wordnet.hdf",
+        "mappers/S01_mappers.hdf",
+        "responses/S01_responses.hdf",
+]:
+    download_datalad(file_name, destination=directory,
+                     source="https://gin.g-node.org/gallantlab/shortclips")
+
+
+def run_model(X_train, X_test, Y_train, Y_test, run_onsets):
+    ##############
+    # define model
+    n_samples_train = Y_train.shape[0]
+    cv = generate_leave_one_run_out(n_samples_train, run_onsets,
+                                    random_state=0, n_runs_out=1)
+    cv = check_cv(cv)
+
+    alphas = np.logspace(-4, 15, 20)
+
+    model = make_pipeline(
+        StandardScaler(with_mean=True, with_std=False),
+        Delayer(delays=[1, 2, 3, 4]),
+        KernelRidgeCV(
+            kernel="linear", alphas=alphas, cv=cv,
+            solver_params=dict(n_targets_batch=1000, n_alphas_batch=10)),
+    )
+
+    ###########
+    # run model
+    model.fit(X_train, Y_train)
+    test_scores = model.score(X_test, Y_test)
+
+    test_scores = backend.to_numpy(test_scores)
+    # cv_scores = backend.to_numpy(model[-1].cv_scores_)
+
+    return test_scores
+
+
+@pytest.mark.parametrize('feature_space', feature_spaces)
+def test_model_fitting(feature_space):
+    ###########################################
+    # load the data
+
+    # load X
+    features_file = os.path.join(directory, 'features',
+                                 feature_space + ".hdf")
+    features = load_hdf5_array(features_file)
+    X_train = features['X_train']
+    X_test = features['X_test']
+
+    # load Y
+    responses_file = os.path.join(directory, 'responses',
+                                  subject + "_responses.hdf")
+    responses = load_hdf5_array(responses_file)
+    Y_train = responses['Y_train']
+    Y_test_repeats = responses['Y_test']
+    run_onsets = responses['run_onsets']
+
+    #############################################
+    # select voxels based on explainable variance
+    ev = explainable_variance(Y_test_repeats)
+    mask = ev > 0.4
+    assert mask.sum() > 0
+    Y_train = Y_train[:, mask]
+    Y_test = Y_test_repeats[:, :, mask].mean(0)
+
+    ###########################################
+    # fit a ridge model and compute test scores
+    test_scores = run_model(X_train, X_test, Y_train, Y_test, run_onsets)
+    assert np.percentile(test_scores, 95) > 0.05
+    assert np.percentile(test_scores, 99) > 0.15
+    assert np.percentile(test_scores, 100) > 0.35