ENH add get_data_home to localize the downloaded files

Author: TomDLT

tutorials/movies_3T/00_download_vim4.py

@@ -29,8 +29,11 @@
 # Download
 # --------
 
-# Update the directory variable to link to the directory containing the data.
-directory = '/data1/tutorials/vim-4/'
+# path of the data directory
+import os
+from voxelwise_tutorials.io import get_data_home
+directory = os.path.join(get_data_home(), "vim-4")
+print(directory)
 
 ###############################################################################
 # We will only use the first subject in this tutorial, but you can run the same
@@ -73,11 +76,10 @@
     # "TBD/stimuli/train_11.hdf",
 ]
 
-if __name__ == "__main__":
-
-    username = input("CRCNS username: ")
-    password = getpass.getpass("CRCNS password: ")
+###############################################################################
+username = input("CRCNS username: ")
+password = getpass.getpass("CRCNS password: ")
 
-    for datafile in DATAFILES:
-        local_filename = download_crcns(datafile, username, password,
-                                        destination=directory)
+for datafile in DATAFILES:
+    local_filename = download_crcns(datafile, username, password,
+                                    destination=directory)

tutorials/movies_3T/01_plot_explainable_variance.py

@@ -21,15 +21,17 @@
 ###############################################################################
 
 # path of the data directory
-directory = '/data1/tutorials/vim-4/'
+import os
+from voxelwise_tutorials.io import get_data_home
+directory = os.path.join(get_data_home(), "vim-4")
+print(directory)
 
 # modify to use another subject
 subject = "S01"
 
 ###############################################################################
 # Compute the explainable variance
 # --------------------------------
-import os
 import numpy as np
 
 from voxelwise_tutorials.io import load_hdf5_array

tutorials/movies_3T/02_plot_wordnet_model.py

@@ -28,7 +28,10 @@
 ###############################################################################
 
 # path of the data directory
-directory = '/data1/tutorials/vim-4/'
+import os
+from voxelwise_tutorials.io import get_data_home
+directory = os.path.join(get_data_home(), "vim-4")
+print(directory)
 
 # modify to use another subject
 subject = "S01"
@@ -38,7 +41,6 @@
 # -------------
 #
 # We first load the fMRI responses.
-import os
 import numpy as np
 from voxelwise_tutorials.io import load_hdf5_array
 

tutorials/movies_3T/03_plot_motion_energy_model.py

@@ -32,7 +32,10 @@
 ###############################################################################
 
 # path of the data directory
-directory = '/data1/tutorials/vim-4/'
+import os
+from voxelwise_tutorials.io import get_data_home
+directory = os.path.join(get_data_home(), "vim-4")
+print(directory)
 
 # modify to use another subject
 subject = "S01"
@@ -42,7 +45,6 @@
 # -------------
 #
 # We first load the fMRI responses.
-import os
 import numpy as np
 
 from voxelwise_tutorials.io import load_hdf5_array

tutorials/movies_3T/04_plot_banded_ridge_model.py

@@ -18,7 +18,10 @@
 ###############################################################################
 
 # path of the data directory
-directory = '/data1/tutorials/vim-4/'
+import os
+from voxelwise_tutorials.io import get_data_home
+directory = os.path.join(get_data_home(), "vim-4")
+print(directory)
 
 # modify to use another subject
 subject = "S01"
@@ -29,7 +32,6 @@
 #
 # As in the previous examples, we first load the fMRI responses, which are our
 # regression targets.
-import os
 import numpy as np
 
 from voxelwise_tutorials.io import load_hdf5_array

tutorials/movies_3T/05_extract_motion_energy.py

@@ -26,7 +26,10 @@
 ###############################################################################
 
 # path of the data directory
-directory = '/data1/tutorials/vim-4/'
+import os
+from voxelwise_tutorials.io import get_data_home
+directory = os.path.join(get_data_home(), "vim-4")
+print(directory)
 
 ###############################################################################
 # Load the stimuli images
@@ -35,7 +38,6 @@
 # Here the data is not loaded in memory, we only take a peak at the data shape.
 
 import h5py
-import os
 
 first_file_name = os.path.join(directory, 'stimuli', 'train_00.hdf')
 print(f"Content of {first_file_name}:")

tutorials/movies_4T/00_download_vim2.py

@@ -27,8 +27,11 @@
 # Download
 # --------
 
-# Update the directory variable to link to the directory containing the data.
-directory = '/data1/tutorials/vim-2/'
+# path of the data directory
+import os
+from voxelwise_tutorials.io import get_data_home
+directory = os.path.join(get_data_home(), "vim-2")
+print(directory)
 
 ###############################################################################
 # We will only use the first subject in this tutorial, but you can run the same
@@ -50,11 +53,10 @@
     'vim-2/docs/crcns-vim-2-data-description.pdf',
 ]
 
-if __name__ == "__main__":
-
-    username = input("CRCNS username: ")
-    password = getpass.getpass("CRCNS password: ")
+###############################################################################
+username = input("CRCNS username: ")
+password = getpass.getpass("CRCNS password: ")
 
-    for datafile in DATAFILES:
-        local_filename = download_crcns(datafile, username, password,
-                                        destination=directory, unpack=True)
+for datafile in DATAFILES:
+    local_filename = download_crcns(datafile, username, password,
+                                    destination=directory, unpack=True)

tutorials/movies_4T/01_extract_motion_energy.py

@@ -25,11 +25,13 @@
 ###############################################################################
 # Load the stimuli images
 # -----------------------
-#
-# We downloaded the files in the previous script, and here we update the path
-# variable to link to the directory containing the data.
+# (We downloaded the files in the previous script.)
 
-directory = '/data1/tutorials/vim-2/'
+# path of the data directory
+import os
+from voxelwise_tutorials.io import get_data_home
+directory = os.path.join(get_data_home(), "vim-2")
+print(directory)
 
 ###############################################################################
 # Here the data is not loaded in memory, we only take a peak at the data shape.

tutorials/movies_4T/02_plot_ridge_model.py

@@ -26,16 +26,22 @@
 ###############################################################################
 # Load the data
 # -------------
-#
+
 # path of the data directory
-directory = '/data1/tutorials/vim-2/'
+import os
+from voxelwise_tutorials.io import get_data_home
+directory = os.path.join(get_data_home(), "vim-2")
+print(directory)
+
+# modify to use another subject
+subject = "subject1"
 
 ###############################################################################
 # Here the data is not loaded in memory, we only take a peak at the data shape.
 import h5py
 import os.path as op
 
-with h5py.File(op.join(directory, 'VoxelResponses_subject1.mat'), 'r') as f:
+with h5py.File(op.join(directory, f'VoxelResponses_{subject}.mat'), 'r') as f:
     print(f.keys())  # Show all variables
     for key in f.keys():
         print(f[key])
@@ -47,7 +53,7 @@
 
 from voxelwise_tutorials.io import load_hdf5_array
 
-file_name = op.join(directory, 'VoxelResponses_subject1.mat')
+file_name = op.join(directory, f'VoxelResponses_{subject}.mat')
 Y_train = load_hdf5_array(file_name, key='rt')
 Y_test_repeats = load_hdf5_array(file_name, key='rva')
 

voxelwise_tutorials/io.py

@@ -1,5 +1,4 @@
 import os
-import os.path as op
 import requests
 import shutil
 
@@ -51,16 +50,17 @@ def download_crcns(datafile, username, password, destination,
                 raise RuntimeError(response.text)
 
         # remove the dataset name
-        filename = op.join(*login_data['fn'].split('/')[1:])
-        local_filename = op.join(destination, filename)
+        filename = os.path.join(*login_data['fn'].split('/')[1:])
+        local_filename = os.path.join(destination, filename)
 
         # create subdirectory if necessary
-        local_directory = op.dirname(local_filename)
-        if not op.exists(local_directory) or not op.isdir(local_directory):
+        local_directory = os.path.dirname(local_filename)
+        if not os.path.exists(local_directory) or not os.path.isdir(
+                local_directory):
             os.makedirs(local_directory)
 
         # download the file if it does not already exist
-        if op.exists(local_filename):
+        if os.path.exists(local_filename):
             print("%s already exists." % local_filename)
         else:
             bar = ProgressBar(title=filename, max_value=content_length)
@@ -71,7 +71,7 @@ def download_crcns(datafile, username, password, destination,
                         f.write(chunk)
 
     # uncompress archives
-    if unpack and op.splitext(local_filename)[1] in [".zip", ".gz"]:
+    if unpack and os.path.splitext(local_filename)[1] in [".zip", ".gz"]:
         unpack_archive(local_filename)
 
     return local_filename
@@ -86,7 +86,7 @@ def unpack_archive(archive_name):
         Local name of the archive.
     """
     print('\tUnpacking')
-    extract_dir = op.dirname(archive_name)
+    extract_dir = os.path.dirname(archive_name)
     shutil.unpack_archive(archive_name, extract_dir=extract_dir)
 
 
@@ -169,3 +169,43 @@ def save_hdf5_dataset(file_name, dataset, mode='w'):
                 hf.create_dataset(name, data=array, compression='gzip')
 
     print("Saved %s" % file_name)
+
+
+def get_data_home(data_home=None) -> str:
+    """Return the path of the voxelwise tutorials data dir.
+
+    This folder is used by some large dataset loaders to avoid downloading the
+    data several times. By default the data dir is set to a folder named
+    'voxelwise_tutorials' in the user home folder. Alternatively, it can be set
+    by the 'VOXELWISE_TUTORIALS_DATA' environment variable or programmatically
+    by giving an explicit folder path. The '~' symbol is expanded to the user
+    home folder. If the folder does not already exist, it is automatically
+    created.
+
+    Parameters
+    ----------
+    data_home : str | None
+        The path to voxelwise tutorials data dir.
+    """
+    if data_home is None:
+        data_home = os.environ.get(
+            'VOXELWISE_TUTORIALS_DATA',
+            os.path.join('~', 'voxelwise_tutorials_data'))
+
+    data_home = os.path.expanduser(data_home)
+    if not os.path.exists(data_home):
+        os.makedirs(data_home)
+
+    return data_home
+
+
+def clear_data_home(data_home=None):
+    """Delete all the content of the data home cache.
+
+    Parameters
+    ----------
+    data_home : str | None
+        The path to voxelwise tutorials data dir.
+    """
+    data_home = get_data_home(data_home)
+    shutil.rmtree(data_home)