Globstar Support; Fixes #51 (PR#59)

* Added multi-threading code / plus black formatting
* allow glob with relative and absolute paths; +tests
* fix broken README links & isort config

---------

Co-authored-by: Gregory Parkes <gregorymparkes@gmail.com>
Co-authored-by: Teque5 <teque5@gmail.com>

Author: 3 people

README.md

@@ -4,8 +4,7 @@ This python module makes it easy to interact with Signal Metadata Format
 (SigMF) recordings. This module works with Python 3.7+ and is distributed
 freely under the terms GNU Lesser GPL v3 License.
 
-The [SigMF specification document](https://github.com/sigmf/SigMF/blob/HEAD/sigmf-spec.md)
-is located in the [SigMF](https://github.com/sigmf/SigMF) repository.
+This module follows the SigMF specification [html](https://sigmf.org/)/[pdf](https://sigmf.github.io/SigMF/sigmf-spec.pdf) from the [spec repository](https://github.com/sigmf/SigMF).
 
 # Installation
 

pyproject.toml

@@ -91,6 +91,9 @@ inputs = ['sigmf', 'tests']
 [tool.black]
 line-length = 120
 
+[tool.isort]
+profile = "black"
+
 [tool.tox]
 legacy_tox_ini = '''
     [tox]

sigmf/validate.py

@@ -4,11 +4,20 @@
 #
 # SPDX-License-Identifier: LGPL-3.0-or-later
 
-'''SigMF Validator'''
-
+"""SigMF Validator"""
 import argparse
+import glob
 import json
 import logging
+import os
+import sys
+
+# required for Python 3.7
+from typing import Optional, Tuple
+
+# multi-threading library - should work well as I/O will be the primary
+# cost for small SigMF files. Swap to ProcessPool if files are large.
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 import jsonschema
 
@@ -17,43 +26,47 @@
 
 
 def extend_with_default(validator_class):
-    '''
+    """
     Boilerplate code from [1] to retrieve jsonschema default dict.
 
     References
     ----------
     [1] https://python-jsonschema.readthedocs.io/en/stable/faq/
-    '''
+    """
     validate_properties = validator_class.VALIDATORS["properties"]
 
     def set_defaults(validator, properties, instance, topschema):
         for property, subschema in properties.items():
             if "default" in subschema:
                 instance.setdefault(property, subschema["default"])
 
-        for error in validate_properties(
-            validator, properties, instance, topschema,
+        for err in validate_properties(
+            validator,
+            properties,
+            instance,
+            topschema,
         ):
-            yield error
+            yield err
 
     return jsonschema.validators.extend(
-        validator_class, {"properties": set_defaults},
+        validator_class,
+        {"properties": set_defaults},
     )
 
 
 def get_default_metadata(ref_schema=schema.get_schema()):
-    '''
+    """
     retrieve defaults from schema
     FIXME: not working yet
-    '''
+    """
     default = {}
     validator = extend_with_default(jsonschema.Draft7Validator)
     validator(ref_schema).validate(default)
     return default
 
 
 def validate(metadata, ref_schema=schema.get_schema()):
-    '''
+    """
     Check that the provided `metadata` dict is valid according to the `ref_schema` dict.
     Walk entire schema and check all keys.
 
@@ -69,30 +82,67 @@ def validate(metadata, ref_schema=schema.get_schema()):
     Returns
     -------
     None, will raise error if invalid.
-    '''
+    """
     jsonschema.validators.validate(instance=metadata, schema=ref_schema)
 
     # assure capture and annotation order
     # TODO: There is a way to do this with just the schema apparently.
-    for key in ['captures', 'annotations']:
+    for key in ["captures", "annotations"]:
         count = -1
         for item in metadata[key]:
-            new_count = item['core:sample_start']
+            new_count = item["core:sample_start"]
             if new_count < count:
-                raise jsonschema.exceptions.ValidationError(f'{key} has bad order')
+                raise jsonschema.exceptions.ValidationError(f"{key} has bad order")
             else:
                 count = new_count
 
 
-def main():
-    parser = argparse.ArgumentParser(description='Validate SigMF Archive or file pair against JSON schema.',
-                                     prog='sigmf_validate')
-    parser.add_argument('filename', help='SigMF path (extension optional).')
-    parser.add_argument('--skip-checksum', action='store_true', help='Skip reading dataset to validate checksum.')
-    parser.add_argument('-v', '--verbose', action='count', default=0)
-    parser.add_argument('--version', action='version', version=f'%(prog)s {toolversion}')
+def _validate_single_file(filename, skip_checksum: bool, logger: logging.Logger) -> int:
+    """Validates a single SigMF file.
 
-    args = parser.parse_args()
+    To be called as part of a multithreading / multiprocess application.
+
+    Parameters
+    ----------
+    filename : str
+        Path and name to sigmf.data or sigmf.meta file.
+    skip_checksum : bool
+        Whether to perform checksum computation.
+    logger : logging.Logger
+        Logging object to log errors to.
+
+    Returns
+    -------
+    rc : int
+        0 if OK, 1 if err
+    """
+    try:
+        # load signal
+        signal = sigmffile.fromfile(filename, skip_checksum=skip_checksum)
+        # validate
+        signal.validate()
+
+    # handle any of 4 exceptions at once...
+    except (jsonschema.exceptions.ValidationError, error.SigMFFileError, json.decoder.JSONDecodeError, IOError) as err:
+        # catch the error, log, and continue
+        logger.error("file `{}`: {}".format(filename, err))
+        return 1
+    else:
+        return 0
+
+
+def main(arg_tuple: Optional[Tuple[str, ...]] = None) -> None:
+    """entry-point for command-line validator"""
+    parser = argparse.ArgumentParser(
+        description="Validate SigMF Archive or file pair against JSON schema.", prog="sigmf_validate"
+    )
+    parser.add_argument("path", nargs="*", help="SigMF path(s). Accepts * wildcards and extensions are optional.")
+    parser.add_argument("--skip-checksum", action="store_true", help="Skip reading dataset to validate checksum.")
+    parser.add_argument("-v", "--verbose", action="count", default=0)
+    parser.add_argument("--version", action="version", version=f"%(prog)s {toolversion}")
+
+    # allow pass-in arg_tuple for testing purposes
+    args = parser.parse_args(arg_tuple)
 
     level_lut = {
         0: logging.WARNING,
@@ -102,22 +152,35 @@ def main():
     log = logging.getLogger()
     logging.basicConfig(level=level_lut[min(args.verbose, 2)])
 
-    try:
-        signal = sigmffile.fromfile(args.filename, skip_checksum=args.skip_checksum)
-    except error.SigMFFileError as err:
-        # this happens if checksum fails
-        log.error(err)
-        exit(1)
-    except IOError as err:
-        log.error(err)
-        log.error('Unable to read SigMF, bad path?')
-        exit(1)
-    except json.decoder.JSONDecodeError as err:
-        log.error(err)
-        log.error('Unable to decode malformed JSON.')
-        exit(1)
-    signal.validate()
-    log.info('Validation OK!')
+    paths = []
+    # resolve possible wildcards
+    for path in args.path:
+        paths += glob.glob(path)
+
+    # multi-processing / threading pathway.
+    n_completed = 0
+    n_total = len(paths)
+    # estimate number of CPU cores
+    # https://stackoverflow.com/questions/1006289/how-to-find-out-the-number-of-cpus-using-python
+    est_cpu_cores = len(os.sched_getaffinity(0))
+    # create a thread pool
+    # https://docs.python.org/3.7/library/concurrent.futures.html#threadpoolexecutor
+    with ThreadPoolExecutor(max_workers=est_cpu_cores - 1) as executor:
+        # submit jobs
+        future_validations = {executor.submit(_validate_single_file, path, args.skip_checksum, log) for path in paths}
+        # load and await jobs to complete... no return
+        for future in as_completed(future_validations):
+            if future.result() == 0:
+                n_completed += 1
+
+    if n_total == 0:
+        log.error("No paths to validate.")
+        sys.exit(1)
+    elif n_completed != n_total:
+        log.info(f"Validated {n_completed} of {n_total} files OK")
+        sys.exit(1)
+    else:
+        log.info(f"Validated all {n_total} files OK!")
 
 
 if __name__ == "__main__":

tests/test_validation.py

@@ -8,6 +8,7 @@
 
 import tempfile
 import unittest
+from pathlib import Path
 
 from jsonschema.exceptions import ValidationError
 
@@ -23,6 +24,54 @@ def test_valid_data():
     SigMFFile(TEST_METADATA).validate()
 
 
+class CommandLineValidator(unittest.TestCase):
+    """Check behavior of command-line parser"""
+
+    def setUp(self):
+        """Create a directory with some valid files"""
+        self.tmp_dir = tempfile.TemporaryDirectory()
+        self.tmp_path = tmp_path = Path(self.tmp_dir.name)
+        junk_path = tmp_path / "junk"
+        TEST_FLOAT32_DATA.tofile(junk_path)
+        some_meta = SigMFFile(TEST_METADATA, data_file=junk_path)
+        some_meta.tofile(tmp_path / "a")
+        some_meta.tofile(tmp_path / "b")
+        some_meta.tofile(tmp_path / "c", toarchive=True)
+
+    def tearDown(self):
+        """cleanup"""
+        self.tmp_dir.cleanup()
+
+    def test_normal(self):
+        """able to parse archives and non-archives"""
+        args = (str(self.tmp_path / "*.sigmf*"),)
+        sigmf.validate.main(args)
+
+    def test_normal_skip(self):
+        """able to skip checksum"""
+        args = (str(self.tmp_path / "*.sigmf*"), "--skip-checksum")
+        sigmf.validate.main(args)
+
+    def test_partial(self):
+        """checks some but not all files"""
+        args = (str(self.tmp_path / "*"),)
+        with self.assertRaises(SystemExit):
+            sigmf.validate.main(args)
+
+    def test_missing(self):
+        """exit with rc=1 when run on empty"""
+        with self.assertRaises(SystemExit) as cm:
+            sigmf.validate.main(tuple())
+        self.assertEqual((1,), cm.exception.args)
+
+    def test_version(self):
+        """exit with rc=0 after printing version"""
+        args = ("--version",)
+        with self.assertRaises(SystemExit) as cm:
+            sigmf.validate.main(args)
+        self.assertEqual((0,), cm.exception.args)
+
+
 class FailingCases(unittest.TestCase):
     """Cases where the validator should throw an exception."""