ci: remove custom error collection and use pytest

modify tox.ini to run pytest in validate environment with -vv so that
the nested dictionary comparison is actually shown

Author: james-garner-canonical

tests/validate/test_facades.py

@@ -21,96 +21,73 @@ class Versions(TypedDict, total=True):
 
 
 @pytest.fixture
-def project_root(request: pytest.FixtureRequest):
-    return request.config.rootpath
-
-
-class TestFacades:
-    def test_client_facades(self, project_root: Path) -> None:
-        good_facades = self.generate_client_facades(project_root)
-        client_facades = cast(ClientFacades, connection.client_facades)
-
-        errors: List[Tuple[str, Optional[List[int]], Optional[List[int]]]] = []
-        all_names = sorted(set(connection.client_facades).union(good_facades))
-        for name in all_names:
-            expected = self._versions_from_facades(good_facades, name)
-            actual = self._versions_from_facades(client_facades, name)
-            if expected != actual:
-                errors.append((name, expected, actual))
-
-        if errors:
-            print('The following errors were found in connection.client_facades:')
-        for name, expected, actual in errors:
-            expected_msg = (
-                f'should be {expected},'
-                if expected is not None
-                else 'should not be present,'
-            )
-            actual_msg = (
-                f'not {actual}'
-                if actual is not None
-                else 'but is not present'
-            )
-            print(f'    {name!r} {expected_msg} {actual_msg}')
-
-        assert not errors
-
-    @classmethod
-    def generate_client_facades(cls, project_root: Path) -> ClientFacades:
-        """Return a client_facades dictionary from generated code under project_root.
-        """
-        files_by_version: List[Tuple[int, Path]] = []
-        # [(facade_version, Path), ...]
-        for file in (project_root / 'juju' / 'client').glob('_client[0-9]*.py'):
-            files_by_version.append((cls._version_from_filename(file), file))
-        files_by_version.sort()
-
-        # _clientN.py files import * from _definitions
-        # so we will ignore any names from there
-        ignore = dir(importlib.import_module('juju.client._definitions'))
-
-        facades_by_version: Dict[int, Set[str]] = {}
-        # {facade_version: {facade_name, ...}, ...}
-        for version, file in files_by_version:
-            module = cls._try_import(f'juju.client.{file.stem}')
-            facades = {
-                name.removesuffix("Facade")
-                for name in dir(module)
-                if not (name.startswith('_') or name in ignore)
-            }
-            facades_by_version[version] = facades
-
-        # client_facades in connection.py is sorted
-        # so we sort facade names before constructing it
-        first, *rest = facades_by_version.values()
-        sorted_facade_names: list[str] = sorted(first.union(*rest))
-
-        client_facades: ClientFacades = {}
-        # {facade_name: {'versions': [1, 2, 3, ...]}, ...}
-        for name in sorted_facade_names:
-            versions: List[int] = []
-            for version, facades in facades_by_version.items():
-                if name in facades:
-                    versions.append(version)
-            client_facades[name] = {'versions': versions}
-        return client_facades
-
-    @staticmethod
-    def _try_import(module_name: str) -> Optional[ModuleType]:
-        try:
-            return importlib.import_module(module_name)
-        except NameError as e:
-            warnings.warn(f'error on importing {module_name}:\n{type(e).__name__}: {e}')
-            return None
-
-    @staticmethod
-    def _version_from_filename(path: Path) -> int:
-        match = re.search('_client([0-9]+).py', path.name)
-        assert match
-        return int(match.group(1))
-
-    @staticmethod
-    def _versions_from_facades(facades: ClientFacades, name: str) -> Optional[List[int]]:
-        if name not in facades:
-            return None
-        return facades[name]['versions']
+def project_root(pytestconfig: pytest.Config) -> Path:
+    return pytestconfig.rootpath
+
+
+def test_client_facades(project_root: Path) -> None:
+    good_facades = make_client_facades_from_generated_code(project_root)
+    client_facades = cast(ClientFacades, connection.client_facades)
+
+    assert {
+        k: v['versions'] for k, v in client_facades.items()
+    } == {
+        k: v['versions'] for k, v in good_facades.items()
+    }
+
+
+def make_client_facades_from_generated_code(project_root: Path) -> ClientFacades:
+    """Return a client_facades dictionary from generated code under project_root.
+    """
+    files_by_version: List[Tuple[int, Path]] = []
+    # [(facade_version, Path), ...]
+    for file in (project_root / 'juju' / 'client').glob('_client[0-9]*.py'):
+        files_by_version.append((_version_from_filename(file), file))
+    files_by_version.sort()
+
+    # _clientN.py files import * from _definitions
+    # so we will ignore any names from there
+    ignore = dir(importlib.import_module('juju.client._definitions'))
+
+    facades_by_version: Dict[int, Set[str]] = {}
+    # {facade_version: {facade_name, ...}, ...}
+    for version, file in files_by_version:
+        module = _try_import(f'juju.client.{file.stem}')
+        facades = {
+            name.removesuffix("Facade")
+            for name in dir(module)
+            if not (name.startswith('_') or name in ignore)
+        }
+        facades_by_version[version] = facades
+
+    # client_facades in connection.py is sorted
+    # so we sort facade names before constructing it
+    first, *rest = facades_by_version.values()
+    sorted_facade_names: list[str] = sorted(first.union(*rest))
+
+    client_facades: ClientFacades = {}
+    # {facade_name: {'versions': [1, 2, 3, ...]}, ...}
+    for name in sorted_facade_names:
+        versions: List[int] = []
+        for version, facades in facades_by_version.items():
+            if name in facades:
+                versions.append(version)
+        client_facades[name] = {'versions': versions}
+    return client_facades
+
+def _try_import(module_name: str) -> Optional[ModuleType]:
+    try:
+        return importlib.import_module(module_name)
+    except NameError as e:
+        warnings.warn(f'error on importing {module_name}:\n{type(e).__name__}: {e}')
+        return None
+
+def _version_from_filename(path: Path) -> int:
+    match = re.search('_client([0-9]+).py', path.name)
+    assert match
+    return int(match.group(1))
+
+def _versions_from_facades(facades: ClientFacades, name: str) -> Optional[List[int]]:
+    if name not in facades:
+        return None
+    return facades[name]['versions']

tox.ini

@@ -105,7 +105,7 @@ commands =
 
 [testenv:validate]
 envdir = {toxworkdir}/validate
-commands = python -m pytest --tb native -ra -v tests/validate
+commands = python -m pytest --tb native -ra -vv tests/validate
 
 [testenv:example]
 envdir = {toxworkdir}/py3