better valdiation result

Author: valentin-gauthier-geosiris

energyml-utils/README.md

@@ -318,6 +318,26 @@ Validate a folder containing EPC/XML/JSON files:
 poetry run validate --file "path/to/your/folder" *> output_logs.json
 ```
 
+Ignore specific error types (e.g., INFO):
+```bash
+poetry run validate --file "path/to/file.epc" --ignore-err-type INFO *> output_logs.json
+```
+
+Group errors by their class for better organization:
+```bash
+poetry run validate --file "path/to/file.epc" --group-by-err-class *> output_logs.json
+```
+
+Include PRODML version errors in validation (by default they are ignored):
+```bash
+poetry run validate --file "path/to/file.epc" --ignore-prodml-version-errs *> output_logs.json
+```
+
+Combined example with multiple options:
+```bash
+poetry run validate --file "path/to/file.epc" -i INFO WARNING --group-by-err-class *> output_logs.json
+```
+
 ### Extract 3D Representations
 
 Extract all representations from an EPC to OBJ files:

energyml-utils/example/tools.py

@@ -4,6 +4,7 @@
 import json
 import os
 import pathlib
+import traceback
 from typing import Optional, List, Dict, Any
 import sys
 from pathlib import Path
@@ -12,14 +13,15 @@
 src_path = Path(__file__).parent.parent / "src"
 sys.path.insert(0, str(src_path))
 
-from energyml.utils.validation import validate_epc
+from energyml.utils.validation import ErrorType, validate_epc
 
 from energyml.utils.constants import get_property_kind_dict_path_as_xml
 from energyml.utils.data.datasets_io import CSVFileReader, HDF5FileWriter, ParquetFileWriter, DATFileReader
 from energyml.utils.data.mesh import MeshFileFormat, export_multiple_data, export_obj, read_mesh_object
 from energyml.utils.epc import Epc, gen_energyml_object_path
 from energyml.utils.introspection import (
     get_class_from_simple_name,
+    get_enum_values,
     get_module_name_and_type_from_content_or_qualified_type,
     random_value_from_class,
     search_class_in_module_from_partial_name,
@@ -548,6 +550,26 @@ def validate_files():
     parser = argparse.ArgumentParser()
     # parser.add_argument("--folder", type=str, help="Input folder")
     parser.add_argument("--file", "-f", type=str, help="Input file (json or xml or epc)")
+    parser.add_argument(
+        "--ignore-err-type",
+        "-i",
+        type=str,
+        help=f"Error types to ignore. Possible values {get_enum_values(ErrorType)}",
+        nargs="*",
+    )
+
+    parser.add_argument(
+        "--ignore-prodml-version-errs",
+        action="store_false",
+        dest="ignore_prodml_version_errs",
+        help="Disable ignoring errors related to Prodml version (by default, these errors are ignored)",
+    )
+
+    parser.add_argument(
+        "--group-by-err-class",
+        action="store_true",
+        help="Group errors by their class (e.g. all validation errors together, all parsing errors together, etc.)",
+    )
 
     args = parser.parse_args()
 
@@ -615,14 +637,36 @@ def validate_files():
                 else:
                     print(f"File {filename} is NOT a valid EnergyML EPC file: Empty EPC")
             except Exception as e:
+                traceback.print_exc()
                 print(f"File {filename} is NOT a valid EnergyML EPC file: {e}")
 
     epc = Epc()
     epc.energyml_objects = objects
 
-    err_json = [err.toJson() for err in validate_epc(epc)]
+    err_json = [
+        err.toJson()
+        for err in validate_epc(epc)
+        if str(err.error_type).lower() not in (et.lower() for et in (args.ignore_err_type or []))
+    ]
 
-    print(json.dumps(err_json, indent=4))
+    err_json_sorted = sorted(
+        err_json, key=lambda x: (x["err_class"], x["error_type"], x["object_uuid"] if "object_uuid" in x else "")
+    )
+
+    if args.ignore_prodml_version_errs:
+        err_json_sorted = [err for err in err_json_sorted if not ("prodml23" in err.get("msg", ""))]
+
+    if args.group_by_err_class:
+        err_json_grouped = {}
+        for err in err_json_sorted:
+            err_class = err.get("err_class", "UnknownErrorClass")
+            if err_class not in err_json_grouped:
+                err_json_grouped[err_class] = []
+            err_json_grouped[err_class].append(err)
+        print(json.dumps(err_json_grouped, indent=4))
+    else:
+        # print(json.dumps(err_json, indent=4))
+        print(json.dumps(err_json_sorted, indent=4))
 
 
 # def export_wavefront():

energyml-utils/rc/epc/testingPackageCpp.h5

@@ -367,7 +367,7 @@ def gen_opc_content_type(self) -> Types:
 
         return ct
 
-    @log_timestamp
+    # @log_timestamp
     def export_file(self, path: Optional[str] = None, allowZip64: bool = True) -> None:
         """
         Export the epc file. If :param:`path` is None, the epc 'self.epc_file_path' is used
@@ -869,7 +869,7 @@ def write_array(
     # Class methods
 
     @classmethod
-    @log_timestamp
+    # @log_timestamp
     def read_file(cls, epc_file_path: str) -> "Epc":
         with open(epc_file_path, "rb") as f:
             epc = cls.read_stream(BytesIO(f.read()))

energyml-utils/src/energyml/utils/epc.py

@@ -1199,6 +1199,24 @@ def get_obj_title(obj: Any) -> Optional[str]:
                     for ck in obj[k].keys():
                         if re.match(r"title", ck, re.IGNORECASE):
                             return obj[k][ck]
+            # search for title or name if not classical citation.title found
+
+            for k in obj.keys():
+                if re.match(r"title", k, re.IGNORECASE):
+                    return obj[k]
+                elif re.match(r"name", k, re.IGNORECASE):
+                    return obj[k]
+
+        else:
+            # DOR :
+            try:
+                return getattr(obj, "title")
+            except AttributeError:
+                # etp resource meta :
+                try:
+                    return getattr(obj, "name")
+                except AttributeError:
+                    pass
     return None
 
 
@@ -1706,6 +1724,10 @@ def get_all_possible_instanciable_classes_for_attribute(parent_obj: Any, attribu
     return []
 
 
+def get_enum_values(cls: Any) -> List[str]:
+    return cls._member_names_ if is_enum(cls) else []
+
+
 def _random_value_from_class(
     cls: Any,
     energyml_module_context: List[str],

energyml-utils/src/energyml/utils/introspection.py

@@ -37,6 +37,9 @@ class ErrorType(Enum):
     INFO = "info"
     WARNING = "warning"
 
+    def __str__(self):
+        return self.value
+
 
 @dataclass
 class ValidationError:
@@ -52,6 +55,7 @@ def toJson(self):
         return {
             "msg": self.msg,
             "error_type": self.error_type.value,
+            "err_class": self.__class__.__name__,
         }
 
     @property

energyml-utils/src/energyml/utils/validation.py

@@ -658,6 +658,20 @@ def test_get_obj_title(triangulated_set_no_version, fault_interpretation):
     """Test extracting object title."""
     assert get_obj_title(triangulated_set_no_version) == "Test Citation v2.3"
     assert get_obj_title(fault_interpretation) == "Test Citation v2.0"
+    assert get_obj_title(as_dor(fault_interpretation)) == "Test Citation v2.0"
+
+    class MockObjWithTitle:
+        name = "Mock Title"
+
+    assert get_obj_title(MockObjWithTitle()) == "Mock Title"
+
+    assert get_obj_title({"Title": "Dict Title"}) == "Dict Title"
+    assert get_obj_title({"title": "Dict Title Lower"}) == "Dict Title Lower"
+    assert get_obj_title({"what": 42}) is None
+    assert get_obj_title({"name": "Dict Title Lower"}) == "Dict Title Lower"
+
+    # priority to citation.title
+    assert get_obj_title({"name": "Dict Title Lower", "citation": {"title": "Citation Title"}}) == "Citation Title"
 
 
 def test_get_obj_type(triangulated_set_no_version, fault_interpretation):