global xml context

Author: valentin-gauthier-geosiris

energyml-utils/example/attic/compare_inmem_n_stream.py

@@ -140,6 +140,13 @@ def time_comparison(
     print(f"   • Overall speedup: {speedup_factor:.2f}x faster\n")
 
 
+def recompute_rels(epc_file_path: str):
+    with EpcStreamReader(
+        epc_file_path=epc_file_path, enable_parallel_rels=True, rels_update_mode=RelsUpdateMode.UPDATE_ON_CLOSE
+    ) as reader:
+        pass  # Just open and close to trigger rels computation on close
+
+
 if __name__ == "__main__":
     logging.basicConfig(level=logging.DEBUG)
 
@@ -150,11 +157,13 @@ def time_comparison(
     #     output_folder="rc/performance_results",
     # )
 
-    time_comparison(
-        filepath=sys.argv[1] if len(sys.argv) > 1 else "rc/epc/80wells_surf.epc", output_folder="rc/performance_results"
-    )
+    # time_comparison(
+    #     filepath=sys.argv[1] if len(sys.argv) > 1 else "rc/epc/80wells_surf.epc", output_folder="rc/performance_results"
+    # )
 
     # time_comparison(
     #     filepath=sys.argv[1] if len(sys.argv) > 1 else "wip/failingData/fix/sample_mini_firp_201_norels_with_media.epc",
     #     output_folder="rc/performance_results",
     # )
+
+    recompute_rels("C:/Users/Cryptaro/Downloads/Galaxy384-[[Output] EPC file pointset extraction].epc")

energyml-utils/example/attic/parsing_improvement_test.py

@@ -0,0 +1,84 @@
+"""
+Test for parsing.
+
+To test : edit _read_energyml_xml_bytes_as_class in serialization.py :
+
+__ENV__IMPROVEMENT__ = "__ENV__IMPROVEMENT__"
+"__ENV__IMPROVEMENT_LXML__" = ""__ENV__IMPROVEMENT_LXML__""
+
+    if os.environ.get(__ENV__IMPROVEMENT__, "0") == "0":
+        if os.environ.get("__ENV__IMPROVEMENT_LXML__", "0") == "1":
+            parser = XmlParser(config=config, handler=LxmlEventHandler)
+        else:
+            parser = XmlParser(config=config)
+    else:
+        if os.environ.get("__ENV__IMPROVEMENT_LXML__", "0") == "1":
+            parser = XmlParser(config=config, context=GLOBAL_XML_CONTEXT, handler=LxmlEventHandler)
+        else:
+            parser = XmlParser(config=config, context=GLOBAL_XML_CONTEXT)
+
+"""
+
+import logging
+import os
+import sys
+import time
+from typing import Optional
+
+from energyml.utils.epc import Epc
+
+
+def reexport_in_memory_par_read(filepath: str, output_folder: Optional[str] = None):
+    is_opti = os.environ.get("__ENV__IMPROVEMENT__", "0") == "1"
+
+    suffix = "opti" if is_opti else "std"
+    if os.environ.get("__ENV__IMPROVEMENT_LXML__", "0") == "1":
+        suffix += "_lxml"
+
+    path_in_memory = filepath.replace(".epc", f"_parsing_imp_xml_{suffix}.epc")
+    if output_folder:
+        os.makedirs(output_folder, exist_ok=True)
+        path_in_memory = f"{output_folder}/{path_in_memory.split('/')[-1]}"
+    epc = Epc.read_file(epc_file_path=filepath, read_rels_from_files=False, read_parallel=True, recompute_rels=False)
+
+    if os.path.exists(path_in_memory):
+        os.remove(path_in_memory)
+    epc.export_file(path_in_memory, parallel=True)
+
+
+def time_test(f: callable, **kwargs):
+    print(f"⏳ Testing {f.__name__}...")
+    start = time.perf_counter()
+    f(**kwargs)
+    elapsed_inmem = time.perf_counter() - start
+    # results.append(("In-Memory (Epc)", elapsed_inmem))
+    print(f"   ✓ Completed in {elapsed_inmem:.3f}s\n")
+    return ("In-Memory (Epc)", elapsed_inmem)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.DEBUG)
+
+    os.environ["__ENV__IMPROVEMENT__"] = "0"
+    os.environ["__ENV__IMPROVEMENT_LXML__"] = "0"
+
+    time_test(
+        reexport_in_memory_par_read,
+        filepath=sys.argv[1] if len(sys.argv) > 1 else "rc/epc/80wells_surf.epc",
+        output_folder="results",
+    )
+
+    os.environ["__ENV__IMPROVEMENT__"] = "1"
+    time_test(
+        reexport_in_memory_par_read,
+        filepath=sys.argv[1] if len(sys.argv) > 1 else "rc/epc/80wells_surf.epc",
+        output_folder="results",
+    )
+
+    os.environ["__ENV__IMPROVEMENT__"] = "1"
+    os.environ["__ENV__IMPROVEMENT_LXML__"] = "1"
+    time_test(
+        reexport_in_memory_par_read,
+        filepath=sys.argv[1] if len(sys.argv) > 1 else "rc/epc/80wells_surf.epc",
+        output_folder="results",
+    )

energyml-utils/src/energyml/utils/serialization.py

@@ -42,6 +42,13 @@
     ENERGYML_NAMESPACES,
 )
 
+from xsdata.formats.dataclass.parsers.handlers import LxmlEventHandler
+
+GLOBAL_XML_CONTEXT = XmlContext(
+    # element_name_generator=text.camel_case,
+    # attribute_name_generator=text.kebab_case
+)
+
 
 class JSON_VERSION(Enum):
     XSDATA = "XSDATA"
@@ -65,7 +72,7 @@ def _read_energyml_xml_bytes_as_class(
         fail_on_unknown_attributes=fail_on_unknown_attributes,
         # process_xinclude=True,
     )
-    parser = XmlParser(config=config)
+    parser = XmlParser(config=config, context=GLOBAL_XML_CONTEXT, handler=LxmlEventHandler)
     try:
         return parser.from_bytes(file, obj_class)
     except ParserError as e:
@@ -81,11 +88,6 @@ def _read_energyml_xml_bytes_as_class(
 
 
 def read_energyml_xml_tree(file: etree, obj_type: Optional[type] = None) -> Any:
-    # if obj_type is None:
-    #     obj_type = get_class_from_name(get_class_name_from_xml(file))
-    # parser = XmlParser(handler=XmlEventHandler)
-    # # parser = XmlParser(handler=LxmlEventHandler)
-    # return parser.parse(file, obj_type)
     return read_energyml_xml_bytes(etree.tostring(file, encoding="utf8"))
 
 
@@ -155,7 +157,7 @@ def _read_energyml_json_bytes_as_class(file: bytes, json_version: JSON_VERSION,
             # fail_on_unknown_attributes=False,
             # process_xinclude=True,
         )
-        parser = JsonParser(config=config)
+        parser = JsonParser(config=config, context=GLOBAL_XML_CONTEXT)
         try:
             return parser.from_bytes(file, obj_class)
         except ParserError as e:
@@ -269,12 +271,8 @@ def serialize_xml(obj, check_obj_prefixed_classes: bool = True) -> str:
     # logging.debug(f"[1] Serializing object of type {type(obj)}")
     obj = as_obj_prefixed_class_if_possible(obj) if check_obj_prefixed_classes else obj
     # logging.debug(f"[2] Serializing object of type {type(obj)}")
-    context = XmlContext(
-        # element_name_generator=text.camel_case,
-        # attribute_name_generator=text.kebab_case
-    )
     serializer_config = SerializerConfig(indent="  ")
-    serializer = XmlSerializer(context=context, config=serializer_config)
+    serializer = XmlSerializer(context=GLOBAL_XML_CONTEXT, config=serializer_config)
     # res = serializer.render(obj)
     res = serializer.render(obj, ns_map=ENERGYML_NAMESPACES)
     # logging.debug(f"[3] Serialized XML with meta namespace : {obj.Meta.namespace}: {serialize_json(obj)}")
@@ -286,12 +284,8 @@ def serialize_json(
 ) -> str:
     obj = as_obj_prefixed_class_if_possible(obj) if check_obj_prefixed_classes else obj
     if json_version == JSON_VERSION.XSDATA:
-        context = XmlContext(
-            # element_name_generator=text.camel_case,
-            # attribute_name_generator=text.kebab_case
-        )
         serializer_config = SerializerConfig(indent="  ")
-        serializer = JsonSerializer(context=context, config=serializer_config)
+        serializer = JsonSerializer(context=GLOBAL_XML_CONTEXT, config=serializer_config)
         return serializer.render(obj)
     elif json_version == JSON_VERSION.OSDU_OFFICIAL:
         return json.dumps(to_json_dict(obj), indent=4, sort_keys=True)