optimization

Author: valentin-gauthier-geosiris

energyml-utils/example/attic/compare_inmem_n_stream.py

@@ -40,19 +40,19 @@ def reexport_in_memory(filepath: str, output_folder: Optional[str] = None):
         os.makedirs(output_folder, exist_ok=True)
         path_in_memory = f"{output_folder}/{path_in_memory.split('/')[-1]}"
     epc = Epc.read_file(epc_file_path=filepath, read_rels_from_files=False, recompute_rels=False)
-
+    print(len(epc.list_objects()))
     if os.path.exists(path_in_memory):
         os.remove(path_in_memory)
     epc.export_file(path_in_memory)
 
 
 def reexport_in_memory_par_read(filepath: str, output_folder: Optional[str] = None):
-    path_in_memory = filepath.replace(".epc", "_in_memory_par_read.epc")
+    path_in_memory = filepath.replace(".epc", f"_in_memory_par_read_v{os.environ['EPC_FAST_V2']}.epc")
     if output_folder:
         os.makedirs(output_folder, exist_ok=True)
         path_in_memory = f"{output_folder}/{path_in_memory.split('/')[-1]}"
     epc = Epc.read_file(epc_file_path=filepath, read_rels_from_files=False, read_parallel=True, recompute_rels=False)
-
+    print(len(epc.list_objects()))
     if os.path.exists(path_in_memory):
         os.remove(path_in_memory)
     epc.export_file(path_in_memory, parallel=True)
@@ -80,13 +80,23 @@ def time_comparison(
     print(f"   ✓ Completed in {elapsed_inmem:.3f}s\n")
 
     # Test 1b: In-Memory with Parallel Read
+    os.environ["EPC_FAST_V2"] = "0"
     print("⏳ Testing In-Memory EPC processing with Parallel Read...")
     start = time.perf_counter()
     reexport_in_memory_par_read(filepath, output_folder)
     elapsed_inmem_par = time.perf_counter() - start
     results.append(("In-Memory (Epc) Parallel Read", elapsed_inmem_par))
     print(f"   ✓ Completed in {elapsed_inmem_par:.3f}s\n")
 
+    # Test 1b: In-Memory with Parallel Read v2
+    os.environ["EPC_FAST_V2"] = "1"
+    print("⏳ Testing In-Memory EPC processing with Parallel Read v2...")
+    start = time.perf_counter()
+    reexport_in_memory_par_read(filepath, output_folder)
+    elapsed_inmem_par = time.perf_counter() - start
+    results.append(("In-Memory (Epc) Parallel Read v2", elapsed_inmem_par))
+    print(f"   ✓ Completed in {elapsed_inmem_par:.3f}s\n")
+
     if not skip_sequential_stream:
         # Test 2: Streaming Sequential
         print("⏳ Testing Streaming Sequential processing...")
@@ -157,13 +167,13 @@ def recompute_rels(epc_file_path: str):
     #     output_folder="rc/performance_results",
     # )
 
-    # time_comparison(
-    #     filepath=sys.argv[1] if len(sys.argv) > 1 else "rc/epc/80wells_surf.epc", output_folder="rc/performance_results"
-    # )
+    time_comparison(
+        filepath=sys.argv[1] if len(sys.argv) > 1 else "rc/epc/80wells_surf.epc", output_folder="rc/performance_results"
+    )
 
     # time_comparison(
     #     filepath=sys.argv[1] if len(sys.argv) > 1 else "wip/failingData/fix/sample_mini_firp_201_norels_with_media.epc",
     #     output_folder="rc/performance_results",
     # )
 
-    recompute_rels("C:/Users/Cryptaro/Downloads/Galaxy384-[[Output] EPC file pointset extraction].epc")
+    # recompute_rels("C:/Users/Cryptaro/Downloads/Galaxy384-[[Output] EPC file pointset extraction].epc")

energyml-utils/example/attic/perf_tests.py

@@ -0,0 +1,58 @@
+# Benchmark de performance pour get_obj_uuid
+import time
+import re
+from types import SimpleNamespace
+
+UUID_RGX: re.Pattern = re.compile(r"[Uu]u?id|UUID")
+
+
+# Version originale
+def get_obj_uuid_original(obj):
+    try:
+        return getattr(obj, "uuid", None) or getattr(obj, "uid")
+    except AttributeError:
+        if isinstance(obj, dict):
+            for k in obj.keys():
+                if UUID_RGX.match(k):
+                    return obj[k]
+    return None
+
+
+# Version optimisée
+def get_obj_uuid_fast(obj):
+    for attr in dir(obj):
+        if UUID_RGX.match(attr):
+            value = getattr(obj, attr, None)
+            if value is not None:
+                return value
+    if isinstance(obj, dict):
+        for k, v in obj.items():
+            if UUID_RGX.match(k):
+                if v is not None:
+                    return v
+    return None
+
+
+# Simulation d'une classe TriangulatedSetRepresentation
+class TriangulatedSetRepresentation:
+    def __init__(self, uuid):
+        self.uuid = uuid
+
+
+N = 10000
+objs = [TriangulatedSetRepresentation(f"uuid-{i}") for i in range(N)]
+
+# Test version originale
+start = time.perf_counter()
+for obj in objs:
+    assert get_obj_uuid_original(obj) == obj.uuid
+elapsed_original = time.perf_counter() - start
+
+# Test version optimisée
+start = time.perf_counter()
+for obj in objs:
+    assert get_obj_uuid_fast(obj) == obj.uuid
+elapsed_fast = time.perf_counter() - start
+
+print(f"Original version: {elapsed_original:.6f} s for {N} calls")
+print(f"Optimized version: {elapsed_fast:.6f} s for {N} calls")

energyml-utils/src/energyml/utils/epc.py

@@ -1553,15 +1553,22 @@ def read_file(
         """
         with open(epc_file_path, "rb") as f:
             if read_parallel:
-                epc = cls.read_stream_ultra_fast(
-                    BytesIO(f.read()), read_rels_from_files=read_rels_from_files, recompute_rels=recompute_rels
+                epc = (
+                    cls.read_stream_ultra_fast(
+                        BytesIO(f.read()), read_rels_from_files=read_rels_from_files, recompute_rels=recompute_rels
+                    )
+                    if not os.environ.get("EPC_FAST_V2", "0") == "1"
+                    else cls.read_stream_ultra_fast_v2(
+                        BytesIO(f.read()), read_rels_from_files=read_rels_from_files, recompute_rels=recompute_rels
+                    )
                 )
             else:
                 epc = cls.read_stream(
                     BytesIO(f.read()), read_rels_from_files=read_rels_from_files, recompute_rels=recompute_rels
                 )
-            epc.epc_file_path = epc_file_path
-            return epc
+            if epc is not None:
+                epc.epc_file_path = epc_file_path
+                return epc
         raise IOError(f"Failed to open EPC file {epc_file_path}")
 
     @classmethod
@@ -1575,6 +1582,7 @@ def read_stream(
         :param recompute_rels: If True, recompute all relationships after loading
         :return: an :class:`EPC` instance
         """
+        print("Reading EPC file seq...")
         try:
             _read_files = []
             obj_list = []
@@ -1727,6 +1735,8 @@ def read_stream_ultra_fast(
         from concurrent.futures import ProcessPoolExecutor, as_completed
         import multiprocessing
 
+        print("Reading EPC file parrallel v1...")
+
         obj_to_process = {}
         rels_to_process = {}
         raw_files = []
@@ -1813,6 +1823,87 @@ def read_stream_ultra_fast(
 
         return epc
 
+    @classmethod
+    def read_stream_ultra_fast_v2(
+        cls, epc_file_io: BytesIO, read_rels_from_files: bool = True, recompute_rels: bool = False
+    ) -> Optional["Epc"]:
+        from concurrent.futures import ThreadPoolExecutor  # Passage au ThreadPool
+
+        print("Reading EPC file parrallel v2...")
+
+        obj_list = []
+        path_to_obj = {}
+        rels_content_map = {}
+        raw_files = []
+        core_props = None
+
+        # On utilise un ThreadPool pour éviter le coût de sérialisation Pickle
+        # lxml libère le GIL, donc c'est très efficace
+        with ThreadPoolExecutor() as executor:
+            futures = []
+
+            with zipfile.ZipFile(epc_file_io, "r") as epc_file:
+                # On récupère l'index d'abord
+                ct_path = get_epc_content_type_path()
+                content_type_obj = read_energyml_xml_bytes(epc_file.read(ct_path))
+
+                # Identification des types via le ContentTypes
+                energyml_paths = {}
+                for ov in content_type_obj.override:
+                    path = ov.part_name.lstrip("/\\")
+                    if is_energyml_content_type(ov.content_type):
+                        energyml_paths[path] = ov.content_type
+                    elif get_class_from_content_type(ov.content_type) == CoreProperties:
+                        core_props = read_energyml_xml_bytes(epc_file.read(path), CoreProperties)
+
+                for info in epc_file.infolist():
+                    fname = info.filename
+
+                    # STREAMING : On lance la tâche dès qu'on a les bytes
+                    if fname in energyml_paths:
+                        data = epc_file.read(fname)
+                        f = executor.submit(_parallel_xml_read, data, energyml_paths[fname])
+                        futures.append((f, "OBJ", fname))
+
+                    elif read_rels_from_files and fname.lower().endswith(".rels"):
+                        data = epc_file.read(fname)
+                        f = executor.submit(_parallel_rels_read, data)
+                        futures.append((f, "REL", fname))
+                    elif (
+                        not fname.lower().endswith(".rels")
+                        and not fname.lower().endswith(gen_core_props_path().lower())
+                        and fname not in energyml_paths
+                        and fname != ct_path
+                    ):
+                        raw_files.append(RawFile(path=fname, content=BytesIO(epc_file.read(fname))))
+
+            # 2. Récupération des résultats (pendant que le ZIP continue d'être lu si possible)
+            for future, kind, path in futures:
+                res = future.result()
+                if isinstance(res, Exception):
+                    continue
+
+                if kind == "OBJ":
+                    path_to_obj[path] = res
+                    obj_list.append(res)
+                else:
+                    o_path = str(Path(path).parent.parent / Path(path).stem).replace("\\", "/")
+                    rels_content_map[o_path] = res
+
+        # 3. Assemblage final dans le processus parent
+        epc = Epc(energyml_objects=EnergymlObjectCollection(obj_list), raw_files=raw_files, core_props=core_props)
+
+        if read_rels_from_files:
+            for obj_path, rels_obj in rels_content_map.items():
+                if obj_path in path_to_obj:
+                    target_obj = path_to_obj[obj_path]
+                    epc._rels_cache.set_rels_from_file(target_obj, rels_obj)  # type: ignore
+
+        if recompute_rels:
+            epc._rels_cache.recompute_cache()  # type: ignore
+
+        return epc
+
 
 #     ______                                      __   ____                 __  _
 #    / ____/___  ___  _________ ___  ______ ___  / /  / __/_  ______  _____/ /_(_)___  ____  _____

energyml-utils/src/energyml/utils/introspection.py

@@ -909,7 +909,8 @@ def search_attribute_matching_type_with_path(
     elif not is_primitive(obj):
         for att_name in get_class_attributes(obj):
             res = res + search_attribute_matching_type_with_path(
-                obj=get_object_attribute_rgx(obj, att_name),
+                obj=get_object_attribute_no_verif(obj, att_name),
+                # obj=get_object_attribute_rgx(obj, att_name),
                 type_rgx=type_rgx,
                 re_flags=re_flags,
                 return_self=True,