support to read properties, columnBasedTable, and timeSeries as arrays/dict

Author: valentin-gauthier-geosiris

energyml-utils/example/attic/arrays_test.py

@@ -1,12 +1,26 @@
 import logging
+from sqlite3 import NotSupportedError
+import traceback
 from typing import List, Optional
 import numpy as np
 from energyml.utils.data.helper import _ARRAY_NAMES_, read_array
-from energyml.utils.data.mesh import AbstractMesh, SurfaceMesh, PolylineSetMesh, read_mesh_object
+from energyml.utils.data.mesh import (
+    AbstractMesh,
+    SurfaceMesh,
+    PolylineSetMesh,
+    read_column_based_table,
+    read_mesh_object,
+    read_property_interpreted_with_cbt,
+    read_property,
+    read_time_series,
+)
 from energyml.utils.storage_interface import EnergymlStorageInterface
 from energyml.utils.epc import Epc
-
+from energyml.utils.epc_stream import EpcStreamReader, RelsUpdateMode
+from energyml.utils.introspection import get_obj_title
 from energyml.resqml.v2_2.resqmlv2 import Point3DLatticeArray
+from energyml.eml.v2_3.commonv2 import TimeSeries
+from energyml.eml.v2_1.commonv2 import TimeSeries as TimeSeries21
 
 from energyml.utils.serialization import read_energyml_xml_str, serialize_json
 
@@ -232,27 +246,107 @@ def read_representation_set_representation() -> List[AbstractMesh]:
     return read_mesh_object(energyml_object=rep_set_rep, workspace=epc)
 
 
+def read_props_and_cbt(
+    epc_path: List[str] = [
+        "rc/epc/testingPackageCpp22.epc",
+        "D:/Geosiris/Clients/BRGM/git/csv-to-energyml/rc/output/full-local/attic/result-out-EpcStream-egis-full.epc",
+    ],
+    p_or_cbt_uuids: List = [
+        "1c5a3e99-e997-4bd7-a94d-c45d7b7405ce",
+        "be17c053-9189-4bc0-9db1-75aa51a026cd",
+        "da73937c-2c60-4e10-8917-5154fde4ded5",
+        "6561b499-82ed-4233-8a83-ea5d5aaf56a9",
+        "0d6aba60-b37e-498c-aedc-334561eb0749",
+        "d64d0ed0-72fa-4495-8e3a-a01175194e25",
+        "5abecfe6-b951-4802-9002-e597169a9923",
+        "49207072-563b-404a-9707-9a9b70168d33",
+    ],
+) -> None:
+
+    epcs = []
+    for path in epc_path:
+        epc = EpcStreamReader(
+            epc_file_path=path,
+            rels_update_mode=RelsUpdateMode.MANUAL,
+        )
+        # epc = Epc.read_file(f"{path}", read_rels_from_files=False, recompute_rels=False)
+        epcs.append(epc)
+
+    for uuid in p_or_cbt_uuids:
+        read = False
+        prop_or_cbt = None
+        for epc in epcs:
+            try:
+                prop_or_cbt_lst = epc.get_object_by_uuid(uuid)
+                if not prop_or_cbt_lst:
+                    continue
+                prop_or_cbt = prop_or_cbt_lst[0]
+                array = None
+                reshaped_array = None
+                if "column" in str(type(prop_or_cbt)).lower():
+                    array = read_column_based_table(prop_or_cbt, workspace=epc)
+                elif "time" in str(type(prop_or_cbt)).lower():
+                    array = read_time_series(prop_or_cbt, workspace=epc)
+                else:
+                    array = read_property(
+                        prop_or_cbt,
+                        workspace=epc,
+                    )
+                    reshaped_array = read_property_interpreted_with_cbt(
+                        prop_or_cbt,
+                        workspace=epc,
+                        _cache_property_arrays=array,
+                        _return_none_if_no_category_lookup=True,
+                    )
+                print("=" * 40)
+                print(f"{type(prop_or_cbt)} : {get_obj_title(prop_or_cbt)} - uuid: {uuid}")
+                print(array)
+
+                if reshaped_array is not None:
+                    print(" # => interpreted array:")
+                    print(reshaped_array)
+
+                print("\n")
+                read = True
+                break
+            # except NotSupportedError as e:
+            #     print(f"Object with uuid {uuid} found but not supported: {e}")
+            except Exception as e:
+                traceback.print_exc()
+                print(f"Error reading object with uuid {uuid}: {e}")
+                pass
+        if not read:
+            print("[E]" + "=" * 40)
+            if prop_or_cbt is not None:
+                print(f"Object with uuid {get_obj_title(prop_or_cbt)} found but could not be read.")
+            else:
+                print(f"Object with uuid {uuid} not found in any EPC file.")
+            print("\n")
+
+
 if __name__ == "__main__":
     logging.basicConfig(level=logging.DEBUG)
 
     # meshes = read_grid()
     # meshes = read_polyline()
     # meshes = read_wellbore_frame_repr()
-    meshes = read_representation_set_representation()
-
-    for m in meshes:
-        print("=" * 40)
-        print(f"Mesh identifier: {m.identifier}")
-        print("points:")
-        print(np.array(m.point_list))
-
-        if isinstance(m, SurfaceMesh):
-            print("face indices:")
-            print(np.array(m.faces_indices))
-        elif isinstance(m, PolylineSetMesh):
-            print("line indices:")
-            try:
-                print(np.array(m.line_indices))
-            except Exception as e:
-                print(m.line_indices)
-                raise e
+    # meshes = read_representation_set_representation()
+
+    # for m in meshes:
+    #     print("=" * 40)
+    #     print(f"Mesh identifier: {m.identifier}")
+    #     print("points:")
+    #     print(np.array(m.point_list))
+
+    #     if isinstance(m, SurfaceMesh):
+    #         print("face indices:")
+    #         print(np.array(m.faces_indices))
+    #     elif isinstance(m, PolylineSetMesh):
+    #         print("line indices:")
+    #         try:
+    #             print(np.array(m.line_indices))
+    #         except Exception as e:
+    #             print(m.line_indices)
+    #             raise e
+
+    read_props_and_cbt()

energyml-utils/src/energyml/utils/data/helper.py

@@ -23,6 +23,8 @@
     get_object_attribute,
     get_object_attribute_rgx,
     get_object_attribute_advanced,
+    is_primitive,
+    get_obj_title,
 )
 
 from .datasets_io import get_path_in_external_with_path
@@ -382,18 +384,18 @@ def prod_n_tab(val: Union[float, int, str], tab: List[Union[float, int, str]]):
     """
     if val is None:
         return [None] * len(tab)
-    logging.debug(f"Multiplying list by {val}: {tab}")
+    # logging.debug(f"Multiplying list by {val}: {tab}")
     # Convert to numpy array for vectorized operations, handling None values
     arr = np.array(tab, dtype=object)
-    logging.debug(f"arr: {arr}")
+    # logging.debug(f"arr: {arr}")
     # Create mask for non-None values
     mask = arr != None  # noqa: E711
     # Create result array filled with None
     result = np.full(len(tab), None, dtype=object)
-    logging.debug(f"result before multiplication: {result}")
+    # logging.debug(f"result before multiplication: {result}")
     # Multiply only non-None values
     result[mask] = arr[mask].astype(float) * val
-    logging.debug(f"result after multiplication: {result}")
+    # logging.debug(f"result after multiplication: {result}")
     return result.tolist()
 
 
@@ -453,10 +455,12 @@ def get_crs_obj(
         logging.error("@get_crs_obj no Epc file given")
     else:
         crs_list = search_attribute_matching_name(context_obj, r"\.*Crs", search_in_sub_obj=True, deep_search=False)
-        if crs_list is not None and len(crs_list) > 0:
+        if crs_list is not None and len(crs_list) > 0 and crs_list[0] is not None:
             # logging.debug(crs_list[0])
+            # logging.debug(f"CRS found for {get_obj_title(context_obj)} : {crs_list[0]}")
             crs = workspace.get_object(get_obj_uri(crs_list[0]))
             if crs is None:
+                # logging.debug(f"CRS {crs_list[0]} not found (or not read correctly)")
                 crs = workspace.get_object_by_uuid(get_obj_uuid(crs_list[0]))
             if crs is None:
                 logging.error(f"CRS {crs_list[0]} not found (or not read correctly)")
@@ -755,17 +759,19 @@ def _array_name_mapping(array_type_name: str) -> str:
     :param array_type_name:
     :return:
     """
-    array_type_name = array_type_name.replace("3D", "3d").replace("2D", "2d")
-    if array_type_name.endswith("ConstantArray"):
+    array_type_name = array_type_name.replace("3D", "3d").replace("2D", "2d").lower()
+    # logging.debug(f"=============> Mapping array type name '{array_type_name}' to reader function name...")
+    if array_type_name.endswith("constantarray"):
         return "ConstantArray"
-    elif "External" in array_type_name or "Hdf5" in array_type_name:
+    elif "external" in array_type_name or "hdf5" in array_type_name:
         return "ExternalArray"
-    elif array_type_name.endswith("XmlArray"):
+    elif "xml" in array_type_name:
+        # logging.debug("=============> XML array detected, be careful with the performance !")
         return "XmlArray"
-    elif "Jagged" in array_type_name:
+    elif "jagged" in array_type_name:
         return "JaggedArray"
-    elif "Lattice" in array_type_name:
-        if "Integer" in array_type_name or "Double" in array_type_name or "FloatingPoint" in array_type_name:
+    elif "lattice" in array_type_name:
+        if "integer" in array_type_name or "double" in array_type_name or "floatingpoint" in array_type_name:
             return "int_double_lattice_array"
     return array_type_name
 
@@ -879,7 +885,7 @@ def read_external_array(
             for ext_part in external_parts:
                 start_indices, counts, external_uri = _extract_external_data_array_part_params(ext_part)
                 pief_list = get_path_in_external_with_path(obj=ext_part)
-
+                # logging.debug(f"Pief : {pief_list}")
                 for pief_path_in_obj, pief in pief_list:
                     arr = workspace.read_array(
                         proxy=crs or root_obj,
@@ -890,6 +896,7 @@ def read_external_array(
                     )
                     if arr is not None:
                         array = arr if array is None else np.concatenate((array, arr))
+                    # logging.debug(f"\t ExternalDataArrayPart read successfully. arr : {arr} : array : {array}")
         else:
             # RESQML v2.0.1: Extract count from parent object, no StartIndex or URI
             counts = None
@@ -933,6 +940,7 @@ def read_external_array(
             # Fallback for non-numpy arrays
             array = [array[idx] for idx in sub_indices]
 
+    # logging.debug(f"External array read successfully. => {array}")
     return array
 
 
@@ -964,8 +972,26 @@ def read_array(
     :param sub_indices: for SubRepresentation
     :return:
     """
-    if isinstance(energyml_array, list):
+    if isinstance(energyml_array, np.ndarray):
+        # if isinstance(energyml_array, list):
         return energyml_array
+    elif isinstance(energyml_array, list):
+        # logging.debug("Warning: the array is a list, not a numpy array, be careful with the performance !")
+        # logging.debug(energyml_array)
+        if len(energyml_array) > 0 and is_primitive(energyml_array[0]):
+            return energyml_array
+        else:
+            return [
+                read_array(
+                    energyml_array=elem,
+                    root_obj=root_obj,
+                    path_in_root=path_in_root,
+                    workspace=workspace,
+                    sub_indices=sub_indices,
+                )
+                for elem in energyml_array
+                if elem is not None
+            ]
     array_type_name = _array_name_mapping(type(energyml_array).__name__)
 
     reader_func = get_array_reader_function(array_type_name)
@@ -1030,8 +1056,10 @@ def read_xml_array(
     :param sub_indices:
     :return:
     """
+
     values = get_object_attribute_no_verif(energyml_array, "values")
     # count = get_object_attribute_no_verif(energyml_array, "count_per_value")
+    # logging.debug("values: ", values)
 
     if sub_indices is not None and len(sub_indices) > 0:
         if isinstance(values, np.ndarray):