starting optimisation for mesh reading

Author: valentin-gauthier-geosiris

energyml-utils/example/attic/arrays_test_fast.py

@@ -616,7 +616,7 @@ def read_external_dataset_array(
 ):
     if additional_sources is None:
         additional_sources = []
-    result_array = []
+    result_array = None
 
     for path_in_obj, path_in_external in get_path_in_external_with_path(energyml_array):
         succeed = False
@@ -630,10 +630,15 @@ def read_external_dataset_array(
         )
         for s in sources:
             try:
-                # TODO: take care of the "Counts" and "Starts" list in ExternalDataArrayPart to fill array correctly
-                result_array = result_array + read_dataset(
-                    source=s, path_in_external_file=path_in_external, mimetype=mimetype
-                )
+                if result_array is None:
+                    result_array = read_dataset(
+                        source=s, path_in_external_file=path_in_external, mimetype=mimetype
+                    )
+                else:
+                    # TODO: take care of the "Counts" and "Starts" list in ExternalDataArrayPart to fill array correctly
+                    result_array = result_array + read_dataset(
+                        source=s, path_in_external_file=path_in_external, mimetype=mimetype
+                    )
                 succeed = True
                 break  # stop after the first read success
             except MissingExtraInstallation as mei:
@@ -855,7 +860,7 @@ def open_file_no_cache(self, file_path: str, mode: str = "r") -> Optional[Any]:
             try:
                 return h5py.File(file_path, mode)  # type: ignore
             except Exception as e:
-                logging.error(f"Failed to open HDF5 file {file_path}: {e}")
+                logging.debug(f"Failed to open HDF5 file {file_path}: {e}")
                 return None
 
         def read_array(
@@ -880,6 +885,41 @@ def read_array(
                 with self.file_cache.get_or_open(source, self, "r") as f:  # type: ignore
                     return self.read_array(f, path_in_external_file, start_indices, counts)
 
+        def read_array_view(
+            self,
+            source: Union[BytesIO, str, Any],
+            path_in_external_file: Optional[str] = None,
+            start_indices: Optional[List[int]] = None,
+            counts: Optional[List[int]] = None,
+        ) -> Optional[np.ndarray]:
+            """Read array from HDF5 with best-effort zero-copy semantics.
+
+            For contiguous, uncompressed datasets the returned array is backed
+            by the memory-mapped file buffer (no copy).  For chunked or
+            compressed datasets h5py transparently falls back to a copy, but
+            sub-selection is done by h5py in C before the data reaches Python
+            (avoids loading the full dataset then slicing in Python).
+
+            The caller **must not mutate** the returned array.
+            """
+            if isinstance(source, h5py.File):  # type: ignore
+                if not path_in_external_file:
+                    return None
+                d_group = source[path_in_external_file]
+                if start_indices is not None and counts is not None:
+                    # h5py reads only the required chunks/slabs from disk
+                    slices = tuple(
+                        slice(start, start + count) for start, count in zip(start_indices, counts)
+                    )
+                    return d_group[slices]  # type: ignore
+                # np.array with copy=False returns a view for contiguous datasets
+                # Note: copy= kwarg on np.asarray requires numpy >=2.0;
+                # np.array(x, copy=False) works on all numpy versions.
+                return np.array(d_group, copy=False)  # type: ignore
+            else:
+                with self.file_cache.get_or_open(source, self, "r") as f:  # type: ignore
+                    return self.read_array_view(f, path_in_external_file, start_indices, counts)
+
         def write_array(
             self,
             target: Union[str, BytesIO, Any],

energyml-utils/src/energyml/utils/data/datasets_io.py

@@ -279,10 +279,15 @@ def apply_crs_transform(
     return transformed
 
 
-def get_crs_origin_offset(crs_obj: Any) -> List[float | int]:
+def get_crs_origin_offset(crs_obj: Any) -> np.ndarray:
     """
-    Return a list [X,Y,Z] corresponding to the crs Offset [XOffset/OriginProjectedCoordinate1, ... ] depending on the
-    crs energyml version.
+    Return a ``(3,) float64`` numpy array ``[X, Y, Z]`` corresponding to the
+    CRS origin offset (``XOffset``/``OriginProjectedCoordinate1``, …) depending
+    on the energyml version.
+
+    Returning an ndarray instead of a plain list avoids the ``np.asarray()``
+    call in callers such as :func:`mesh_numpy.crs_displacement_np`.
+
     :param crs_obj:
     :return:
     """
@@ -298,17 +303,18 @@ def get_crs_origin_offset(crs_obj: Any) -> List[float | int]:
     if tmp_offset_z is None:
         tmp_offset_z = get_object_attribute_rgx(crs_obj, "OriginProjectedCoordinate3")
 
-    crs_point_offset = [0.0, 0.0, 0.0]
     try:
-        crs_point_offset = [
-            float(tmp_offset_x) if tmp_offset_x is not None else 0.0,
-            float(tmp_offset_y) if tmp_offset_y is not None else 0.0,
-            float(tmp_offset_z) if tmp_offset_z is not None else 0.0,
-        ]
+        return np.array(
+            [
+                float(tmp_offset_x) if tmp_offset_x is not None else 0.0,
+                float(tmp_offset_y) if tmp_offset_y is not None else 0.0,
+                float(tmp_offset_z) if tmp_offset_z is not None else 0.0,
+            ],
+            dtype=np.float64,
+        )
     except Exception as e:
         logging.info(f"ERR reading crs offset {e}")
-
-    return crs_point_offset
+        return np.zeros(3, dtype=np.float64)
 
 
 def get_datum_information(
@@ -1037,28 +1043,33 @@ def read_constant_array(
     path_in_root: Optional[str] = None,
     workspace: Optional[EnergymlStorageInterface] = None,
     sub_indices: Optional[Union[List[int], np.ndarray]] = None,
-) -> List[Any]:
+) -> Union[np.ndarray, List[Any]]:
     """
-    Read a constant array ( BooleanConstantArray, DoubleConstantArray, FloatingPointConstantArray, IntegerConstantArray ...)
+    Read a constant array (BooleanConstantArray, DoubleConstantArray,
+    FloatingPointConstantArray, IntegerConstantArray …).
+
+    For numeric (int / float / bool) values a ``numpy.ndarray`` is returned
+    via :func:`numpy.full`, avoiding a Python-list allocation.  String values
+    fall back to a plain list because numpy object arrays add no benefit.
+
     :param energyml_array:
     :param root_obj:
     :param path_in_root:
     :param workspace:
     :param sub_indices:
     :return:
     """
-    # logging.debug(f"Reading constant array\n\t{energyml_array}")
-
     value = get_object_attribute_no_verif(energyml_array, "value")
     count = (
         len(sub_indices)
         if sub_indices is not None and len(sub_indices) > 0
         else get_object_attribute_no_verif(energyml_array, "count")
     )
 
-    # logging.debug(f"\tValue : {[value for i in range(0, count)]}")
-
-    return [value] * count
+    if isinstance(value, (int, float, bool, np.integer, np.floating)):
+        return np.full(int(count), value)
+    # Non-numeric (e.g. string) — keep as Python list.
+    return [value] * int(count)
 
 
 def read_xml_array(
@@ -1402,44 +1413,45 @@ def read_point3d_lattice_array(
                 # Add slowest offsets where i > 0
                 result_arr[1:, :, :] += slowest_cumsum[:-1, np.newaxis, :]
 
-            # Flatten to list of points
-            result = result_arr.reshape(-1, 3).tolist()
+            # Return the (N, 3) float64 numpy array directly — no .tolist().
+            result = result_arr.reshape(-1, 3)
 
         except (ValueError, TypeError) as e:
-            # Fallback to original implementation if NumPy conversion fails
+            # Fallback to original implementation if NumPy conversion fails.
             logging.warning(f"NumPy vectorization failed ({e}), falling back to iterative approach")
+            fallback: List = []
             for i in range(slowest_size):
                 for j in range(fastest_size):
                     previous_value = origin
 
                     if j > 0:
                         if i > 0:
                             line_idx = i * fastest_size
-                            previous_value = result[line_idx + j - 1]
+                            previous_value = fallback[line_idx + j - 1]
                         else:
-                            previous_value = result[j - 1]
+                            previous_value = fallback[j - 1]
                         if zincreasing_downward:
-                            result.append(sum_lists(previous_value, slowest_table[i - 1]))
+                            fallback.append(sum_lists(previous_value, slowest_table[i - 1]))
                         else:
-                            result.append(sum_lists(previous_value, fastest_table[j - 1]))
+                            fallback.append(sum_lists(previous_value, fastest_table[j - 1]))
                     else:
                         if i > 0:
                             prev_line_idx = (i - 1) * fastest_size
-                            previous_value = result[prev_line_idx]
+                            previous_value = fallback[prev_line_idx]
                             if zincreasing_downward:
-                                result.append(sum_lists(previous_value, fastest_table[j - 1]))
+                                fallback.append(sum_lists(previous_value, fastest_table[j - 1]))
                             else:
-                                result.append(sum_lists(previous_value, slowest_table[i - 1]))
+                                fallback.append(sum_lists(previous_value, slowest_table[i - 1]))
                         else:
-                            result.append(previous_value)
+                            fallback.append(previous_value)
+            # Convert fallback list to ndarray to keep the return type consistent.
+            result = np.array(fallback, dtype=np.float64).reshape(-1, 3)
     else:
         raise Exception(f"{type(energyml_array)} read with an offset of length {len(offset)} is not supported")
 
     if sub_indices is not None and len(sub_indices) > 0:
-        if isinstance(result, np.ndarray):
-            result = result[sub_indices].tolist()
-        else:
-            result = [result[idx] for idx in sub_indices]
+        # result is always an ndarray here; index directly without .tolist().
+        result = result[np.asarray(sub_indices, dtype=np.int64)]
 
     return result
 

energyml-utils/src/energyml/utils/data/helper.py

@@ -172,7 +172,7 @@ def crs_displacement(points: List[Point], crs_obj: Any) -> Tuple[List[Point], Po
     crs_point_offset = get_crs_origin_offset(crs_obj=crs_obj)
     zincreasing_downward = is_z_reversed(crs_obj)
 
-    if crs_point_offset != [0, 0, 0]:
+    if np.any(crs_point_offset):
         for p in points:
             for xyz in range(len(p)):
                 p[xyz] = (p[xyz] + crs_point_offset[xyz]) if p[xyz] is not None else None
@@ -241,7 +241,7 @@ def read_mesh_object(
         ):  # WellboreFrameRep has allready the displacement applied
             # TODO: the displacement should be done in each reader function to manage specific cases
             for s in surfaces:
-                logging.debug(f"CRS : {s.crs_object}")
+                # logging.debug(f"CRS : {s.crs_object}")
                 crs_displacement(
                     s.point_list,
                     s.crs_object[0] if isinstance(s.crs_object, list) and len(s.crs_object) > 0 else s.crs_object,