dev sep 25 (#15)

* Bug fix : 
- Property dictionnary access
- DOR reading with epc.as_dor() function
- set_attribute_from_path: take care of list parent

* New :
- Epc/Object validations have been improved. 
- New function to ease upload of data arrays to etp server (to get the proxy uri or the uri of the object itself) : energyml.utils.data.datasets_io.get_proxy_uri_for_path_in_external(...)
- Regex optimisation by using precompiled ones
- New class for huge file : EpcStreamReader

Author: valentin-gauthier-geosiris

energyml-utils/.gitignore

@@ -57,4 +57,5 @@ manip*
 
 
 # WIP
-src/energyml/utils/wip*
+src/energyml/utils/wip*
+scripts

energyml-utils/.pre-commit-config.yaml

@@ -0,0 +1,14 @@
+# .pre-commit-config.yaml
+repos:
+  - repo: https://github.com/psf/black
+    rev: 23.1.0
+    hooks:
+      - id: black
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.0.0
+    hooks:
+      - id: flake8

energyml-utils/README.md

@@ -76,6 +76,144 @@ energyml-prodml2-2 = "^1.12.0"
   - The "EnergymlWorkspace" class allows to abstract the access of numerical data like "ExternalArrays". This class can thus be extended to interact with ETP "GetDataArray" request etc...
 - ETP URI support : the "Uri" class allows to parse/write an etp uri.
 
+## EPC Stream Reader
+
+The **EpcStreamReader** provides memory-efficient handling of large EPC files through lazy loading and smart caching. Unlike the standard `Epc` class which loads all objects into memory, the stream reader loads objects on-demand, making it ideal for handling very large EPC files with thousands of objects.
+
+### Key Features
+
+- **Lazy Loading**: Objects are loaded only when accessed, reducing memory footprint
+- **Smart Caching**: LRU (Least Recently Used) cache with configurable size  
+- **Automatic EPC Version Detection**: Supports both CLASSIC and EXPANDED EPC formats
+- **Add/Remove/Update Operations**: Full CRUD operations with automatic file structure maintenance
+- **Context Management**: Automatic resource cleanup with `with` statements
+- **Memory Monitoring**: Track cache efficiency and memory usage statistics
+
+### Basic Usage
+
+```python
+from energyml.utils.epc_stream import EpcStreamReader
+
+# Open EPC file with context manager (recommended)
+with EpcStreamReader('large_file.epc', cache_size=50) as reader:
+    # List all objects without loading them
+    print(f"Total objects: {reader.stats.total_objects}")
+    
+    # Get object by identifier
+    obj: Any = reader.get_object_by_identifier("uuid.version")
+    
+    # Get objects by type
+    features: List[Any] = reader.get_objects_by_type("BoundaryFeature")
+    
+    # Get all objects with same UUID
+    versions: List[Any] = reader.get_object_by_uuid("12345678-1234-1234-1234-123456789abc")
+```
+
+### Adding Objects
+
+```python
+from energyml.utils.epc_stream import EpcStreamReader
+from energyml.utils.constants import gen_uuid
+import energyml.resqml.v2_2.resqmlv2 as resqml
+import energyml.eml.v2_3.commonv2 as eml
+
+# Create a new EnergyML object
+boundary_feature = resqml.BoundaryFeature()
+boundary_feature.uuid = gen_uuid()
+boundary_feature.citation = eml.Citation(title="My Feature")
+
+with EpcStreamReader('my_file.epc') as reader:
+    # Add object - path is automatically generated based on EPC version
+    identifier = reader.add_object(boundary_feature)
+    print(f"Added object with identifier: {identifier}")
+    
+    # Or specify custom path (optional)
+    identifier = reader.add_object(boundary_feature, "custom/path/MyFeature.xml")
+```
+
+### Removing Objects
+
+```python
+with EpcStreamReader('my_file.epc') as reader:
+    # Remove specific version by full identifier
+    success = reader.remove_object("uuid.version")
+    
+    # Remove ALL versions by UUID only
+    success = reader.remove_object("12345678-1234-1234-1234-123456789abc")
+    
+    if success:
+        print("Object(s) removed successfully")
+```
+
+### Updating Objects
+
+```python
+...
+from energyml.utils.introspection import set_attribute_from_path
+
+with EpcStreamReader('my_file.epc') as reader:
+    # Get existing object
+    obj = reader.get_object_by_identifier("uuid.version")
+    
+    # Modify the object
+    set_attribute_from_path(obj, "citation.title", "Updated Title")
+    
+    # Update in EPC file
+    new_identifier = reader.update_object(obj)
+    print(f"Updated object: {new_identifier}")
+```
+
+### Performance Monitoring
+
+```python
+with EpcStreamReader('large_file.epc', cache_size=100) as reader:
+    # Access some objects...
+    for i in range(10):
+        obj = reader.get_object_by_identifier(f"uuid-{i}.1")
+    
+    # Check performance statistics
+    print(f"Cache hit rate: {reader.stats.cache_hit_rate:.1f}%")
+    print(f"Memory efficiency: {reader.stats.memory_efficiency:.1f}%") 
+    print(f"Objects in cache: {reader.stats.loaded_objects}/{reader.stats.total_objects}")
+```
+
+### EPC Version Support
+
+The EpcStreamReader automatically detects and handles both EPC packaging formats:
+
+- **CLASSIC Format**: Flat file structure (e.g., `obj_BoundaryFeature_{uuid}.xml`)
+- **EXPANDED Format**: Namespace structure (e.g., `namespace_resqml201/version_{id}/obj_BoundaryFeature_{uuid}.xml` or `namespace_resqml201/obj_BoundaryFeature_{uuid}.xml`)
+
+```python
+with EpcStreamReader('my_file.epc') as reader:
+    print(f"Detected EPC version: {reader.export_version}")
+    # Objects added will use the same format as the existing EPC file
+```
+
+### Advanced Usage
+
+```python
+# Initialize without preloading metadata for faster startup
+reader = EpcStreamReader('huge_file.epc', preload_metadata=False, cache_size=200)
+
+try:
+    # Manual metadata loading when needed
+    reader._load_metadata()
+    
+    # Get object dependencies
+    deps = reader.get_object_dependencies("uuid.version")
+    
+    # Batch processing with memory monitoring
+    for obj_type in ["BoundaryFeature", "PropertyKind"]:
+        objects = reader.get_objects_by_type(obj_type)
+        print(f"Processing {len(objects)} {obj_type} objects")
+        
+finally:
+    reader.close()  # Manual cleanup if not using context manager
+```
+
+The EpcStreamReader is perfect for applications that need to work with large EPC files efficiently, such as data processing pipelines, web applications, or analysis tools where memory usage is a concern.
+
 
 # Poetry scripts : 
 
@@ -95,25 +233,32 @@ energyml-prodml2-2 = "^1.12.0"
 poetry install
 ```
 
+if you fail to run a script, you may have to add "src" to your PYTHONPATH environment variable. For example, in powershell : 
+
+```powershell
+$env:PYTHONPATH="src"
+```
+
 
 ## Validation examples : 
 
 An epc file:
 ```bash
-poetry run validate --input "path/to/your/energyml/object.epc" *> output_logs.json
+poetry run validate --file "path/to/your/energyml/object.epc" *> output_logs.json
 ```
 
 An xml file:
 ```bash
-poetry run validate --input "path/to/your/energyml/object.xml" *> output_logs.json
+poetry run validate --file "path/to/your/energyml/object.xml" *> output_logs.json
 ```
 
 A json file:
 ```bash
-poetry run validate --input "path/to/your/energyml/object.json" *> output_logs.json
+poetry run validate --file "path/to/your/energyml/object.json" *> output_logs.json
 ```
 
 A folder containing Epc/xml/json files:
 ```bash
-poetry run validate --input "path/to/your/folder" *> output_logs.json
+poetry run validate --file "path/to/your/folder" *> output_logs.json
 ```
+

energyml-utils/example/tools.py

@@ -6,13 +6,13 @@
 import pathlib
 from typing import Optional, List, Dict, Any
 
-from src.energyml.utils.validation import validate_epc
+from energyml.utils.validation import validate_epc
 
-from src.energyml.utils.constants import get_property_kind_dict_path_as_xml
-from src.energyml.utils.data.datasets_io import CSVFileReader, HDF5FileWriter, ParquetFileWriter, DATFileReader
-from src.energyml.utils.data.mesh import MeshFileFormat, export_multiple_data, export_obj, read_mesh_object
-from src.energyml.utils.epc import Epc, gen_energyml_object_path
-from src.energyml.utils.introspection import (
+from energyml.utils.constants import get_property_kind_dict_path_as_xml
+from energyml.utils.data.datasets_io import CSVFileReader, HDF5FileWriter, ParquetFileWriter, DATFileReader
+from energyml.utils.data.mesh import MeshFileFormat, export_multiple_data, export_obj, read_mesh_object
+from energyml.utils.epc import Epc, gen_energyml_object_path
+from energyml.utils.introspection import (
     get_class_from_simple_name,
     get_module_name_and_type_from_content_or_qualified_type,
     random_value_from_class,
@@ -27,7 +27,7 @@
     get_class_from_qualified_type,
     get_object_attribute_or_create,
 )
-from src.energyml.utils.serialization import (
+from energyml.utils.serialization import (
     serialize_json,
     JSON_VERSION,
     serialize_xml,

energyml-utils/pyproject.toml

@@ -46,8 +46,12 @@ include = [
 #    "src/energyml/main.py"
 #]
 
-#[tool.pytest.ini_options]
-#pythonpath = [ "src" ]
+[tool.pytest.ini_options]
+pythonpath = [ "src" ]
+testpaths = [ "tests" ]
+python_files = [ "test_*.py", "*_test.py" ]
+python_classes = [ "Test*" ]
+python_functions = [ "test_*" ]
 
 [tool.poetry.extras]
 parquet = ["pyarrow", "numpy", "pandas"]
@@ -61,7 +65,7 @@ h5py = { version = "^3.7.0", optional = false }
 pyarrow = { version = "^14.0.1", optional = false }
 numpy = { version = "^1.16.6", optional = false }
 
-[poetry.group.dev.dependencies]
+[tool.poetry.group.dev.dependencies]
 pandas = { version = "^1.1.0", optional = false }
 coverage = {extras = ["toml"], version = "^6.2"}
 pytest = "^8.1.1"
@@ -83,6 +87,12 @@ energyml-witsml2-1 = "^1.12.0"
 energyml-prodml2-0 = "^1.12.0"
 energyml-prodml2-2 = "^1.12.0"
 
+mypy = "^0.971"
+bandit = "^1.7.0"
+safety = "^1.10.0"
+memory-profiler = "^0.60.0"
+line-profiler = "^4.0.0"
+
 [tool.coverage.run]
 branch = true
 source = ["src/energyml"]

energyml-utils/rc/epc/testingPackageCpp.h5

@@ -1,2 +1,5 @@
 # Copyright (c) 2023-2024 Geosiris.
 # SPDX-License-Identifier: Apache-2.0
+
+# This is a namespace package
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)