Dev 03 25 (#11)

* Bugfix in introspection.py.

Author: valentin-gauthier-geosiris

energyml-utils/.flake8

@@ -0,0 +1,16 @@
+[flake8]
+# Ignore specific error codes (comma-separated list)
+ignore = E501, E722 #, W503, F403
+
+# Max line length (default is 79, can be changed)
+max-line-length = 120
+
+# Set the allowed complexity for functions (default is 10)
+complexity = 15
+
+# Exclude certain files and directories
+exclude = 
+    .git,
+    __pycache__,
+    old_version_files,
+    migrations

energyml-utils/.gitignore

@@ -53,4 +53,8 @@ manip*
 *.zip
 
 *.xml
-*.json
+*.json
+
+
+# WIP
+src/energyml/utils/wip*

energyml-utils/example/main.py

@@ -104,8 +104,8 @@ def tests_0():
     print("==>", get_object_attribute(adict, "a"))
     print("==>", get_object_attribute(tr, "citation.title"))
 
-    print(re.split(r"(?<!\\)\.+", "[Cc]itation.[Tt]it\.*"))
-    print("==>", get_object_attribute_rgx(fi, "[Cc]itation.[Tt]it\.*"))
+    print(re.split(r"(?<!\\)\.+", r"[Cc]itation.[Tt]it\.*"))
+    print("==>", get_object_attribute_rgx(fi, r"[Cc]itation.[Tt]it\.*"))
 
     # print("==>", type(cit), type(Citation))
     # print("==>", type(cit) == type, type(Citation) == type)
@@ -458,6 +458,12 @@ def class_field():
     # print(create_external_part_reference("2.0", "myfile.h5"))
 
 
+def test_dor_conversion():
+
+    print(serialize_json(dor_correct))
+    print(serialize_json(as_dor(dor_correct, "eml20.DataObjectReference")))
+
+
 if __name__ == "__main__":
     # tests_0()
     # tests_content_type()
@@ -476,4 +482,5 @@ def class_field():
     # test_copy_values()
     # class_field()
     # test_get_projected_uom()
-    test_crs()
+    # test_crs()
+    test_dor_conversion()

energyml-utils/example/tools.py

@@ -266,6 +266,7 @@ def generate_data():
         default="energyml.resqml.v2_2.resqmlv2.TriangulatedSetRepresentation",
         help="Object type (e.g. energyml.resqml.v2_2.resqmlv2.TriangulatedSetRepresentation)",
     )
+
     parser.add_argument(
         "--file-format",
         "-ff",

energyml-utils/pyproject.toml

@@ -57,7 +57,7 @@ h5py = { version = "^3.7.0", optional = false }
 pyarrow = { version = "^14.0.1", optional = false }
 numpy = { version = "^1.16.6", optional = false }
 
-[tool.poetry.dev-dependencies]
+[poetry.group.dev.dependencies]
 pandas = { version = "^1.1.0", optional = false }
 coverage = {extras = ["toml"], version = "^6.2"}
 pytest = "^8.1.1"

energyml-utils/src/energyml/utils/constants.py

@@ -72,37 +72,29 @@
 RGX_CT_TOKEN_TYPE = r"type=(?P<type>[\w\_]+)"
 
 RGX_CONTENT_TYPE = (
-        RGX_MIME_TYPE_MEDIA
-        + "/"
-        + "(?P<rawDomain>("
-        + RGX_CT_ENERGYML_DOMAIN
-        + ")|("
-        + RGX_CT_XML_DOMAIN
-        + r")|([\w-]+\.?)+)"
-        + "(;(("
-        + RGX_CT_TOKEN_VERSION
-        + ")|("
-        + RGX_CT_TOKEN_TYPE
-        + ")))*"
-)
-RGX_QUALIFIED_TYPE = (
-        r"(?P<domain>[a-zA-Z]+)" + RGX_DOMAIN_VERSION_FLAT + r"\.(?P<type>[\w_]+)"
+    RGX_MIME_TYPE_MEDIA
+    + "/"
+    + "(?P<rawDomain>("
+    + RGX_CT_ENERGYML_DOMAIN
+    + ")|("
+    + RGX_CT_XML_DOMAIN
+    + r")|([\w-]+\.?)+)"
+    + "(;(("
+    + RGX_CT_TOKEN_VERSION
+    + ")|("
+    + RGX_CT_TOKEN_TYPE
+    + ")))*"
 )
+RGX_QUALIFIED_TYPE = r"(?P<domain>[a-zA-Z]+)" + RGX_DOMAIN_VERSION_FLAT + r"\.(?P<type>[\w_]+)"
 # =========
 
 RGX_SCHEMA_VERSION = (
-        r"(?P<name>[eE]ml|[cC]ommon|[rR]esqml|[wW]itsml|[pP]rodml|[oO]pc)?\s*v?"
-        + RGX_DOMAIN_VERSION
-        + r"\s*$"
+    r"(?P<name>[eE]ml|[cC]ommon|[rR]esqml|[wW]itsml|[pP]rodml|[oO]pc)?\s*v?" + RGX_DOMAIN_VERSION + r"\s*$"
 )
 
 RGX_ENERGYML_FILE_NAME_OLD = r"(?P<type>[\w]+)_" + RGX_UUID_NO_GRP + r"\.xml$"
-RGX_ENERGYML_FILE_NAME_NEW = (
-        RGX_UUID_NO_GRP + r"\.(?P<objectVersion>\d+(\.\d+)*)\.xml$"
-)
-RGX_ENERGYML_FILE_NAME = (
-    rf"^(.*/)?({RGX_ENERGYML_FILE_NAME_OLD})|({RGX_ENERGYML_FILE_NAME_NEW})"
-)
+RGX_ENERGYML_FILE_NAME_NEW = RGX_UUID_NO_GRP + r"\.(?P<objectVersion>\d+(\.\d+)*)\.xml$"
+RGX_ENERGYML_FILE_NAME = rf"^(.*/)?({RGX_ENERGYML_FILE_NAME_OLD})|({RGX_ENERGYML_FILE_NAME_NEW})"
 
 RGX_XML_HEADER = r"^\s*<\?xml(\s+(encoding\s*=\s*\"(?P<encoding>[^\"]+)\"|version\s*=\s*\"(?P<version>[^\"]+)\"|standalone\s*=\s*\"(?P<standalone>[^\"]+)\"))+"  # pylint: disable=C0301
 
@@ -128,47 +120,43 @@
 URI_RGX_GRP_QUERY = "query"
 
 # Patterns
-_URI_RGX_PKG_NAME = "|".join(
-    ENERGYML_NAMESPACES.keys()
-)  # "[a-zA-Z]+\w+" //witsml|resqml|prodml|eml
+_URI_RGX_PKG_NAME = "|".join(ENERGYML_NAMESPACES.keys())  # "[a-zA-Z]+\w+" //witsml|resqml|prodml|eml
 URI_RGX = (
-        r"^eml:\/\/\/(?:dataspace\('(?P<"
-        + URI_RGX_GRP_DATASPACE
-        + r">[^']*?(?:''[^']*?)*)'\)\/?)?((?P<"
-        + URI_RGX_GRP_DOMAIN
-        + r">"
-        + _URI_RGX_PKG_NAME
-        + r")(?P<"
-        + URI_RGX_GRP_DOMAIN_VERSION
-        + r">[1-9]\d)\.(?P<"
-        + URI_RGX_GRP_OBJECT_TYPE
-        + r">\w+)(\((?:(?P<"
-        + URI_RGX_GRP_UUID
-        + r">(uuid=)?"
-        + RGX_UUID_NO_GRP
-        + r")|uuid=(?P<"
-        + URI_RGX_GRP_UUID2
-        + r">"
-        + RGX_UUID_NO_GRP
-        + r"),\s*version='(?P<"
-        + URI_RGX_GRP_VERSION
-        + r">[^']*?(?:''[^']*?)*)')\))?)?(\/(?P<"
-        + URI_RGX_GRP_COLLECTION_DOMAIN
-        + r">"
-        + _URI_RGX_PKG_NAME
-        + r")(?P<"
-        + URI_RGX_GRP_COLLECTION_DOMAIN_VERSION
-        + r">[1-9]\d)\.(?P<"
-        + URI_RGX_GRP_COLLECTION_TYPE
-        + r">\w+))?(?:\?(?P<"
-        + URI_RGX_GRP_QUERY
-        + r">[^#]+))?$"
+    r"^eml:\/\/\/(?:dataspace\('(?P<"
+    + URI_RGX_GRP_DATASPACE
+    + r">[^']*?(?:''[^']*?)*)'\)\/?)?((?P<"
+    + URI_RGX_GRP_DOMAIN
+    + r">"
+    + _URI_RGX_PKG_NAME
+    + r")(?P<"
+    + URI_RGX_GRP_DOMAIN_VERSION
+    + r">[1-9]\d)\.(?P<"
+    + URI_RGX_GRP_OBJECT_TYPE
+    + r">\w+)(\((?:(?P<"
+    + URI_RGX_GRP_UUID
+    + r">(uuid=)?"
+    + RGX_UUID_NO_GRP
+    + r")|uuid=(?P<"
+    + URI_RGX_GRP_UUID2
+    + r">"
+    + RGX_UUID_NO_GRP
+    + r"),\s*version='(?P<"
+    + URI_RGX_GRP_VERSION
+    + r">[^']*?(?:''[^']*?)*)')\))?)?(\/(?P<"
+    + URI_RGX_GRP_COLLECTION_DOMAIN
+    + r">"
+    + _URI_RGX_PKG_NAME
+    + r")(?P<"
+    + URI_RGX_GRP_COLLECTION_DOMAIN_VERSION
+    + r">[1-9]\d)\.(?P<"
+    + URI_RGX_GRP_COLLECTION_TYPE
+    + r">\w+))?(?:\?(?P<"
+    + URI_RGX_GRP_QUERY
+    + r">[^#]+))?$"
 )
 
 # ================================
-RELS_CONTENT_TYPE = (
-    "application/vnd.openxmlformats-package.core-properties+xml"
-)
+RELS_CONTENT_TYPE = "application/vnd.openxmlformats-package.core-properties+xml"
 RELS_FOLDER_NAME = "_rels"
 
 primitives = (bool, str, int, float, type(None))
@@ -179,6 +167,7 @@
 
 class MimeType(Enum):
     """Some mime types"""
+
     CSV = "text/csv"
     HDF5 = "application/x-hdf5"
     PARQUET = "application/x-parquet"
@@ -199,7 +188,7 @@ class EpcExportVersion(Enum):
 
 
 class EPCRelsRelationshipType(Enum):
-    """ Rels relationship types """
+    """Rels relationship types"""
 
     #: The object in Target is the destination of the relationship.
     DESTINATION_OBJECT = "destinationObject"
@@ -244,10 +233,12 @@ def get_type(self) -> str:
 
 @dataclass
 class RawFile:
-    """ A class for a non energyml file to be stored in an EPC file """
+    """A class for a non energyml file to be stored in an EPC file"""
+
     path: str = field(default="_")
     content: BytesIO = field(default=None)
 
+
 #     ______                 __  _
 #    / ____/_  ______  _____/ /_(_)___  ____  _____
 #   / /_  / / / / __ \/ ___/ __/ / __ \/ __ \/ ___/
@@ -335,7 +326,10 @@ def get_domain_version_from_content_or_qualified_type(cqt: str) -> Optional[str]
 
 def split_identifier(identifier: str) -> Tuple[str, Optional[str]]:
     match = re.match(RGX_IDENTIFIER, identifier)
-    return (match.group(URI_RGX_GRP_UUID), match.group(URI_RGX_GRP_VERSION), )
+    return (
+        match.group(URI_RGX_GRP_UUID),
+        match.group(URI_RGX_GRP_VERSION),
+    )
 
 
 def now(time_zone=datetime.timezone.utc) -> float:
@@ -356,12 +350,10 @@ def date_to_epoch(date: str) -> int:
 
 
 def epoch_to_date(
-        epoch_value: int,
+    epoch_value: int,
 ) -> str:
     date = datetime.datetime.fromtimestamp(epoch_value, datetime.timezone.utc)
-    return date.astimezone(datetime.timezone.utc).strftime(
-        "%Y-%m-%dT%H:%M:%SZ"
-    )
+    return date.astimezone(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
     # date = datetime.datetime.fromtimestamp(epoch_value, datetime.timezone.utc)
     # return date.astimezone(datetime.timezone(datetime.timedelta(hours=0), "UTC")).strftime('%Y-%m-%dT%H:%M:%SZ')
     # return date.strftime("%Y-%m-%dT%H:%M:%SZ%z")
@@ -379,9 +371,9 @@ def mime_type_to_file_extension(mime_type: str) -> Optional[str]:
     if mime_type is not None:
         mime_type_lw = mime_type.lower()
         if (
-                mime_type_lw == "application/x-parquet"
-                or mime_type_lw == "application/parquet"
-                or mime_type_lw == "application/vnd.apache.parquet"
+            mime_type_lw == "application/x-parquet"
+            or mime_type_lw == "application/parquet"
+            or mime_type_lw == "application/vnd.apache.parquet"
         ):
             return "parquet"
         elif mime_type_lw == "application/x-hdf5":
@@ -419,7 +411,10 @@ def _get_property_kind_dict_path_as_str(file_type: str = "xml") -> str:
     try:
         import energyml.utils.rc as RC
     except:
-        import src.energyml.utils.rc as RC
+        try:
+            import src.energyml.utils.rc as RC
+        except:
+            import utils.rc as RC
     return files(RC).joinpath(f"PropertyKindDictionary_v2.3.{file_type.lower()}").read_text(encoding="utf-8")
 
 
@@ -452,3 +447,5 @@ def get_property_kind_dict_path_as_xml() -> str:
     print(path_iter(".Citation.Title.Coucou"))
     print(path_iter(".Citation.Ti\\.*.Coucou"))
 
+    print(URI_RGX)
+    print(RGX_UUID_NO_GRP)

energyml-utils/src/energyml/utils/data/datasets_io.py

@@ -93,6 +93,7 @@ def write_array(
         ):
             if isinstance(array, list):
                 array = np.asarray(array)
+            print("writing array", target)
             with h5py.File(target, "a") as f:
                 # print(array.dtype, h5py.string_dtype(), array.dtype == 'O')
                 # print("\t", dtype or (h5py.string_dtype() if array.dtype == '0' else array.dtype))
@@ -175,9 +176,7 @@ def write_array(
 
     @dataclass
     class ParquetFileReader:
-        def read_array(
-            self, source: Union[BytesIO, str], path_in_external_file: Optional[str] = None
-        ) -> None:
+        def read_array(self, source: Union[BytesIO, str], path_in_external_file: Optional[str] = None) -> None:
             raise MissingExtraInstallation(extra_name="parquet")
 
         def get_array_dimension(self, source: Union[BytesIO, str], path_in_external_file: str) -> Optional[List[Any]]:
@@ -254,7 +253,7 @@ def read_array(
                 logging.debug("cst", _cst)
 
                 max_line_number = 0
-                for (_, n, _) in items:
+                for _, n, _ in items:
                     if n > max_line_number:
                         max_line_number = n