Dev (#9)

* moving part to extras + adding poetry scripts

* jsonify validation errors

Author: valentin-gauthier-geosiris

.github/actions/prepare-poetry/action.yml

@@ -5,7 +5,7 @@
 ---
 
 name: Prepare Python and Poetry
-Description: Install Python, Poetry and dev dependencies, cached for speed
+description: Install Python, Poetry and dev dependencies, cached for speed
 
 inputs:
   python-version:  

README.md

@@ -0,0 +1,31 @@
+# energyml-utils
+
+
+This project contains multiple projects to use RESQML/WITSML/PRODML with python.
+
+## Supported versions:
+
+- Resqml 
+  - [2.0.1](https://github.com/geosiris-technologies/energyml-python/tree/main/energyml-resqml2-0-1) : **energyml-resqml2-0-1**
+  - [2.2dev3](https://github.com/geosiris-technologies/energyml-python/tree/main/energyml-resqml2-2-dev3) : **energyml-resqml2-2-dev3**
+  - [2.2](https://github.com/geosiris-technologies/energyml-python/tree/main/energyml-resqml2-0-1) : **energyml-resqml2-2**
+
+- Witsml 
+  - [2.0](https://github.com/geosiris-technologies/energyml-python/tree/main/energyml-witsml2-0) : **energyml-witsml2-0**
+  - [2.1](https://github.com/geosiris-technologies/energyml-python/tree/main/energyml-witsml2-1) : **energyml-witsml2-1**
+
+- Prodml 
+  - [2.0](https://github.com/geosiris-technologies/energyml-python/tree/main/energyml-prodml2-0) : **energyml-prodml2-0**
+  - [2.1](https://github.com/geosiris-technologies/energyml-python/tree/main/energyml-prodml2-2) : **energyml-prodml2-2**
+
+- Common (eml)
+  - [2.0](https://github.com/geosiris-technologies/energyml-python/tree/main/energyml-common2-0) : **energyml-common2-0**
+  - [2.1](https://github.com/geosiris-technologies/energyml-python/tree/main/energyml-common2-1) : **energyml-common2-1**
+  - [2.2](https://github.com/geosiris-technologies/energyml-python/tree/main/energyml-common2-2) : **energyml-common2-2**
+  - [2.3](https://github.com/geosiris-technologies/energyml-python/tree/main/energyml-common2-3) : **energyml-common2-3**
+
+
+## Utility package (EPC + h5 support):
+
+To ease the use of the energyml data, we provided a module called **energyml-utils**.
+For more information, check the sub project : https://github.com/geosiris-technologies/energyml-python/tree/main/energyml-utils.

energyml-utils/.gitignore

@@ -33,6 +33,8 @@ dask-worker-space
 
 # Example for local test
 example-local/
+wip
+src/energyml/utils/converter/
 # utils/
 
 # Other files
@@ -45,4 +47,10 @@ manip*
 *.off
 *.obj
 *.log
-*.geojson
+*.geojson
+*.json
+*.csv
+*.zip
+
+*.xml
+*.json

energyml-utils/README.md

@@ -76,3 +76,12 @@ energyml-prodml2-2 = "^1.12.0"
   - The "EnergymlWorkspace" class allows to abstract the access of numerical data like "ExternalArrays". This class can thus be extended to interact with ETP "GetDataArray" request etc...
 - ETP URI support : the "Uri" class allows to parse/write an etp uri.
 
+
+# Poetry scripts : 
+
+- extract_3d : extract a representation into an 3D file (obj/off)
+- csv_to_dataset : translate csv data into h5 dataset
+- generate_data : generate a random data from a qualified_type 
+- xml_to_json : translate an energyml xml file into json.
+- json_to_xml : translate an energyml json file into an xml file
+- describe_as_csv : create a csv description of an EPC content

energyml-utils/example/main.py

@@ -27,6 +27,7 @@
     correct_dor,
 )
 from src.energyml.utils.xml import *
+from utils.data.datasets_io import HDF5FileReader
 
 fi_cit = Citation(
     title="An interpretation",
@@ -130,11 +131,7 @@ def file_test():
     print(find_schema_version_in_element(get_tree(xml_content)))
     print(get_class_name_from_xml(get_tree(xml_content)))
 
-    print(
-        get_class_from_name(
-            "energyml.resqml.v2_2.resqmlv2.TriangulatedSetRepresentation"
-        )
-    )
+    print(get_class_from_name("energyml.resqml.v2_2.resqmlv2.TriangulatedSetRepresentation"))
 
     print(read_energyml_xml_str(xml_content))
     print(read_energyml_xml_file(path))
@@ -143,53 +140,27 @@ def file_test():
 def tests_content_type():
     print(RGX_CONTENT_TYPE)
 
-    print(
-        parse_content_type(
-            "application/x-resqml+xml;version=2.2;type=TriangulatedSetRepresentation"
-        )
-    )
-    print(
-        parse_content_type(
-            "application/vnd.openxmlformats-package.core-properties+xml"
-        ).group("domain")
-    )
+    print(parse_content_type("application/x-resqml+xml;version=2.2;type=TriangulatedSetRepresentation"))
+    print(parse_content_type("application/vnd.openxmlformats-package.core-properties+xml").group("domain"))
 
-    print(
-        get_class_from_content_type(
-            "application/x-resqml+xml;version=2.2;type=TriangulatedSetRepresentation"
-        )
-    )
+    print(get_class_from_content_type("application/x-resqml+xml;version=2.2;type=TriangulatedSetRepresentation"))
     print(
         "CT 201 : ",
-        get_class_from_content_type(
-            "application/x-resqml+xml;version=2.0;type=obj_HorizonInterpretation"
-        ),
-    )
-    print(
-        parse_content_type(
-            "application/x-resqml+xml;version=2.0;type=obj_HorizonInterpretation"
-        )
+        get_class_from_content_type("application/x-resqml+xml;version=2.0;type=obj_HorizonInterpretation"),
     )
+    print(parse_content_type("application/x-resqml+xml;version=2.0;type=obj_HorizonInterpretation"))
 
-    print(
-        get_class_from_content_type(
-            "application/vnd.openxmlformats-package.core-properties+xml"
-        )
-    )
+    print(get_class_from_content_type("application/vnd.openxmlformats-package.core-properties+xml"))
 
     print(get_content_type_from_class(tr))
     print(get_qualified_type_from_class(tr))
     print(
         get_qualified_type_from_class(DoubleHdf5Array()),
-        get_class_from_qualified_type(
-            get_qualified_type_from_class(DoubleHdf5Array())
-        ),
+        get_class_from_qualified_type(get_qualified_type_from_class(DoubleHdf5Array())),
     )
     print(
         get_qualified_type_from_class(dor_correct),
-        get_class_from_qualified_type(
-            get_qualified_type_from_class(dor_correct)
-        ),
+        get_class_from_qualified_type(get_qualified_type_from_class(dor_correct)),
     )
 
     print(gen_energyml_object_path(tr, EpcExportVersion.EXPANDED))
@@ -204,17 +175,11 @@ def tests_epc():
     print(epc)
     epc.export_file("D:/Geosiris/Github/energyml/energyml-python/test.epc")
     epc.export_version = EpcExportVersion.EXPANDED
-    epc.export_file(
-        "D:/Geosiris/Github/energyml/energyml-python/test_EXPANDED.epc"
-    )
+    epc.export_file("D:/Geosiris/Github/energyml/energyml-python/test_EXPANDED.epc")
     epc.core_props = None
-    epc.export_file(
-        "D:/Geosiris/Github/energyml/energyml-python/test_no_core.epc"
-    )
+    epc.export_file("D:/Geosiris/Github/energyml/energyml-python/test_no_core.epc")
 
-    epc201 = Epc.read_file(
-        "D:/Geosiris/OSDU/manifestTranslation/#Data/VOLVE_STRUCT.epc"
-    )
+    epc201 = Epc.read_file("D:/Geosiris/OSDU/manifestTranslation/#Data/VOLVE_STRUCT.epc")
     print(epc201)
 
     print(f"NB errors {len(validate_epc(epc201))}")
@@ -239,10 +204,7 @@ def tests_dor():
 
     print(
         json.dumps(
-            {
-                k: [get_obj_uuid(x) for x in v]
-                for k, v in get_reverse_dor_list(epc.energyml_objects).items()
-            },
+            {k: [get_obj_uuid(x) for x in v] for k, v in get_reverse_dor_list(epc.energyml_objects).items()},
             indent=4,
         )
     )
@@ -301,16 +263,8 @@ def test_introspection():
     print(search_attribute_matching_type(tr, "Citation"))
     print(search_attribute_matching_type(tr, "DataObjectreference"))
     print(search_attribute_matching_type_with_path(tr, "DataObjectreference"))
-    print(
-        class_match_rgx(
-            ContactElement, "DataObjectreference", super_class_search=False
-        )
-    )
-    print(
-        class_match_rgx(
-            ContactElement, "DataObjectreference", super_class_search=True
-        )
-    )
+    print(class_match_rgx(ContactElement, "DataObjectreference", super_class_search=False))
+    print(class_match_rgx(ContactElement, "DataObjectreference", super_class_search=True))
     print(Enum in ExistenceKind.__bases__)
     print(Enum in TriangulatedSetRepresentation.__bases__)
     print(is_primitive(int))
@@ -331,9 +285,7 @@ def test_introspection():
     print(TriangulatedSetRepresentation.__dataclass_params__)
 
     # print(random_value_from_class(int))
-    print(
-        serialize_xml(random_value_from_class(TriangulatedSetRepresentation))
-    )
+    print(serialize_xml(random_value_from_class(TriangulatedSetRepresentation)))
     # print(serialize_json(random_value_from_class(TriangulatedSetRepresentation)))
 
     print(search_attribute_matching_name_with_path(tr, "[tT]it.*"))
@@ -342,13 +294,7 @@ def test_introspection():
     print(AbstractPoint3DArray.__dict__)
     print(TriangulatedSetRepresentation.__dict__)
     print(get_sub_classes(AbstractObject))
-    print(
-        list(
-            filter(
-                lambda _c: not is_abstract(_c), get_sub_classes(AbstractObject)
-            )
-        )
-    )
+    print(list(filter(lambda _c: not is_abstract(_c), get_sub_classes(AbstractObject))))
     print(AbstractColorMap.__name__.startswith("Abstract"))
     print(is_abstract(AbstractColorMap))
 
@@ -366,17 +312,13 @@ def test_introspection():
 
     print(f"object: {is_abstract(object)}")
     print(f"HDF5FileReader: {is_abstract(HDF5FileReader)}")
-    print(
-        f"TriangulatedSetRepresentation: {is_abstract(TriangulatedSetRepresentation)}"
-    )
+    print(f"TriangulatedSetRepresentation: {is_abstract(TriangulatedSetRepresentation)}")
 
     # print("HDF5FileReader")
     # for func in dir(HDF5FileReader):
     #     if callable(getattr(HDF5FileReader, func)) and not func.startswith("__"):
     #         print(f"\t{func} {type(getattr(HDF5FileReader, func))}")
-    print(
-        get_classes_matching_name(TriangulatedSetRepresentation, "Abstract.*")
-    )
+    print(get_classes_matching_name(TriangulatedSetRepresentation, "Abstract.*"))
     # print(get_matching_class_attribute_name(ExternalDataArrayPart, "(PathInHdfFile|PathInExternalFile)"))
     # print(object.__module__)
     # print(serialize_xml(random_value_from_class(PointSetRepresentation)))
@@ -390,21 +332,15 @@ def test_introspection():
     # print(serialize_xml(poly))
 
     print("=====] ", r"ClosedPolylines.\d+")
-    for array_path, array_value in search_attribute_matching_name_with_path(
-        poly, r"ClosedPolylines.\d+"
-    ):
+    for array_path, array_value in search_attribute_matching_name_with_path(poly, r"ClosedPolylines.\d+"):
         print(f"{array_path}\n\t{array_value}")
 
     print("=====] ", r"ClosedPolylines.values.\d+")
-    for array_path, array_value in search_attribute_matching_name_with_path(
-        poly, r"ClosedPolylines.values.\d+"
-    ):
+    for array_path, array_value in search_attribute_matching_name_with_path(poly, r"ClosedPolylines.values.\d+"):
         print(f"{array_path}\n\t{array_value}")
 
     print("=====] ", r"LinePatch.\d+")
-    for array_path, array_value in search_attribute_matching_name_with_path(
-        poly, r"LinePatch.\d+"
-    ):
+    for array_path, array_value in search_attribute_matching_name_with_path(poly, r"LinePatch.\d+"):
         print(f"{array_path}\n\t{array_value}")
 
 
@@ -424,17 +360,15 @@ def tests_hdf():
 
     for o in tr_list:
         print(o.__class__)
-        print(get_hdf_reference_with_path(o))
+        print(get_path_in_external_with_path(o))
         exit(0)
 
 
 def test_local_depth_crs():
     # Fails because the xsi:type="VerticalCrsEpsgCode" doesn't
     # contain the namespace : xsi:type="eml:VerticalCrsEpsgCode"
     try:
-        depth3d = read_energyml_xml_file(
-            "../rc/obj_LocalDepth3dCrs_716f6472-18a3-4f19-a57c-d4f5642ccc53.xml"
-        )
+        depth3d = read_energyml_xml_file("../rc/obj_LocalDepth3dCrs_716f6472-18a3-4f19-a57c-d4f5642ccc53.xml")
         print(serialize_json(depth3d, JSON_VERSION.XSDATA))
         print(serialize_xml(depth3d))
     except Exception as e:
@@ -445,9 +379,7 @@ def test_get_projected_uom():
     # Fails because the xsi:type="VerticalCrsEpsgCode" doesn't
     # contain the namespace : xsi:type="eml:VerticalCrsEpsgCode"
     try:
-        depth3d = read_energyml_xml_file(
-            "../rc/obj_LocalDepth3dCrs_716f6472-18a3-4f19-a57c-d4f5642ccc53.xml"
-        )
+        depth3d = read_energyml_xml_file("../rc/obj_LocalDepth3dCrs_716f6472-18a3-4f19-a57c-d4f5642ccc53.xml")
         print(get_projected_uom(depth3d).value)
     except Exception as e:
         print(e)
@@ -502,28 +434,18 @@ def class_field():
     print(get_class_fields(tr)["citation"])
     print(get_class_pkg_version(tr))
     print(create_energyml_object("resqml22.TriangulatedSetRepresentation"))
-    ext_20 = create_energyml_object(
-        "application/x-eml+xml;version=2.0;type=obj_EpcExternalPartReference"
-    )
+    ext_20 = create_energyml_object("application/x-eml+xml;version=2.0;type=obj_EpcExternalPartReference")
     print(ext_20)
     print(gen_energyml_object_path(ext_20))
     print(create_external_part_reference("2.0", "my_h5"))
 
-    print(
-        parse_content_or_qualified_type(
-            "application/x-eml+xml;version=2.0;type=obj_EpcExternalPartReference"
-        )
-    )
+    print(parse_content_or_qualified_type("application/x-eml+xml;version=2.0;type=obj_EpcExternalPartReference"))
     print(
         get_domain_version_from_content_or_qualified_type(
             "application/x-eml+xml;version=2.0;type=obj_EpcExternalPartReference"
         )
     )
-    print(
-        get_domain_version_from_content_or_qualified_type(
-            "resqml20.obj_EpcExternalPartReference"
-        )
-    )
+    print(get_domain_version_from_content_or_qualified_type("resqml20.obj_EpcExternalPartReference"))
 
     # print(create_external_part_reference("2.2", "myfile.h5"))
     # print(create_external_part_reference("2.0", "myfile.h5"))

energyml-utils/example/main201.py

@@ -9,9 +9,7 @@ def import_modules():
     # import energyml.opc.opc
     # import energyml.resqml.v2_0_1.resqmlv2
     # import_modules()
-    epc201 = Epc.read_file(
-        "D:/Geosiris/OSDU/manifestTranslation/#Data/VOLVE_STRUCT.epc"
-    )
+    epc201 = Epc.read_file("D:/Geosiris/OSDU/manifestTranslation/#Data/VOLVE_STRUCT.epc")
     print(epc201)