Skip to content

Commit 9a864ea

Browse files
valentin-gauthier-geosirisvalentin-gauthier-geosiris
committed
better putDataarray
1 parent afd5da2 commit 9a864ea

4 files changed

Lines changed: 265 additions & 67 deletions

File tree

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<groupId>com.geosiris</groupId>
55
<artifactId>etpproto-java</artifactId>
66
<name>${project.groupId}:${project.artifactId}</name>
7-
<version>1.0.3</version>
7+
<version>1.0.4</version>
88
<organization>
99
<name>Geosiris</name>
1010
<url>http://www.geosiris.com</url>

‎src/example/com/geosiris/etp/example/Main.java‎

Lines changed: 63 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,14 +58,56 @@ public static void main(String[] args) throws Exception {
5858
// etpClientTest2(args);
5959
// test_big_message(args);
6060
// test_multiple_data_array(args);
61-
test_get_big_data_array(args);
61+
// test_get_big_data_array(args);
6262
// test_multiple_data_arraySmall(args);
6363
// test_multiple_data_REST(args);
6464
// test_get_dataspaces(args);
6565
// test_get_resources(args);
66-
// test_big_messageREST(args);
66+
// test_big_messagePUTREST(args);
67+
// test_get_big_data_array_REST(args);
68+
// putDA_REST(args);
69+
putDA_split(args);
6770
}
6871

72+
73+
public static void putDA_REST(String[] args) throws Exception {
74+
AnyArray aa_points = ETPHelper.constructArray(List.of(0,1,2,3,4,5));
75+
76+
DataArray da = DataArray.newBuilder()
77+
.setDimensions(List.of(2L, 3L))
78+
.setData(aa_points)
79+
.build();
80+
81+
ETPHelperREST.putDataArrays(args[0],
82+
"eml:///dataspace('volve-eqn-plus')/resqml20.obj_Grid2dRepresentation(3a45fb70-8ba9-4341-a701-0f514270ba9c)",
83+
List.of(new Pair<>("/RESQML/3a45fb70-8ba9-4341-a701-0f514270ba9c/a_fake_data", da)));
84+
}
85+
86+
public static void putDA_split(String[] args) throws Exception {
87+
ETPClient client = getClient(1<<18, args);
88+
String uri = "eml:///dataspace('volve-eqn-plus')/resqml20.obj_Grid2dRepresentation(3a45fb70-8ba9-4341-a701-0f514270ba9c)";
89+
String pathInRes = "/RESQML/3a45fb70-8ba9-4341-a701-0f514270ba9c/a_fake_data6";
90+
91+
try {
92+
93+
ETPHelper.sendPutDataArray(client,
94+
uri,
95+
pathInRes,
96+
List.of(0,1,2,3,4,5,6,7,8),
97+
List.of(3L, 3L),
98+
AnyLogicalArrayType.arrayOfInt64BE,
99+
AnyArrayType.arrayOfInt,
100+
64 * 3 * 2
101+
);
102+
}finally {
103+
client.closeClient();
104+
client.close();
105+
}
106+
logger.info("{}", ETPHelperREST.getMultipleDataArrays(args[0], uri, List.of(pathInRes)));
107+
}
108+
109+
110+
69111
public static void test_get_resources(String[] args) throws Exception {
70112
logger.info(ETPHelperREST.getResources(args[0], new ETPUri("volve-eqn-plus").toString(), ContextScopeKind.self));
71113
}
@@ -219,7 +261,24 @@ public static void test_get_big_data_array(String[] args) throws Exception {
219261
String uri = "eml:///dataspace('volve-eqn-plus')/resqml20.obj_Grid2dRepresentation(3a45fb70-8ba9-4341-a701-0f514270ba9c)";
220262
try {
221263
// ETPHelper.sendGetDataArray(client, uri, "/RESQML/3a45fb70-8ba9-4341-a701-0f514270ba9c/points_patch0", 50000);
222-
logger.info(ETPHelper.getMultipleDataArrays(client, uri, List.of("/RESQML/3a45fb70-8ba9-4341-a701-0f514270ba9c/points_patch0"), 50000));
264+
var m = ETPHelper.getMultipleDataArrays(client, uri, List.of("/RESQML/3a45fb70-8ba9-4341-a701-0f514270ba9c/points_patch0"), 50000);
265+
// logger.info(m);
266+
// Thread.sleep(3000000);
267+
}finally {
268+
client.closeClient();
269+
}
270+
271+
}
272+
273+
public static void test_get_big_data_array_REST(String[] args) throws Exception {
274+
ETPClient client = getClient(1<<22, args);
275+
276+
String uri = "eml:///dataspace('volve-eqn-plus')/resqml20.obj_Grid2dRepresentation(3a45fb70-8ba9-4341-a701-0f514270ba9c)";
277+
try {
278+
// ETPHelper.sendGetDataArray(client, uri, "/RESQML/3a45fb70-8ba9-4341-a701-0f514270ba9c/points_patch0", 50000);
279+
var m = ETPHelperREST.getMultipleDataArrays(args[0], uri, List.of("/RESQML/3a45fb70-8ba9-4341-a701-0f514270ba9c/points_patch0"));
280+
logger.info(m.getClass());
281+
logger.info((m.getDataArrays().values().iterator().next().getData().getItem()).getClass());
223282
// Thread.sleep(3000000);
224283
}finally {
225284
client.closeClient();
@@ -264,7 +323,7 @@ public static void test_big_message(String[] args) throws Exception {
264323
}
265324
}
266325

267-
public static void test_big_messageREST(String[] args) throws Exception {
326+
public static void test_big_messagePUTREST(String[] args) throws Exception {
268327
List<Double> values = new ArrayList<>();
269328
int nbTr = 1000000;
270329
for (int i = 0; i < nbTr; i++) {

‎src/main/java/com/geosiris/etp/utils/ETPHelper.java‎

Lines changed: 148 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,7 @@
1515
*/
1616
package com.geosiris.etp.utils;
1717

18-
import Energistics.Etp.v12.Datatypes.AnyArray;
19-
import Energistics.Etp.v12.Datatypes.AnyLogicalArrayType;
18+
import Energistics.Etp.v12.Datatypes.*;
2019
import Energistics.Etp.v12.Datatypes.DataArrayTypes.*;
2120
import Energistics.Etp.v12.Datatypes.Object.*;
2221
import Energistics.Etp.v12.Protocol.DataArray.*;
@@ -384,7 +383,7 @@ public static Map<String, List<Number>> getMultipleDataArrays(ETPClient client,
384383
).build()
385384
);
386385
GetDataSubarrays gdsa = GetDataSubarrays.newBuilder()
387-
.setDataSubarrays(mapSubArr).build();
386+
.setDataSubarrays(mapSubArr).build();
388387
long msgId = client.send(gdsa);
389388
msgIdsToWaitSubArray.put(msgId, e_subArray.getKey());
390389

@@ -451,16 +450,16 @@ public static Map<String, List<Number>> getMultipleDataArrays(ETPClient client,
451450
.flatMap(List::stream).collect(Collectors.toList());
452451
result.put(sub_arr_r.getKey(),numbers);*/
453452
List<Number> das = sub_arr_r.getValue().stream()
454-
.sorted(Comparator.comparingLong(m0 -> m0.getHeader().getMessageId()))
455-
.map(m -> ((GetDataSubarraysResponse) m.getBody()).getDataSubarrays().values())
456-
.flatMap(Collection::stream)
457-
.map(e -> {
458-
try {
459-
return (List<Number>) ETPUtils.getAttributeValue(e.getData().getItem(), "values");
460-
} catch (Exception ex) {
461-
throw new RuntimeException(ex);
462-
}
463-
}).filter(Objects::nonNull)
453+
.sorted(Comparator.comparingLong(m0 -> m0.getHeader().getMessageId()))
454+
.map(m -> ((GetDataSubarraysResponse) m.getBody()).getDataSubarrays().values())
455+
.flatMap(Collection::stream)
456+
.map(e -> {
457+
try {
458+
return (List<Number>) ETPUtils.getAttributeValue(e.getData().getItem(), "values");
459+
} catch (Exception ex) {
460+
throw new RuntimeException(ex);
461+
}
462+
}).filter(Objects::nonNull)
464463
.flatMap(List::stream)
465464
.collect(Collectors.toList());
466465
result.put(sub_arr_r.getKey(), das);
@@ -548,6 +547,78 @@ public static List<Message> sendGetDataArray_withSubArray(ETPClient etpClient, S
548547
return result;
549548
}
550549

550+
public static void sendPutDataArray(ETPClient etpClient, String uri, String pathInResource, List<?> dataFlat, List<Long> dimensions, AnyLogicalArrayType alat, AnyArrayType aat, int maxSize){
551+
Map<CharSequence, PutDataSubarraysType> map = new HashMap<>();
552+
PutUninitializedDataArrays pud = PutUninitializedDataArrays.newBuilder()
553+
.setDataArrays(
554+
Map.of(
555+
"0", PutUninitializedDataArrayType.newBuilder()
556+
.setUid(DataArrayIdentifier.newBuilder()
557+
.setPathInResource(pathInResource)
558+
.setUri(uri)
559+
.build()
560+
).setMetadata(
561+
DataArrayMetadata.newBuilder()
562+
.setDimensions(dimensions)
563+
.setStoreCreated(new Date().getTime())
564+
.setStoreLastWrite(new Date().getTime())
565+
.setLogicalArrayType(alat)
566+
.setCustomData(new HashMap<>())
567+
.setTransportArrayType(aat)
568+
.build()
569+
).build()
570+
)
571+
).build();
572+
long pud_id = etpClient.send(pud);
573+
List<Message> pud_resp = etpClient.getEtpClientSession().waitForResponse(pud_id, 50000);
574+
575+
long nbBitPerElt = getBitSize(alat);
576+
Long nbEltPerLine = dimensions.subList(1, dimensions.size()).stream().reduce(1L, (a, b) -> a * b);
577+
Long nbLinePerSplit = maxSize / (nbEltPerLine * nbBitPerElt);
578+
579+
final long nbLineInData = dataFlat.size() / nbEltPerLine;
580+
581+
for(long li=0; li<nbLineInData; li+=nbLinePerSplit) {
582+
long first = (li*nbEltPerLine);
583+
long last = Math.min(dataFlat.size(), (int) ((li + nbLinePerSplit)*nbEltPerLine));
584+
585+
List<Long> counts = new ArrayList<>();
586+
counts.add((last - first) / nbEltPerLine);
587+
counts.addAll(dimensions.subList(1, dimensions.size()));
588+
589+
List<Long> starts = new ArrayList<>();
590+
starts.add(li);
591+
for(int di=1; di<dimensions.size(); di++){
592+
starts.add(0L);
593+
}
594+
595+
System.out.print("starts [");
596+
starts.forEach(s -> System.out.print(s + ", "));
597+
System.out.print("]\n");
598+
System.out.print("counts[");
599+
counts.forEach(s -> System.out.print(s + ", "));
600+
System.out.print("]\n");
601+
AnyArray aa = constructArray(dataFlat.subList((int) (li*nbEltPerLine), Math.min(dataFlat.size(), (int) ((li + nbLinePerSplit)*nbEltPerLine))));
602+
DataArrayIdentifier dai = DataArrayIdentifier.newBuilder()
603+
.setUri(uri)
604+
.setPathInResource(pathInResource)
605+
.build();
606+
map.put(String.valueOf(map.size()), PutDataSubarraysType.newBuilder()
607+
.setUid(dai)
608+
.setData(aa)
609+
.setStarts(starts)
610+
.setCounts(counts)
611+
.build()
612+
);
613+
}
614+
PutDataSubarrays msg = PutDataSubarrays.newBuilder()
615+
.setDataSubarrays(map).build();
616+
long msg_id = etpClient.send(msg);
617+
logger.info("Exporting : PP " + uri + " " + pathInResource + " msg id : " + msg_id + "\n");
618+
619+
List<Message> pdar = etpClient.getEtpClientSession().waitForResponse(msg_id, 50000);
620+
}
621+
551622
public static Long sendGetDataSubArray_nowait(ETPClient etpClient, String uri, String pathInHDF5, Long[] start, Long[] count){
552623
DataArrayIdentifier identifier = DataArrayIdentifier.newBuilder()
553624
.setUri(uri)
@@ -672,20 +743,26 @@ public static List<Number> sendGetDataArray_prettier(ETPClient etpClient, String
672743
/* PutDataArray */
673744

674745
public static PutDataArrays buildPutDataArray(String uri, List<Pair<String, DataArray>> das){
675-
Map<CharSequence, PutDataArraysType> map = new HashMap<>();
676-
for(Pair<String, DataArray> pathAndDa: das) {
677-
DataArrayIdentifier dai = DataArrayIdentifier.newBuilder()
678-
.setUri(uri)
679-
.setPathInResource(pathAndDa.l())
680-
.build();
681-
map.put(String.valueOf(map.size()), PutDataArraysType.newBuilder()
682-
.setUid(dai)
683-
.setArray(pathAndDa.r())
684-
.build()
685-
);
746+
try {
747+
Map<CharSequence, PutDataArraysType> map = new HashMap<>();
748+
for (Pair<String, DataArray> pathAndDa : das) {
749+
DataArrayIdentifier dai = DataArrayIdentifier.newBuilder()
750+
.setUri(uri)
751+
.setPathInResource(pathAndDa.l())
752+
.build();
753+
map.put(String.valueOf(map.size()), PutDataArraysType.newBuilder()
754+
.setUid(dai)
755+
.setArray(pathAndDa.r())
756+
.setCustomData(new HashMap<>())
757+
.build()
758+
);
759+
}
760+
return PutDataArrays.newBuilder()
761+
.setDataArrays(map).build();
762+
}catch (Exception e){
763+
e.printStackTrace();
686764
}
687-
return PutDataArrays.newBuilder()
688-
.setDataArrays(map).build();
765+
return null;
689766
}
690767

691768
public static List<Message> sendPutDataArray(ETPClient etpClient, String uri, String pathInHDF5, DataArray da, int timeoutMS){
@@ -705,6 +782,51 @@ public static List<Message> sendPutDataArray(ETPClient etpClient, String uri, St
705782
return etpClient.getEtpClientSession().waitForResponse(msg_id, timeoutMS);
706783
}
707784

785+
public static AnyArray constructArray(List<?> data){
786+
if (data.get(0).getClass().equals(Double.class)) {
787+
return AnyArray.newBuilder()
788+
.setItem(ArrayOfDouble.newBuilder()
789+
.setValues((List<Double>) data)
790+
.build()
791+
).build();
792+
} else if (data.get(0).getClass().equals(Integer.class)) {
793+
return AnyArray.newBuilder()
794+
.setItem(ArrayOfInt.newBuilder()
795+
.setValues((List<Integer>) data)
796+
.build()
797+
).build();
798+
} else if (data.get(0).getClass().equals(Long.class)) {
799+
return AnyArray.newBuilder()
800+
.setItem(ArrayOfLong.newBuilder()
801+
.setValues((List<Long>) data)
802+
.build()
803+
).build();
804+
} else if (data.get(0).getClass().equals(Float.class)) {
805+
return AnyArray.newBuilder()
806+
.setItem(ArrayOfFloat.newBuilder()
807+
.setValues((List<Float>) data)
808+
.build()
809+
).build();
810+
} else if (data.get(0).getClass().equals(String.class)) {
811+
return AnyArray.newBuilder()
812+
.setItem(ArrayOfString.newBuilder()
813+
.setValues((List<CharSequence>) data)
814+
.build()
815+
).build();
816+
} else if (data.get(0).getClass().equals(Boolean.class)) {
817+
return AnyArray.newBuilder()
818+
.setItem(ArrayOfBoolean.newBuilder()
819+
.setValues((List<Boolean>) data)
820+
.build()
821+
).build();
822+
}
823+
logger.error("Not supported type for any array {}", data);
824+
if (!data.isEmpty())
825+
logger.error("\t" + data.get(0));
826+
return null;
827+
828+
}
829+
708830
public static void main(String[] argv){
709831
String xml = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><ns2:TriangulatedSetRepresentation xmlns=\"http://www.energistics.org/energyml/data/commonv2\" xmlns:ns2=\"http://www.energistics.org/energyml/data/resqmlv2\" uuid=\"f9a85fba-7f74-401f-99a2-03ff00a50086\" schemaVersion=\"2.2\" objectVersion=\"0\"><Citation><Title>null[Simplified]</Title><Creation>2023-10-27T17:04:25.403+02:00</Creation><LastUpdate>2023-10-27T17:04:25.404+02:00</LastUpdate><Description>A simplification of the representation null realised by Jerboa (a tool from University of Poitiers [FRANCE])</Description></Citation><ns2:TrianglePatch><ns2:NodeCount>10755</ns2:NodeCount><ns2:Triangles xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"IntegerExternalArray\"><NullValue>0</NullValue><CountPerValue>1</CountPerValue><Values><ExternalDataArrayPart><PathInExternalFile>RESQML/f9a85fba-7f74-401f-99a2-03ff00a50086triangle_patch0</PathInExternalFile></ExternalDataArrayPart></Values></ns2:Triangles><ns2:Geometry><ns2:Points xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:type=\"ns2:Point3dExternalArray\"><ns2:Coordinates><ExternalDataArrayPart><PathInExternalFile>RESQML/f9a85fba-7f74-401f-99a2-03ff00a50086point_patch0</PathInExternalFile></ExternalDataArrayPart></ns2:Coordinates></ns2:Points></ns2:Geometry></ns2:TrianglePatch> </ns2:TriangulatedSetRepresentation>";
710832
Document xmlDoc = null;

0 commit comments

Comments
 (0)