Skip to content

Commit ae5f793

Browse files
authored
[java-based-impl] Add paimon-python-java-bridge (#1)
1 parent df2c3f1 commit ae5f793

9 files changed

Lines changed: 1271 additions & 0 deletions

File tree

.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
.idea/
22
!.idea/vcs.xml
3+
target
4+
dependency-reduced-pom.xml
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one
4+
~ or more contributor license agreements. See the NOTICE file
5+
~ distributed with this work for additional information
6+
~ regarding copyright ownership. The ASF licenses this file
7+
~ to you under the Apache License, Version 2.0 (the
8+
~ "License"); you may not use this file except in compliance
9+
~ with the License. You may obtain a copy of the License at
10+
~
11+
~ http://www.apache.org/licenses/LICENSE-2.0
12+
~
13+
~ Unless required by applicable law or agreed to in writing, software
14+
~ distributed under the License is distributed on an "AS IS" BASIS,
15+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
~ See the License for the specific language governing permissions and
17+
~ limitations under the License.
18+
-->
19+
<project xmlns="http://maven.apache.org/POM/4.0.0"
20+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
21+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
22+
<modelVersion>4.0.0</modelVersion>
23+
24+
<groupId>org.apache.paimon</groupId>
25+
<artifactId>paimon-python-java-bridge</artifactId>
26+
<version>0.9-SNAPSHOT</version>
27+
<name>Paimon : Python-Java Bridge</name>
28+
29+
<packaging>jar</packaging>
30+
31+
<properties>
32+
<paimon.version>0.9-SNAPSHOT</paimon.version>
33+
<flink.shaded.hadoop.version>2.8.3-10.0</flink.shaded.hadoop.version>
34+
<py4j.version>0.10.9.7</py4j.version>
35+
<slf4j.version>1.7.32</slf4j.version>
36+
<spotless.version>2.13.0</spotless.version>
37+
<spotless.delimiter>package</spotless.delimiter>
38+
</properties>
39+
40+
<dependencies>
41+
42+
<!-- Java dependencies -->
43+
44+
<dependency>
45+
<groupId>org.apache.paimon</groupId>
46+
<artifactId>paimon-bundle</artifactId>
47+
<version>${paimon.version}</version>
48+
</dependency>
49+
50+
<dependency>
51+
<groupId>org.slf4j</groupId>
52+
<artifactId>slf4j-api</artifactId>
53+
<version>${slf4j.version}</version>
54+
</dependency>
55+
56+
<dependency>
57+
<groupId>org.apache.flink</groupId>
58+
<artifactId>flink-shaded-hadoop-2-uber</artifactId>
59+
<version>${flink.shaded.hadoop.version}</version>
60+
</dependency>
61+
62+
<!-- Python API dependencies -->
63+
64+
<dependency>
65+
<groupId>net.sf.py4j</groupId>
66+
<artifactId>py4j</artifactId>
67+
<version>${py4j.version}</version>
68+
</dependency>
69+
70+
</dependencies>
71+
72+
<build>
73+
<plugins>
74+
<plugin>
75+
<groupId>com.diffplug.spotless</groupId>
76+
<artifactId>spotless-maven-plugin</artifactId>
77+
<version>${spotless.version}</version>
78+
<configuration>
79+
<java>
80+
<googleJavaFormat>
81+
<version>1.7</version>
82+
<style>AOSP</style>
83+
</googleJavaFormat>
84+
85+
<!-- \# refers to the static imports -->
86+
<importOrder>
87+
<order>org.apache.paimon,org.apache.paimon.shade,,javax,java,scala,\#</order>
88+
</importOrder>
89+
90+
<licenseHeader>
91+
<!-- replace it with ${project.rootDirectory} after maven 4.0.0, see MNG-7038 -->
92+
<file>${maven.multiModuleProjectDirectory}/copyright.txt</file>
93+
<delimiter>${spotless.delimiter}</delimiter>
94+
</licenseHeader>
95+
</java>
96+
</configuration>
97+
<executions>
98+
<execution>
99+
<id>spotless-check</id>
100+
<phase>validate</phase>
101+
<goals>
102+
<goal>check</goal>
103+
</goals>
104+
</execution>
105+
</executions>
106+
</plugin>
107+
108+
<plugin>
109+
<groupId>org.apache.maven.plugins</groupId>
110+
<artifactId>maven-shade-plugin</artifactId>
111+
<executions>
112+
<execution>
113+
<id>shade-paimon</id>
114+
<phase>package</phase>
115+
<goals>
116+
<goal>shade</goal>
117+
</goals>
118+
<configuration>
119+
<artifactSet>
120+
<includes combine.children="append">
121+
<include>org.apache.paimon:paimon-bundle</include>
122+
<include>org.slf4j:slf4j-api</include>
123+
<include>org.apache.flink:flink-shaded-hadoop-2-uber</include>
124+
<include>net.sf.py4j:py4j</include>
125+
</includes>
126+
</artifactSet>
127+
</configuration>
128+
</execution>
129+
</executions>
130+
</plugin>
131+
</plugins>
132+
</build>
133+
134+
</project>
Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.paimon.python;
20+
21+
import org.apache.paimon.utils.Preconditions;
22+
23+
import java.io.File;
24+
import java.io.FileOutputStream;
25+
import java.io.IOException;
26+
import java.nio.file.Path;
27+
import java.nio.file.Paths;
28+
29+
/** A file lock used for avoiding race condition among multiple threads/processes. */
30+
public class FileLock {
31+
private static final String TEMP_DIR = System.getProperty("java.io.tmpdir");
32+
private final File file;
33+
private FileOutputStream outputStream;
34+
private java.nio.channels.FileLock lock;
35+
36+
/**
37+
* Initialize a FileLock using a file located at fullPath.
38+
*
39+
* @param fullPath The path of the locking file
40+
*/
41+
public FileLock(String fullPath) {
42+
Preconditions.checkNotNull(fullPath, "fullPath should not be null");
43+
Path path = Paths.get(fullPath);
44+
String normalizedFileName = normalizeFileName(path.getFileName().toString());
45+
if (normalizedFileName.isEmpty()) {
46+
throw new IllegalArgumentException("There are no legal characters in the file name");
47+
}
48+
this.file =
49+
path.getParent() == null
50+
? new File(TEMP_DIR, normalizedFileName)
51+
: new File(path.getParent().toString(), normalizedFileName);
52+
}
53+
54+
/**
55+
* Initialize a FileLock using a file located at parentDir/fileName.
56+
*
57+
* @param parentDir The parent dir of the locking file
58+
* @param fileName The name of the locking file
59+
*/
60+
public FileLock(String parentDir, String fileName) {
61+
Preconditions.checkNotNull(parentDir, "parentDir should not be null");
62+
Preconditions.checkNotNull(fileName, "fileName should not be null");
63+
this.file = new File(parentDir, normalizeFileName(fileName));
64+
}
65+
66+
/**
67+
* Initialize a FileLock using a file located inside temp folder.
68+
*
69+
* @param fileName The name of the locking file
70+
* @return The initialized FileLock
71+
*/
72+
public static FileLock inTempFolder(String fileName) {
73+
return new FileLock(TEMP_DIR, fileName);
74+
}
75+
76+
/**
77+
* Check whether the locking file exists in the file system. Create it if it does not exist.
78+
* Then create a FileOutputStream for it.
79+
*
80+
* @throws IOException If the file path is invalid or the parent dir does not exist
81+
*/
82+
private void init() throws IOException {
83+
if (!this.file.exists()) {
84+
this.file.createNewFile();
85+
}
86+
outputStream = new FileOutputStream(this.file);
87+
}
88+
89+
/**
90+
* Try to acquire a lock on the locking file. This method immediately returns whenever the lock
91+
* is acquired or not.
92+
*
93+
* @return True if successfully acquired the lock
94+
* @throws IOException If the file path is invalid
95+
*/
96+
public boolean tryLock() throws IOException {
97+
if (outputStream == null) {
98+
init();
99+
}
100+
try {
101+
lock = outputStream.getChannel().tryLock();
102+
} catch (Exception e) {
103+
return false;
104+
}
105+
106+
return lock != null;
107+
}
108+
109+
/**
110+
* Release the file lock.
111+
*
112+
* @throws IOException If the FileChannel is closed
113+
*/
114+
public void unlock() throws IOException {
115+
if (lock != null && lock.channel().isOpen()) {
116+
lock.release();
117+
}
118+
}
119+
120+
/**
121+
* Release the file lock, close the fileChannel and FileOutputStream then try deleting the
122+
* locking file if other file lock does not need it, which means the lock will not be used
123+
* anymore.
124+
*
125+
* @throws IOException If an I/O error occurs
126+
*/
127+
public void unlockAndDestroy() throws IOException {
128+
try {
129+
unlock();
130+
if (lock != null) {
131+
lock.channel().close();
132+
lock = null;
133+
}
134+
if (outputStream != null) {
135+
outputStream.close();
136+
outputStream = null;
137+
}
138+
139+
} finally {
140+
this.file.delete();
141+
}
142+
}
143+
144+
/**
145+
* Check whether a FileLock is actually holding the lock.
146+
*
147+
* @return True if it is actually holding the lock
148+
*/
149+
public boolean isValid() {
150+
if (lock != null) {
151+
return lock.isValid();
152+
}
153+
return false;
154+
}
155+
156+
/**
157+
* Normalize the file name, which only allows slash, backslash, digits and letters.
158+
*
159+
* @param fileName Original file name
160+
* @return File name with illegal characters stripped
161+
*/
162+
private static String normalizeFileName(String fileName) {
163+
return fileName.replaceAll("[^\\w/\\\\]", "");
164+
}
165+
}

0 commit comments

Comments
 (0)