java_based_implementation packages java-bridge jar by default (#13)

Author: yuzelin

README.md

@@ -28,14 +28,6 @@ and running "External Tools" → "flake8".
 
 We can use `py4j` to leverage Java code to read Paimon data. This section describes how to use this implementation.
 
-### Build paimon-python-java-bridge
-
-```bash
-cd java_based_implementation/paimon-python-java-bridge/
-mvn clean install -DskipTests
-```
-The built target is java-based-implementation/paimon-python-java-bridge/target/paimon-python-java-bridge-<version>.jar
-
 ### Set Environment Variables
 
 `py4j` need to access a JVM, so we should set JVM arguments (optional) and Java classpath. A convenient way is using
@@ -44,10 +36,14 @@ The built target is java-based-implementation/paimon-python-java-bridge/target/p
 ```python
 import os
 
-os.environ['PYPAIMON_JAVA_CLASSPATH'] = '/path/to/paimon-python-java-bridge-<version>.jar'
+os.environ['PYPAIMON_JAVA_CLASSPATH'] = '/path/to/dependent_jars/*'
 os.environ['_PYPAIMON_JVM_ARGS'] = 'jvm_arg1 jvm_arg2 ...'
 ```
 
+NOTE: the package has set paimon core and hadoop dependencies. If you just test in local or run code in hadoop, you doesn't
+need to set classpath. If you need other dependencies such as OSS/S3 filesystem jars, or special catalog which isn't implemented 
+in paimon core, please download jars and set classpath.
+
 # API Reference
 TODO
 

dev/build-wheels.sh

@@ -33,7 +33,7 @@ for ((i=0;i<${#py_env[@]};i++)) do
         # force the linker to use the older glibc version in Linux
         export CFLAGS="-I. -include dev/glibc_version_fix.h"
     fi
-    ${PY_ENV_DIR}/${py_env[i]}/bin/python setup.py bdist_wheel
+    ${PY_ENV_DIR}/${py_env[i]}/bin/python setup.py clean bdist_wheel
 done
 
 ## 4. convert linux_x86_64 wheel to manylinux1 wheel in Linux

dev/lint-python.sh

@@ -578,6 +578,9 @@ function tox_check() {
     # Ensure the permission of the scripts set correctly
     chmod +x $PAIMON_PYTHON_DIR/dev/*
 
+    # tox runs codes in virtual env, set var to avoid error
+    export _PYPAIMON_TOX_TEST="true"
+
     if [[ ${BUILD_REASON} = 'IndividualCI' ]]; then
         # Only run test in latest python version triggered by a Git push
         $TOX_PATH -vv -c $PAIMON_PYTHON_DIR/tox.ini -e ${LATEST_PYTHON} --recreate 2>&1 | tee -a $LOG_FILE

java_based_implementation/gateway_server.py

@@ -21,8 +21,9 @@
 import signal
 
 from subprocess import Popen, PIPE
-from java_based_implementation.util.constants import (PYPAIMON_JVM_ARGS, PYPAIMON_JAVA_CLASSPATH,
-                                                      PYPAIMON_MAIN_ARGS, PYPAIMON_MAIN_CLASS)
+from java_based_implementation.util.constants import (PYPAIMON_JVM_ARGS, PYPAIMON_MAIN_ARGS,
+                                                      PYPAIMON_MAIN_CLASS)
+from java_based_implementation.util.setup_utils import get_classpath
 
 
 def on_windows():
@@ -47,7 +48,7 @@ def launch_gateway_server_process(env):
     # TODO construct Java module log settings
     log_settings = []
     jvm_args = env.get(PYPAIMON_JVM_ARGS, '').split()
-    classpath = env.get(PYPAIMON_JAVA_CLASSPATH)
+    classpath = get_classpath(env)
     main_args = env.get(PYPAIMON_MAIN_ARGS, '').split()
     command = [
         java_executable,

java_based_implementation/tests/test_write_and_read.py

@@ -16,27 +16,29 @@
 # limitations under the License.
 ################################################################################
 
-import os
 import tempfile
 import unittest
 import pandas as pd
 import pyarrow as pa
 
 from java_based_implementation.api_impl import Catalog, Table
 from java_based_implementation.java_gateway import get_gateway
-from java_based_implementation.tests.utils import set_bridge_jar, create_simple_table
-from java_based_implementation.util import constants, java_utils
+from java_based_implementation.tests.utils import create_simple_table
+from java_based_implementation.util import java_utils, setup_utils
 from py4j.protocol import Py4JJavaError
 
 
 class TableWriteReadTest(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        classpath = set_bridge_jar()
-        os.environ[constants.PYPAIMON_JAVA_CLASSPATH] = classpath
+        setup_utils.setup_java_bridge()
         cls.warehouse = tempfile.mkdtemp()
 
+    @classmethod
+    def tearDownClass(cls):
+        setup_utils.clean()
+
     def testReadEmptyAppendTable(self):
         create_simple_table(self.warehouse, 'default', 'empty_append_table', False)
         catalog = Catalog.create({'warehouse': self.warehouse})

java_based_implementation/tests/utils.py

@@ -16,35 +16,10 @@
 # limitations under the License.
 ################################################################################
 
-import os
-import shutil
-import subprocess
-import tempfile
-
 from java_based_implementation.java_gateway import get_gateway
 from java_based_implementation.util.java_utils import to_j_catalog_context
 
 
-def set_bridge_jar() -> str:
-    current_file_path = os.path.abspath(__file__)
-    current_dir = os.path.dirname(current_file_path)
-    parent_dir = os.path.dirname(current_dir)
-    java_module = os.path.join(parent_dir, 'paimon-python-java-bridge')
-    # build paimon-python-java-bridge
-    subprocess.run(
-        ["mvn", "clean", "package"],
-        cwd=java_module,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE
-    )
-    jar_name = 'paimon-python-java-bridge-0.9-SNAPSHOT.jar'
-    jar_file = os.path.join(java_module, 'target', jar_name)
-    # move to temp dir
-    temp_dir = tempfile.mkdtemp()
-    shutil.move(jar_file, temp_dir)
-    return os.path.join(temp_dir, jar_name)
-
-
 def create_simple_table(warehouse, database, table_name, has_pk, options=None):
     if options is None:
         options = {

java_based_implementation/util/constants.py

@@ -22,3 +22,4 @@
 PYPAIMON_JAVA_CLASSPATH = '_PYPAIMON_JAVA_CLASSPATH'
 PYPAIMON_MAIN_CLASS = 'org.apache.paimon.python.PythonGatewayServer'
 PYPAIMON_MAIN_ARGS = '_PYPAIMON_MAIN_ARGS'
+PYPAIMON_TOX_TEST = '_PYPAIMON_TOX_TEST'

java_based_implementation/util/setup_utils.py

@@ -0,0 +1,89 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import importlib
+import os
+import shutil
+import subprocess
+
+from java_based_implementation.util import constants
+from java_based_implementation.util.constants import PYPAIMON_JAVA_CLASSPATH
+
+_JAVA_IMPL_MODULE = 'java_based_implementation'
+_JAVA_DEPS = 'java_dependencies'
+_JAVA_BRIDGE = 'paimon-python-java-bridge'
+# TODO configure version
+_JAVA_BRIDGE_VERSION = '0.9-SNAPSHOT'
+
+
+def get_package_data():
+    is_tox_test = os.environ.get(constants.PYPAIMON_TOX_TEST)
+    if is_tox_test and is_tox_test.lower() == "true":
+        return ['']
+
+    setup_java_bridge()
+    return [os.path.join(_JAVA_DEPS, '*')]
+
+
+def clean():
+    java_deps_dir = os.path.join(_find_java_impl_dir(), _JAVA_DEPS)
+    if os.path.exists(java_deps_dir):
+        shutil.rmtree(java_deps_dir)
+
+
+def get_classpath(env):
+    user_defined = env.get(PYPAIMON_JAVA_CLASSPATH)
+
+    module = importlib.import_module(_JAVA_IMPL_MODULE)
+    builtin_java_bridge = os.path.join(*module.__path__, _JAVA_DEPS, _JAVA_BRIDGE + '.jar')
+
+    if user_defined is None:
+        return builtin_java_bridge
+    else:
+        return os.pathsep.join([builtin_java_bridge, user_defined])
+
+
+def setup_java_bridge():
+    java_impl_dir = _find_java_impl_dir()
+
+    java_deps_dir = os.path.join(java_impl_dir, _JAVA_DEPS)
+    if not os.path.exists(java_deps_dir):
+        os.mkdir(java_deps_dir)
+
+    java_bridge_dst = os.path.join(java_deps_dir, _JAVA_BRIDGE + '.jar')
+    if os.path.exists(java_bridge_dst):
+        return
+
+    java_bridge_module = os.path.join(java_impl_dir, _JAVA_BRIDGE)
+    subprocess.run(
+        ["mvn", "clean", "package"],
+        cwd=java_bridge_module,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE
+    )
+
+    shutil.copy(
+        os.path.join(java_bridge_module, 'target/{}-{}.jar'
+                     .format(_JAVA_BRIDGE, _JAVA_BRIDGE_VERSION)),
+        java_bridge_dst
+    )
+
+
+def _find_java_impl_dir():
+    abspath = os.path.abspath(__file__)
+    return os.path.dirname(os.path.dirname(abspath))

setup.py

@@ -16,38 +16,77 @@
 # limitations under the License.
 ################################################################################
 
-from setuptools import setup
-
-PACKAGES = [
-    'paimon_python_api',
-    'java_based_implementation',
-    'java_based_implementation.util'
-]
-
-install_requires = [
-    'py4j==0.10.9.7',
-    'python-dateutil>=2.8.0,<3',
-    'pytz>=2018.3',
-    'numpy>=1.22.4',
-    'pandas>=1.3.0',
-    'pyarrow>=5.0.0'
-]
-
-setup(
-    name='paimon_python',
-    version='0.9.0.dev1',
-    packages=PACKAGES,
-    install_requires=install_requires,
-    description='Apache Paimon Python API',
-    author='Apache Software Foundation',
-    author_email='dev@paimon.apache.org',
-    url='https://paimon.apache.org',
-    classifiers=[
-        'Development Status :: 5 - Production/Stable',
-        'License :: OSI Approved :: Apache Software License',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
-        'Programming Language :: Python :: 3.10',
-        'Programming Language :: Python :: 3.11'],
-    python_requires='>=3.8'
-)
+import fnmatch
+import java_based_implementation.util.setup_utils
+import os
+import shutil
+
+from setuptools import Command, setup
+
+
+class CleanCommand(Command):
+    description = 'Clean up temporary files and directories of last build.'
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        directories_to_delete = ['build', 'dist', '*.egg-info']
+
+        for directory in directories_to_delete:
+            if '*' in directory:
+                for matched_dir in filter(lambda x: fnmatch.fnmatch(x, directory), os.listdir('.')):
+                    if os.path.isdir(matched_dir):
+                        shutil.rmtree(matched_dir)
+            else:
+                if os.path.exists(directory):
+                    shutil.rmtree(directory)
+
+
+try:
+    PACKAGES = [
+        'paimon_python_api',
+        'java_based_implementation',
+        'java_based_implementation.util'
+    ]
+
+    PACKAGE_DATA = {
+        'java_based_implementation': java_based_implementation.util.setup_utils.get_package_data()
+    }
+
+    install_requires = [
+        'py4j==0.10.9.7',
+        'python-dateutil>=2.8.0,<3',
+        'pytz>=2018.3',
+        'numpy>=1.22.4',
+        'pandas>=1.3.0',
+        'pyarrow>=5.0.0'
+    ]
+
+    setup(
+        name='paimon_python',
+        version='0.9.0.dev1',
+        packages=PACKAGES,
+        include_package_data=True,
+        package_data=PACKAGE_DATA,
+        cmdclass={'clean': CleanCommand},
+        install_requires=install_requires,
+        description='Apache Paimon Python API',
+        author='Apache Software Foundation',
+        author_email='dev@paimon.apache.org',
+        url='https://paimon.apache.org',
+        classifiers=[
+            'Development Status :: 5 - Production/Stable',
+            'License :: OSI Approved :: Apache Software License',
+            'Programming Language :: Python :: 3.8',
+            'Programming Language :: Python :: 3.9',
+            'Programming Language :: Python :: 3.10',
+            'Programming Language :: Python :: 3.11'],
+        python_requires='>=3.8'
+    )
+finally:
+    java_based_implementation.util.setup_utils.clean()