Skip to content

Commit 8e07985

Browse files
yuzelinyuzelin
authored
[api][ci] Introduce paimon_python_api and ci framework (#2)
1 parent ae5f793 commit 8e07985

16 files changed

Lines changed: 1280 additions & 12 deletions

‎.github/PULL_REQUEST_TEMPLATE.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
### Purpose
2+
3+
<!-- Linking this pull request to the issue -->
4+
Linked issue: close #xxx
5+
6+
<!-- What is the purpose of the change -->
7+
8+
### Tests
9+
10+
<!-- List UT and IT cases to verify this change -->
11+
12+
### API and Format
13+
14+
<!-- Does this change affect API or storage format -->
15+
16+
### Documentation
17+
18+
<!-- Does this change introduce a new feature -->

‎.github/workflows/paimon-python-checks.yml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
################################################################################
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
################################################################################
18+
19+
name: Check Code Style & Run Tests
20+
21+
on:
22+
push:
23+
pull_request:
24+
paths-ignore:
25+
- 'dev/**'
26+
- 'java-based-implementation/paimon-python-java-bridge/**'
27+
- '**/*.md'
28+
29+
concurrency:
30+
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
31+
cancel-in-progress: true
32+
33+
jobs:
34+
lint-python:
35+
runs-on: ubuntu-latest
36+
37+
steps:
38+
- name: Run lint-python.sh
39+
run: |
40+
chmod +x dev/lint-python.sh
41+
./dev/lint-python.sh

‎.gitignore‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,16 @@
11
.idea/
22
!.idea/vcs.xml
3+
4+
# paimon-python-java-bridge
35
target
46
dependency-reduced-pom.xml
7+
8+
# Python building & testing
9+
.tox/
10+
build/
11+
dev/.conda
12+
dev/.stage.txt
13+
dev/download
14+
dev/log
15+
dist/
16+
paimon_python.egg-info/

‎dev/build-wheels.sh‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/usr/bin/env bash
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
set -e -x
17+
18+
## 1. install python env
19+
dev/lint-python.sh -s py_env
20+
21+
PY_ENV_DIR=`pwd`/dev/.conda/envs
22+
py_env=("3.8" "3.9" "3.10" "3.11")
23+
## 2. install dependency
24+
for ((i=0;i<${#py_env[@]};i++)) do
25+
echo "Installing dependencies for environment: ${py_env[i]}"
26+
${PY_ENV_DIR}/${py_env[i]}/bin/pip install -r dev/dev-requirements.txt
27+
done
28+
29+
## 3. build wheels
30+
for ((i=0;i<${#py_env[@]};i++)) do
31+
echo "Building wheel for environment: ${py_env[i]}"
32+
if [[ "$(uname)" != "Darwin" ]]; then
33+
# force the linker to use the older glibc version in Linux
34+
export CFLAGS="-I. -include dev/glibc_version_fix.h"
35+
fi
36+
${PY_ENV_DIR}/${py_env[i]}/bin/python setup.py bdist_wheel
37+
done
38+
39+
## 4. convert linux_x86_64 wheel to manylinux1 wheel in Linux
40+
if [[ "$(uname)" != "Darwin" ]]; then
41+
echo "Converting linux_x86_64 wheel to manylinux1"
42+
source `pwd`/dev/.conda/bin/activate
43+
# 4.1 install patchelf
44+
conda install -c conda-forge patchelf=0.11 -y
45+
# 4.2 install auditwheel
46+
pip install auditwheel==3.2.0
47+
# 4.3 convert Linux wheel
48+
for wheel_file in dist/*.whl; do
49+
auditwheel repair ${wheel_file} -w dist
50+
rm -f ${wheel_file}
51+
done
52+
source deactivate
53+
fi
54+
## see the result
55+
ls -al dist/

‎dev/dev-requirements.txt‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
################################################################################
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
################################################################################
18+
19+
pip>=20.3
20+
setuptools>=18.0
21+
wheel
22+
pytest~=7.0

‎dev/install_command.sh‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/usr/bin/env bash
2+
################################################################################
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
################################################################################
19+
20+
retry_times=3
21+
install_command="python -m pip install $@"
22+
${install_command}
23+
status=$?
24+
while [[ ${status} -ne 0 ]] && [[ ${retry_times} -gt 0 ]]; do
25+
retry_times=$((retry_times-1))
26+
# sleep 3 seconds and then reinstall.
27+
sleep 3
28+
${install_command}
29+
status=$?
30+
done
31+
32+
exit ${status}

0 commit comments

Comments
 (0)