Skip to content

Commit e8c98d9

Browse files
luoyuxialuoyuxia
authored
feat(python): introduce pypaimon core and DataFusion catalog integration (#204)
1 parent e0b5e5c commit e8c98d9

28 files changed

Lines changed: 999 additions & 54 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,17 @@ jobs:
9090
steps:
9191
- uses: actions/checkout@v6
9292

93+
- name: Rust Cache
94+
uses: actions/cache@v4
95+
with:
96+
path: |
97+
~/.cargo/registry
98+
~/.cargo/git
99+
target
100+
key: integration-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
101+
restore-keys: |
102+
integration-${{ runner.os }}-
103+
93104
- name: Start Docker containers
94105
run: make docker-up
95106

@@ -105,6 +116,16 @@ jobs:
105116
RUST_LOG: DEBUG
106117
RUST_BACKTRACE: full
107118

119+
- name: Install uv
120+
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78
121+
with:
122+
version: "0.9.3"
123+
enable-cache: true
124+
125+
- name: Python Binding Integration Test
126+
working-directory: bindings/python
127+
run: make install && make test
128+
108129
- name: Go Integration Test
109130
working-directory: bindings/go
110131
run: make test

‎.licenserc.yaml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
header:
19+
license:
20+
spdx-id: Apache-2.0
21+
copyright-owner: Apache Software Foundation
22+
23+
paths-ignore:
24+
- "LICENSE"
25+
- "NOTICE"
26+
- ".github/PULL_REQUEST_TEMPLATE.md"
27+
- "crates/paimon/tests/**/*.json"
28+
- "**/go.sum"
29+
- ".devcontainer/devcontainer.json"
30+
- "bindings/python/python/pypaimon_rust/py.typed"
31+
32+
comment: on-failure

‎Cargo.toml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
[workspace]
1919
resolver = "2"
20-
members = ["crates/paimon", "crates/integration_tests", "bindings/c", "crates/integrations/datafusion"]
20+
members = ["crates/paimon", "crates/integration_tests", "bindings/c", "bindings/python", "crates/integrations/datafusion"]
2121

2222
[workspace.package]
2323
version = "0.0.0"
@@ -28,8 +28,11 @@ license = "Apache-2.0"
2828
rust-version = "1.86.0"
2929

3030
[workspace.dependencies]
31+
arrow = "57.0"
3132
arrow-array = { version = "57.0", features = ["ffi"] }
3233
arrow-schema = "57.0"
3334
arrow-cast = "57.0"
35+
datafusion = "52.3.0"
36+
datafusion-ffi = "52.3.0"
3437
parquet = "57.0"
3538
tokio = "1.39.2"

‎LICENSE‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@
186186
same "printed page" as the copyright notice for easier
187187
identification within third-party archives.
188188

189-
Copyright 2021 Datafuse Labs
189+
Copyright [yyyy] [name of copyright owner]
190190

191191
Licensed under the Apache License, Version 2.0 (the "License");
192192
you may not use this file except in compliance with the License.
@@ -198,4 +198,4 @@
198198
distributed under the License is distributed on an "AS IS" BASIS,
199199
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200200
See the License for the specific language governing permissions and
201-
limitations under the License.
201+
limitations under the License.

‎bindings/go/go.mod‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,20 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
118
module github.com/apache/paimon-rust/bindings/go
219

320
go 1.22.4

‎bindings/go/tests/go.mod‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,20 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
118
module paimon_test
219

320
go 1.22.4

‎bindings/python/.gitignore‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
.venv/
19+
.idea/
20+
__pycache__/
21+
.pytest_cache/
22+
23+
*.dSYM/
24+
*.so
25+
uv.lock

‎bindings/python/Cargo.toml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
[package]
19+
name = "pypaimon_rust"
20+
publish = false
21+
edition.workspace = true
22+
version.workspace = true
23+
license.workspace = true
24+
25+
[lib]
26+
crate-type = ["cdylib"]
27+
doc = false
28+
29+
[dependencies]
30+
arrow = { workspace = true, features = ["pyarrow"] }
31+
datafusion = { workspace = true }
32+
datafusion-ffi = { workspace = true }
33+
paimon = { path = "../../crates/paimon", features = ["storage-all"] }
34+
paimon-datafusion = { path = "../../crates/integrations/datafusion" }
35+
pyo3 = { version = "0.26", features = ["abi3-py310"] }
36+
tokio = { workspace = true }

0 commit comments

Comments
 (0)