Skip to content

Commit 5c46a39

Browse files
jacktenggjacktengg
authored
branch-4.0: [fix](crc32c) fix bug of crc32c (#60322) (#60330)
Cherry-picked from #60322 Problem Summary: None - Test <!-- At least one of them must be included. --> - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [ ] No. - [ ] Yes. <!-- Explain the behavior change --> - Does this need documentation? - [ ] No. - [ ] Yes. <!-- Add document PR link here. eg: apache/doris-website#1214 --> - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into --> ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test <!-- At least one of them must be included. --> - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [ ] No. - [ ] Yes. <!-- Explain the behavior change --> - Does this need documentation? - [ ] No. - [ ] Yes. <!-- Add document PR link here. eg: apache/doris-website#1214 --> ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->
1 parent 9f9ae5f commit 5c46a39

5 files changed

Lines changed: 145 additions & 8 deletions

File tree

‎be/src/vec/columns/column_decimal.cpp‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -193,15 +193,14 @@ void ColumnDecimal<T>::update_crc32c_batch(uint32_t* __restrict hashes,
193193
template <PrimitiveType T>
194194
void ColumnDecimal<T>::update_crc32c_single(size_t start, size_t end, uint32_t& hash,
195195
const uint8_t* __restrict null_map) const {
196-
auto s = size();
197196
if (null_map) {
198-
for (size_t i = 0; i < s; ++i) {
197+
for (size_t i = start; i < end; ++i) {
199198
if (null_map[i] == 0) {
200199
hash = HashUtil::crc32c_fixed(data[i], hash);
201200
}
202201
}
203202
} else {
204-
for (size_t i = 0; i < s; ++i) {
203+
for (size_t i = start; i < end; ++i) {
205204
hash = HashUtil::crc32c_fixed(data[i], hash);
206205
}
207206
}

‎be/src/vec/columns/column_nullable.cpp‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,8 +140,7 @@ void ColumnNullable::update_crc32c_single(size_t start, size_t end, uint32_t& ha
140140
const auto* __restrict real_null_data =
141141
assert_cast<const ColumnUInt8&>(get_null_map_column()).get_data().data();
142142
constexpr int NULL_VALUE = 0;
143-
auto s = size();
144-
for (int i = 0; i < s; ++i) {
143+
for (size_t i = start; i < end; ++i) {
145144
if (real_null_data[i] != 0) {
146145
hash = HashUtil::crc32c_fixed(NULL_VALUE, hash);
147146
}

‎be/src/vec/columns/column_vector.cpp‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -222,15 +222,14 @@ void ColumnVector<T>::update_crc32c_batch(uint32_t* __restrict hashes,
222222
template <PrimitiveType T>
223223
void ColumnVector<T>::update_crc32c_single(size_t start, size_t end, uint32_t& hash,
224224
const uint8_t* __restrict null_map) const {
225-
auto s = size();
226225
if (null_map) {
227-
for (size_t i = 0; i < s; ++i) {
226+
for (size_t i = start; i < end; ++i) {
228227
if (null_map[i] == 0) {
229228
hash = _crc32c_hash(hash, i);
230229
}
231230
}
232231
} else {
233-
for (size_t i = 0; i < s; ++i) {
232+
for (size_t i = start; i < end; ++i) {
234233
hash = _crc32c_hash(hash, i);
235234
}
236235
}

‎regression-test/data/datatype_p0/complex_types/test_array.out‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !window_function_partition_by_array_int --
3+
Badminton Racket [3] 3 3
4+
Basketball [1, 3] 1 4
5+
Basketball [1, 3] 3 4
6+
Laptop [1, 2, 3] 1 6
7+
Laptop [1, 2, 3] 2 6
8+
Laptop [1, 2, 3] 3 6
9+
Mechanical Keyboard [1, 2] 1 3
10+
Mechanical Keyboard [1, 2] 2 3
11+
Shirt [4] 4 4
12+
13+
-- !window_function_partition_by_array_decimal --
14+
Badminton Racket [3.00] 3 3
15+
Basketball [1.00, 3.00] 1 4
16+
Basketball [1.00, 3.00] 3 4
17+
Laptop [1.00, 2.00, 3.00] 1 6
18+
Laptop [1.00, 2.00, 3.00] 2 6
19+
Laptop [1.00, 2.00, 3.00] 3 6
20+
Mechanical Keyboard [1.00, 2.00] 1 3
21+
Mechanical Keyboard [1.00, 2.00] 2 3
22+
Shirt [4.00] 4 4
23+

‎regression-test/suites/datatype_p0/complex_types/test_array.groovy‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("test_array") {
19+
multi_sql """
20+
set batch_size=3;
21+
set enable_local_shuffle=false;
22+
set parallel_pipeline_task_num = 2;
23+
set enable_new_shuffle_hash_method=true;
24+
"""
25+
26+
sql "DROP TABLE IF EXISTS `test_array_int_table`"
27+
multi_sql """
28+
CREATE TABLE test_array_int_table(
29+
id INT,
30+
name VARCHAR(50),
31+
tags ARRAY<VARCHAR(50)>,
32+
price DECIMAL(10,2),
33+
category_ids ARRAY<INT>,
34+
cid INT
35+
)
36+
DUPLICATE KEY(id)
37+
DISTRIBUTED BY HASH(id) BUCKETS 2
38+
PROPERTIES (
39+
"replication_allocation" = "tag.location.default: 1"
40+
);
41+
42+
INSERT INTO test_array_int_table VALUES
43+
(1, 'Laptop', ['Electronics', 'Office', 'High-End', 'Laptop'], 5999.99, [1, 2, 3], 1),
44+
(2, 'Mechanical Keyboard', ['Electronics', 'Accessories'], 399.99, [1, 2], 1),
45+
(3, 'Basketball', ['Sports', 'Outdoor'], 199.99, [1,3], 1),
46+
(4, 'Badminton Racket', ['Sports', 'Equipment'], 299.99, [3], 3),
47+
(5, 'Shirt', ['Clothing', 'Office', 'Shirt'], 259.00, [4], 4);
48+
49+
INSERT INTO test_array_int_table VALUES
50+
(1, 'Laptop', ['Electronics', 'Office', 'High-End', 'Laptop'], 5999.99, [1, 2, 3], 2),
51+
(2, 'Mechanical Keyboard', ['Electronics', 'Accessories'], 399.99, [1, 2], 2),
52+
(3, 'Basketball', ['Sports', 'Outdoor'], 199.99, [1,3], 3);
53+
54+
INSERT INTO test_array_int_table VALUES
55+
(1, 'Laptop', ['Electronics', 'Office', 'High-End', 'Laptop'], 5999.99, [1, 2, 3], 3);
56+
"""
57+
58+
qt_window_function_partition_by_array_int """
59+
select
60+
name,
61+
category_ids,
62+
cid,
63+
sum(cid) over(partition by category_ids)
64+
from
65+
test_array_int_table
66+
group by
67+
name, category_ids, cid
68+
order by
69+
name, category_ids, cid;
70+
"""
71+
// test decimal
72+
sql "DROP TABLE IF EXISTS `test_array_decimal_table`"
73+
multi_sql """
74+
CREATE TABLE test_array_decimal_table(
75+
id INT,
76+
name VARCHAR(50),
77+
tags ARRAY<VARCHAR(50)>,
78+
price DECIMAL(10,2),
79+
category_ids ARRAY<decimalv3(10,2)>,
80+
cid INT
81+
)
82+
DUPLICATE KEY(id)
83+
DISTRIBUTED BY HASH(id) BUCKETS 2
84+
PROPERTIES (
85+
"replication_allocation" = "tag.location.default: 1"
86+
);
87+
88+
INSERT INTO test_array_decimal_table VALUES
89+
(1, 'Laptop', ['Electronics', 'Office', 'High-End', 'Laptop'], 5999.99, [1, 2, 3], 1),
90+
(2, 'Mechanical Keyboard', ['Electronics', 'Accessories'], 399.99, [1, 2], 1),
91+
(3, 'Basketball', ['Sports', 'Outdoor'], 199.99, [1,3], 1),
92+
(4, 'Badminton Racket', ['Sports', 'Equipment'], 299.99, [3], 3),
93+
(5, 'Shirt', ['Clothing', 'Office', 'Shirt'], 259.00, [4], 4);
94+
95+
INSERT INTO test_array_decimal_table VALUES
96+
(1, 'Laptop', ['Electronics', 'Office', 'High-End', 'Laptop'], 5999.99, [1, 2, 3], 2),
97+
(2, 'Mechanical Keyboard', ['Electronics', 'Accessories'], 399.99, [1, 2], 2),
98+
(3, 'Basketball', ['Sports', 'Outdoor'], 199.99, [1,3], 3);
99+
100+
INSERT INTO test_array_decimal_table VALUES
101+
(1, 'Laptop', ['Electronics', 'Office', 'High-End', 'Laptop'], 5999.99, [1, 2, 3], 3);
102+
"""
103+
104+
qt_window_function_partition_by_array_decimal """
105+
select
106+
name,
107+
category_ids,
108+
cid,
109+
sum(cid) over(partition by category_ids)
110+
from
111+
test_array_decimal_table
112+
group by
113+
name, category_ids, cid
114+
order by
115+
name, category_ids, cid;
116+
"""
117+
}

0 commit comments

Comments
 (0)