branch-3.1: [fix](nereids) allow merge project for java udf (#57102)

yujun777 · web-flow · commit 5ed06bbcc943 · 2025-10-22T10:28:50.000+08:00
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PushDownFilterThroughProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PushDownFilterThroughProject.java
@@ -45,7 +45,7 @@ public Plan visitPhysicalFilter(PhysicalFilter<? extends Plan> filter, CascadesC
         PhysicalProject<? extends Plan> project = (PhysicalProject<? extends Plan>) child;
         Map<Slot, Expression> childAlias = project.getAliasToProducer();
         if (filter.getInputSlots().stream().map(childAlias::get).filter(Objects::nonNull)
-                .anyMatch(Expression::containsNonfoldable)) {
+                .anyMatch(Expression::containsUniqueFunction)) {
             return filter;
         }
         PhysicalFilter<? extends Plan> newFilter = filter.withConjunctsAndChild(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyConflictCompound.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyConflictCompound.java
@@ -60,7 +60,7 @@ private Expression rewrite(CompoundPredicate compoundPredicate) {
         // ie, predicate contains expression 'expression' and 'not expression'
         Map<Expression, Pair<Boolean, Boolean>> exprExistMarks = Maps.newHashMap();
         for (Expression child : flatten) {
-            if (!child.containsNonfoldable()) {
+            if (!child.containsUniqueFunction()) {
                 if (child instanceof CompoundPredicate) {
                     Expression newChild = rewrite((CompoundPredicate) child);
                     if (!child.equals(newChild)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughProject.java
@@ -113,14 +113,14 @@ private static Pair<Set<Expression>, Set<Expression>> splitConjunctsByChildOutpu
         Set<Expression> remainPredicates = Sets.newLinkedHashSet();
         for (Expression conjunct : conjuncts) {
             Set<Slot> conjunctSlots = conjunct.getInputSlots();
-            // If filter contains non-foldable expression, it can push down, for example:
+            // If filter contains unique function, it can push down, for example:
             // `filter(a + random(1, 10) > 1) -> project(a)` => `project(a) -> filter(a + random(1, 10) > 1)`.
-            // If filter slot is alias and its expression contains non-foldable expression, it can't push down, example:
+            // If filter slot is alias and its expression contains unique function, it can't push down, example:
             // `filter(a > 1) -> project(b + random(1, 10) as a)`, if push down filter, it got
             // `project(b + random(1, 10) as a) -> filter(b + random(1, 10) > 1)`, it contains two distinct RANDOM.
             if (childOutputs.containsAll(conjunctSlots)
                     && conjunctSlots.stream().map(childAlias::get).filter(Objects::nonNull)
-                            .noneMatch(Expression::containsNonfoldable)) {
+                            .noneMatch(Expression::containsUniqueFunction)) {
                 pushDownPredicates.add(conjunct);
             } else {
                 remainPredicates.add(conjunct);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
@@ -28,6 +28,7 @@
 import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
 import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.UniqueFunction;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
 import org.apache.doris.nereids.trees.expressions.shape.LeafExpression;
@@ -422,6 +423,10 @@ public boolean isKeyColumnFromTable() {
                 && ((SlotReference) this).getOriginalColumn().get().isKey();
     }
 
+    public boolean containsUniqueFunction() {
+        return containsType(UniqueFunction.class);
+    }
+
     /** containsNullLiteralChildren */
     public boolean containsNullLiteralChildren() {
         return getOrInitMutableState("CONTAINS_NULL_LITERAL_CHILDREN", () -> {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java
@@ -73,15 +73,13 @@ default String toSql() throws UnboundException {
         throw new UnboundException("sql");
     }
 
-    default boolean foldable() {
-        return true;
-    }
-
     /**
-     * Identify the expression is containing non-foldable expr or not
+     * foldable() mainly use in fold expression. Udf and UniqueFunction are not foldable.
+     * But if want to check an expression contains non-idempotent, such as `rand()`, `uuid()`, etc.,
+     * you should use Expression::containsUniqueFunction instead.
      */
-    default boolean containsNonfoldable() {
-        return anyMatch(expr -> !((ExpressionTrait) expr).foldable());
+    default boolean foldable() {
+        return true;
     }
 
     /**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Random.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Random.java
@@ -35,7 +35,7 @@
  * ScalarFunction 'random'. This class is generated by GenerateFunction.
  */
 public class Random extends ScalarFunction
-        implements ExplicitlyCastableSignature {
+        implements ExplicitlyCastableSignature, UniqueFunction {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(DoubleType.INSTANCE).args(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RandomBytes.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RandomBytes.java
@@ -35,7 +35,7 @@
  * ScalarFunction 'random_bytes'. This class is generated by GenerateFunction.
  */
 public class RandomBytes extends ScalarFunction
-        implements ExplicitlyCastableSignature, PropagateNullable {
+        implements ExplicitlyCastableSignature, PropagateNullable, UniqueFunction {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(StringType.INSTANCE).args(IntegerType.INSTANCE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UniqueFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UniqueFunction.java
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.scalar;
+
+/**
+ * Unique function include random like functions. Two calls to the function with the same arguments
+ * will return different results.
+ */
+public interface UniqueFunction {
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Uuid.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Uuid.java
@@ -32,7 +32,7 @@
  * ScalarFunction 'uuid'. This class is generated by GenerateFunction.
  */
 public class Uuid extends ScalarFunction
-        implements LeafExpression, ExplicitlyCastableSignature, AlwaysNotNullable {
+        implements LeafExpression, ExplicitlyCastableSignature, AlwaysNotNullable, UniqueFunction {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UuidNumeric.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UuidNumeric.java
@@ -32,7 +32,7 @@
  * ScalarFunction 'uuid_numeric'. This class is generated by GenerateFunction.
  */
 public class UuidNumeric extends ScalarFunction
-        implements LeafExpression, ExplicitlyCastableSignature, AlwaysNotNullable {
+        implements LeafExpression, ExplicitlyCastableSignature, AlwaysNotNullable, UniqueFunction {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.ret(LargeIntType.INSTANCE).args()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
@@ -291,7 +291,7 @@ public void computeFd(DataTrait.Builder builder) {
                 continue;
             }
             // a+random(1,10) should continue, otherwise the a(determinant), a+random(1,10) (dependency) will be added.
-            if (expr.containsNonfoldable()) {
+            if (expr.containsUniqueFunction()) {
                 continue;
             }
             builder.addDeps(expr.getInputSlots(), ImmutableSet.of(expr.toSlot()));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java
@@ -154,24 +154,24 @@ public static List<Expression> replaceExpressionByProjections(List<NamedExpressi
     /**
      * replace targetExpressions with project.
      * if the target expression contains a slot which is an alias and its origin expression contains
-     * non-foldable expression and the slot exits multiple times, then can not replace.
+     * unique function and the slot exits multiple times, then can not replace.
      * for example, target expressions: [a, a + 10],  child project: [ t + random() as a ],
      * if replace with the projects, then result expressions: [ t + random(),  t + random() + 10 ],
      * it will calculate random two times, this is error.
      */
     public static boolean canReplaceWithProjections(List<NamedExpression> childProjects,
             List<? extends Expression> targetExpressions) {
-        Set<Slot> nonfoldableSlots = ExpressionUtils.generateReplaceMap(childProjects).entrySet().stream()
-                .filter(entry -> entry.getValue().containsNonfoldable())
+        Set<Slot> uniqueFunctionSlots = ExpressionUtils.generateReplaceMap(childProjects).entrySet().stream()
+                .filter(entry -> entry.getValue().containsUniqueFunction())
                 .map(Entry::getKey)
                 .collect(Collectors.toSet());
-        if (nonfoldableSlots.isEmpty()) {
+        if (uniqueFunctionSlots.isEmpty()) {
             return true;
         }
 
         Set<Slot> counterSet = Sets.newHashSet();
         return targetExpressions.stream().noneMatch(target -> target.anyMatch(
-                e -> (e instanceof Slot) && nonfoldableSlots.contains(e) && !counterSet.add((Slot) e)));
+                e -> (e instanceof Slot) && uniqueFunctionSlots.contains(e) && !counterSet.add((Slot) e)));
     }
 
     public static Plan skipProjectFilterLimit(Plan plan) {
diff --git a/regression-test/data/nereids_p0/javaudf/test_merge_project_with_javaudf.out b/regression-test/data/nereids_p0/javaudf/test_merge_project_with_javaudf.out
@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_shape --
+PhysicalResultSink
+--PhysicalProject[(func_test_merge_project_with_javaudf(a) + 10) AS `k + 10`, (func_test_merge_project_with_javaudf(a) + 20) AS `k + 20`]
+----PhysicalOlapScan[tbl_test_merge_project_with_javaudf]
+
diff --git a/regression-test/suites/nereids_p0/javaudf/test_merge_project_with_javaudf.groovy b/regression-test/suites/nereids_p0/javaudf/test_merge_project_with_javaudf.groovy
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite('test_merge_project_with_javaudf') {
+    def jarPath = """${context.file.parent}/../../javaudf_p0/jars/java-udf-case-jar-with-dependencies.jar"""
+    sql """
+        SET runtime_filter_mode=OFF;
+        SET enable_fallback_to_original_planner=false;
+        SET ignore_shape_nodes='PhysicalDistribute';
+        SET ignore_shape_nodes='PhysicalDistribute';
+        SET detail_shape_nodes='PhysicalProject';
+
+        drop table if exists tbl_test_merge_project_with_javaudf force;
+
+        create table tbl_test_merge_project_with_javaudf(a int)
+            distributed by hash(a) buckets 2 properties('replication_num' = '1');
+
+        insert into tbl_test_merge_project_with_javaudf values(1), (2), (3);
+
+        drop function if exists func_test_merge_project_with_javaudf(int);
+
+        CREATE FUNCTION func_test_merge_project_with_javaudf(int) RETURNS int PROPERTIES (
+            "file"="file://${jarPath}",
+            "symbol"="org.apache.doris.udf.IntTest",
+            "type"="JAVA_UDF"
+        );
+    """
+
+    def sqlStr = '''
+        select k + 10, k + 20
+        from (
+            select func_test_merge_project_with_javaudf(a) as k
+            from tbl_test_merge_project_with_javaudf
+            ) t
+    '''
+
+    qt_select_shape "explain shape plan ${sqlStr}"
+
+    // explain no exception
+    explain {
+        sql sqlStr
+    }
+
+    sql """
+        drop function if exists func_test_merge_project_with_javaudf(int);
+        drop table if exists tbl_test_merge_project_with_javaudf force;
+    """
+}

Original file line number	Diff line number	Diff line change
`@@ -45,7 +45,7 @@ public Plan visitPhysicalFilter(PhysicalFilter<? extends Plan> filter, CascadesC`
`45`	`45`	`PhysicalProject<? extends Plan> project = (PhysicalProject<? extends Plan>) child;`
`46`	`46`	`Map<Slot, Expression> childAlias = project.getAliasToProducer();`
`47`	`47`	`if (filter.getInputSlots().stream().map(childAlias::get).filter(Objects::nonNull)`
`48`		`- .anyMatch(Expression::containsNonfoldable)) {`
	`48`	`+ .anyMatch(Expression::containsUniqueFunction)) {`
`49`	`49`	`return filter;`
`50`	`50`	`}`
`51`	`51`	`PhysicalFilter<? extends Plan> newFilter = filter.withConjunctsAndChild(`