IGNITE-27700 SQL Calcite: Fix merge filter to scan after trimmed fields - Fixes #12675.

Signed-off-by: Aleksey Plekhanov <plehanov.alex@gmail.com>
(cherry picked from commit 77987ea)

Author: alex-plekhanov

modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/IgniteScalarFunction.java

@@ -26,28 +26,41 @@
  * Implementation of {@link ScalarFunction} for Ignite user defined functions.
  */
 public class IgniteScalarFunction extends IgniteReflectiveFunctionBase implements ScalarFunction {
+    /** */
+    private final boolean deterministic;
+
     /**
      * Private constructor.
      */
-    private IgniteScalarFunction(Method method, CallImplementor implementor) {
+    private IgniteScalarFunction(Method method, CallImplementor implementor, boolean deterministic) {
         super(method, implementor);
+
+        this.deterministic = deterministic;
     }
 
     /**
      * Creates {@link ScalarFunction} from given method.
      *
      * @param method Method that is used to implement the function.
+     * @param deterministic Is function deterministic.
      * @return Created {@link ScalarFunction}.
      */
-    public static ScalarFunction create(Method method) {
+    public static ScalarFunction create(Method method, boolean deterministic) {
         CallImplementor implementor = RexImpTable.createImplementor(
             new ReflectiveCallNotNullImplementor(method), NullPolicy.NONE, false);
 
-        return new IgniteScalarFunction(method, implementor);
+        return new IgniteScalarFunction(method, implementor, deterministic);
     }
 
     /** {@inheritDoc} */
     @Override public RelDataType getReturnType(RelDataTypeFactory typeFactory) {
         return typeFactory.createJavaType(method.getReturnType());
     }
+
+    /**
+     * @return Deterministic flag.
+     */
+    public boolean isDeterministic() {
+        return deterministic;
+    }
 }

modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/exp/RexExecutorImpl.java

@@ -19,7 +19,6 @@
 import java.lang.reflect.Modifier;
 import java.lang.reflect.Type;
 import java.util.List;
-
 import com.google.common.collect.ImmutableList;
 import org.apache.calcite.DataContext;
 import org.apache.calcite.adapter.java.JavaTypeFactory;
@@ -35,14 +34,19 @@
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
 import org.apache.calcite.rex.RexExecutable;
 import org.apache.calcite.rex.RexExecutor;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexProgram;
 import org.apache.calcite.rex.RexProgramBuilder;
+import org.apache.calcite.rex.RexVisitor;
+import org.apache.calcite.rex.RexVisitorImpl;
 import org.apache.calcite.sql.validate.SqlConformance;
 import org.apache.calcite.sql.validate.SqlConformanceEnum;
+import org.apache.calcite.sql.validate.SqlUserDefinedFunction;
 import org.apache.calcite.util.BuiltInMethod;
+import org.apache.calcite.util.ControlFlowException;
 import org.apache.calcite.util.Util;
 import org.apache.ignite.internal.processors.query.calcite.type.IgniteCustomType;
 
@@ -133,12 +137,34 @@ public static RexExecutable getExecutable(RexBuilder rexBuilder, List<RexNode> e
      * Do constant reduction using generated code.
      */
     @Override public void reduce(RexBuilder rexBuilder, List<RexNode> constExps, List<RexNode> reducedValues) {
+        RexVisitor<Void> nonDeterministicUdfFinder = new RexVisitorImpl<Void>(true) {
+            @Override public Void visitCall(RexCall call) {
+                if (call.getOperator() instanceof SqlUserDefinedFunction) {
+                    SqlUserDefinedFunction udfFunc = (SqlUserDefinedFunction)call.getOperator();
+
+                    if (udfFunc.getFunction() instanceof IgniteScalarFunction
+                        && !((IgniteScalarFunction)udfFunc.getFunction()).isDeterministic())
+                        throw Util.FoundOne.NULL; // Don't reduce non-deterministic UDF functions.
+                }
+
+                return super.visitCall(call);
+            }
+        };
+
         for (RexNode node : constExps) {
             // Do not simplify custom types, since we can't convert it to literal of this type.
             if (node.getType() instanceof IgniteCustomType) {
                 reducedValues.addAll(constExps);
                 return;
             }
+
+            try {
+                node.accept(nonDeterministicUdfFinder);
+            }
+            catch (ControlFlowException foundNonDeterministic) {
+                reducedValues.addAll(constExps);
+                return;
+            }
         }
         final String code = compile(rexBuilder, constExps,
             (list, index, storageType) -> {

modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlannerHelper.java

@@ -127,10 +127,10 @@ public static IgniteRel optimize(SqlNode sqlNode, IgnitePlanner planner, IgniteL
 
             rel = planner.extractConjunctionOverDisjunctionCommonPart(rel);
 
-            rel = planner.trimUnusedFields(root.withRel(rel)).rel;
-
             rel = planner.transform(PlannerPhase.HEP_FILTER_PUSH_DOWN, rel.getTraitSet(), rel);
 
+            rel = planner.trimUnusedFields(root.withRel(rel)).rel;
+
             // The following pushed down project can erase top-level hints. We store them to reassign hints for join nodes.
             // Clear the inherit pathes to consider the hints as not propogated ones.
             List<RelHint> topHints = HintUtils.allRelHints(rel).stream().map(h -> h.inheritPath.isEmpty()

modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlannerPhase.java

@@ -108,6 +108,7 @@ public enum PlannerPhase {
         @Override public RuleSet getRules(PlanningContext ctx) {
             return ctx.rules(
                 RuleSets.ofList(
+                    CoreRules.FILTER_REDUCE_EXPRESSIONS,
                     FilterScanMergeRule.TABLE_SCAN_SKIP_CORRELATED,
 
                     CoreRules.FILTER_MERGE,

modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rel/AbstractIndexScan.java

@@ -27,6 +27,7 @@
 import org.apache.calcite.rel.RelInput;
 import org.apache.calcite.rel.RelWriter;
 import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexUtil;
@@ -67,12 +68,13 @@ protected AbstractIndexScan(
         RelTraitSet traitSet,
         RelOptTable table,
         String idxName,
+        @Nullable RelDataType rowType,
         @Nullable List<RexNode> proj,
         @Nullable RexNode cond,
         @Nullable List<SearchBounds> searchBounds,
         @Nullable ImmutableBitSet reqColumns
     ) {
-        super(cluster, traitSet, Collections.emptyList(), table, proj, cond, reqColumns);
+        super(cluster, traitSet, Collections.emptyList(), table, rowType, proj, cond, reqColumns);
 
         this.idxName = idxName;
         this.searchBounds = searchBounds;

modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rel/IgniteIndexScan.java

@@ -24,6 +24,7 @@
 import org.apache.calcite.rel.RelCollation;
 import org.apache.calcite.rel.RelInput;
 import org.apache.calcite.rel.RelWriter;
+import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.util.ImmutableBitSet;
 import org.apache.ignite.internal.processors.query.calcite.prepare.bounds.SearchBounds;
@@ -64,6 +65,7 @@ public IgniteIndexScan(RelInput input) {
      * @param traits Traits of this relational expression
      * @param tbl Table definition.
      * @param idxName Index name.
+     * @param rowType Row type.
      * @param proj Projects.
      * @param cond Filters.
      * @param requiredCols Participating columns.
@@ -74,13 +76,14 @@ public IgniteIndexScan(
         RelTraitSet traits,
         RelOptTable tbl,
         String idxName,
+        @Nullable RelDataType rowType,
         @Nullable List<RexNode> proj,
         @Nullable RexNode cond,
         @Nullable List<SearchBounds> searchBounds,
         @Nullable ImmutableBitSet requiredCols,
         RelCollation collation
     ) {
-        this(-1L, cluster, traits, tbl, idxName, proj, cond, searchBounds, requiredCols, collation);
+        this(-1L, cluster, traits, tbl, idxName, rowType, proj, cond, searchBounds, requiredCols, collation);
     }
 
     /**
@@ -89,6 +92,7 @@ public IgniteIndexScan(
      * @param traits Traits of this relational expression
      * @param tbl Table definition.
      * @param idxName Index name.
+     * @param rowType Row type.
      * @param proj Projects.
      * @param cond Filters.
      * @param requiredCols Participating colunms.
@@ -100,13 +104,14 @@ private IgniteIndexScan(
         RelTraitSet traits,
         RelOptTable tbl,
         String idxName,
+        @Nullable RelDataType rowType,
         @Nullable List<RexNode> proj,
         @Nullable RexNode cond,
         @Nullable List<SearchBounds> searchBounds,
         @Nullable ImmutableBitSet requiredCols,
         RelCollation collation
     ) {
-        super(cluster, traits, tbl, idxName, proj, cond, searchBounds, requiredCols);
+        super(cluster, traits, tbl, idxName, rowType, proj, cond, searchBounds, requiredCols);
 
         this.sourceId = sourceId;
         this.collation = collation;
@@ -132,13 +137,13 @@ private IgniteIndexScan(
     /** {@inheritDoc} */
     @Override public IgniteRel clone(long sourceId) {
         return new IgniteIndexScan(sourceId, getCluster(), getTraitSet(), getTable(),
-            idxName, projects, condition, searchBounds, requiredColumns, collation);
+            idxName, rowType, projects, condition, searchBounds, requiredColumns, collation);
     }
 
     /** {@inheritDoc} */
     @Override public IgniteRel clone(RelOptCluster cluster, List<IgniteRel> inputs) {
         return new IgniteIndexScan(sourceId, cluster, getTraitSet(), getTable(),
-            idxName, projects, condition, searchBounds, requiredColumns, collation);
+            idxName, rowType, projects, condition, searchBounds, requiredColumns, collation);
     }
 
     /** {@inheritDoc} */

modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rel/IgniteTableScan.java

@@ -24,6 +24,7 @@
 import org.apache.calcite.plan.RelTraitSet;
 import org.apache.calcite.rel.RelInput;
 import org.apache.calcite.rel.RelWriter;
+import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.util.ImmutableBitSet;
 import org.jetbrains.annotations.Nullable;
@@ -63,14 +64,15 @@ public IgniteTableScan(
         RelTraitSet traits,
         RelOptTable tbl
     ) {
-        this(cluster, traits, tbl, null, null, null);
+        this(cluster, traits, tbl, null, null, null, null);
     }
 
     /**
      * Creates a TableScan.
      * @param cluster Cluster that this relational expression belongs to
      * @param traits Traits of this relational expression
      * @param tbl Table definition.
+     * @param rowType Row type.
      * @param proj Projects.
      * @param cond Filters.
      * @param requiredColunms Participating colunms.
@@ -79,18 +81,20 @@ public IgniteTableScan(
         RelOptCluster cluster,
         RelTraitSet traits,
         RelOptTable tbl,
+        @Nullable RelDataType rowType,
         @Nullable List<RexNode> proj,
         @Nullable RexNode cond,
         @Nullable ImmutableBitSet requiredColunms
     ) {
-        this(-1L, cluster, traits, tbl, proj, cond, requiredColunms);
+        this(-1L, cluster, traits, tbl, rowType, proj, cond, requiredColunms);
     }
 
     /**
      * Creates a TableScan.
      * @param cluster Cluster that this relational expression belongs to
      * @param traits Traits of this relational expression
      * @param tbl Table definition.
+     * @param rowType Row type.
      * @param proj Projects.
      * @param cond Filters.
      * @param requiredColunms Participating colunms.
@@ -100,11 +104,12 @@ private IgniteTableScan(
         RelOptCluster cluster,
         RelTraitSet traits,
         RelOptTable tbl,
+        @Nullable RelDataType rowType,
         @Nullable List<RexNode> proj,
         @Nullable RexNode cond,
         @Nullable ImmutableBitSet requiredColunms
     ) {
-        super(cluster, traits, ImmutableList.of(), tbl, proj, cond, requiredColunms);
+        super(cluster, traits, ImmutableList.of(), tbl, rowType, proj, cond, requiredColunms);
         this.sourceId = sourceId;
     }
 
@@ -126,11 +131,13 @@ private IgniteTableScan(
 
     /** {@inheritDoc} */
     @Override public IgniteRel clone(long sourceId) {
-        return new IgniteTableScan(sourceId, getCluster(), getTraitSet(), getTable(), projects, condition, requiredColumns);
+        return new IgniteTableScan(sourceId, getCluster(), getTraitSet(), getTable(), rowType, projects, condition,
+            requiredColumns);
     }
 
     /** {@inheritDoc} */
     @Override public IgniteRel clone(RelOptCluster cluster, List<IgniteRel> inputs) {
-        return new IgniteTableScan(sourceId, cluster, getTraitSet(), getTable(), projects, condition, requiredColumns);
+        return new IgniteTableScan(sourceId, cluster, getTraitSet(), getTable(), rowType, projects, condition,
+            requiredColumns);
     }
 }

modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rel/ProjectableFilterableTableScan.java

@@ -75,12 +75,16 @@ protected ProjectableFilterableTableScan(
         RelTraitSet traitSet,
         List<RelHint> hints,
         RelOptTable table,
+        @Nullable RelDataType rowType,
         @Nullable List<RexNode> proj,
         @Nullable RexNode cond,
         @Nullable ImmutableBitSet reqColumns
     ) {
         super(cluster, traitSet, hints, table);
 
+        assert proj == null || rowType != null : "rowType should be provided if project != null";
+
+        this.rowType = rowType;
         projects = proj;
         condition = cond;
         requiredColumns = reqColumns;
@@ -90,6 +94,7 @@ protected ProjectableFilterableTableScan(
     protected ProjectableFilterableTableScan(RelInput input) {
         super(input);
         condition = input.getExpression("filters");
+        rowType = input.get("rowType") == null ? null : input.getRowType("rowType");
         projects = input.get("projects") == null ? null : input.getExpressionList("projects");
         requiredColumns = input.get("requiredColumns") == null ? null : input.getBitSet("requiredColumns");
     }
@@ -129,6 +134,7 @@ protected RelWriter explainTerms0(RelWriter pw) {
         }
 
         return pw
+            .itemIf("rowType", rowType, projects != null) // Intentional project check here.
             .itemIf("projects", projects, projects != null)
             .itemIf("requiredColumns", requiredColumns, requiredColumns != null);
     }
@@ -151,10 +157,9 @@ protected RelWriter explainTerms0(RelWriter pw) {
 
     /** {@inheritDoc} */
     @Override public RelDataType deriveRowType() {
-        if (projects != null)
-            return RexUtil.createStructType(Commons.typeFactory(getCluster()), projects);
-        else
-            return table.unwrap(IgniteTable.class).getRowType(Commons.typeFactory(getCluster()), requiredColumns);
+        assert projects == null : "For merged projects rowType should be provided explicetely";
+
+        return table.unwrap(IgniteTable.class).getRowType(Commons.typeFactory(getCluster()), requiredColumns);
     }
 
     /** */

modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rel/logical/IgniteLogicalIndexScan.java

@@ -21,6 +21,7 @@
 import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.plan.RelOptTable;
 import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.util.ImmutableBitSet;
 import org.apache.ignite.internal.processors.query.calcite.prepare.bounds.SearchBounds;
@@ -37,6 +38,7 @@ public static IgniteLogicalIndexScan create(
         RelTraitSet traits,
         RelOptTable table,
         String idxName,
+        @Nullable RelDataType rowType,
         @Nullable List<RexNode> proj,
         @Nullable RexNode cond,
         @Nullable ImmutableBitSet requiredColumns
@@ -51,6 +53,7 @@ public static IgniteLogicalIndexScan create(
             traits,
             table,
             idxName,
+            rowType,
             proj,
             cond,
             searchBounds,
@@ -63,6 +66,7 @@ public static IgniteLogicalIndexScan create(
      * @param traits Traits of this relational expression
      * @param tbl Table definition.
      * @param idxName Index name.
+     * @param rowType Row type.
      * @param proj Projects.
      * @param cond Filters.
      * @param searchBounds Index search bounds.
@@ -73,11 +77,12 @@ private IgniteLogicalIndexScan(
         RelTraitSet traits,
         RelOptTable tbl,
         String idxName,
+        @Nullable RelDataType rowType,
         @Nullable List<RexNode> proj,
         @Nullable RexNode cond,
         @Nullable List<SearchBounds> searchBounds,
         @Nullable ImmutableBitSet requiredCols
     ) {
-        super(cluster, traits, tbl, idxName, proj, cond, searchBounds, requiredCols);
+        super(cluster, traits, tbl, idxName, rowType, proj, cond, searchBounds, requiredCols);
     }
 }