Merge branch 'main' into dsql-async-index

Author: amaksimo

AGENTS.md

@@ -0,0 +1,41 @@
+# Extensible SQL Lexer and Parser for Rust Agents Guidelines
+
+## General Agent Workflow
+1. You will write unit tests to ensure your code change is working as expected.
+2. You will run the commands in the Pre Commit Checks section below to ensure your change is ready for a pull request.
+3. When instructed to open a PR, you will follow the instructions in the Pull Request Guidelines section below.
+
+## General Coding Guidelines
+1. Refrain from adding conditions on specific dialects, such as `dialect_is!(...)` or `dialect_of!(... | ...)`. Instead, define a new function in the `Dialect` trait that describes the condition, so that dialects can turn this condition on more easily.
+2. Make targeted code changes and refrain from refactoring, unless it's absolutely required.
+
+## Unit Tests Guidelines
+- New unit tests should be added to the `tests` module in the corresponding dialect file (e.g., `tests/sqlparser_redshift.rs` for Redshift), and should be placed at the end of the file.
+- If the new functionality is gated using a dialect function, and the SQL is likely relevant in most dialects, tests should be placed under `tests/sqlparser_common.rs`.
+- When testing a multi-line SQL statement, use a raw string literal, i.e. `r#"..."#` to preserve formatting.
+- The parser builds an abstract syntax tree (AST) from the SQL statement and has functionality to display the tree as SQL. Use the following template for simple unit tests where you expect the SQL created from the AST to be the same as the input SQL:
+```rust
+<dialect>().verified_stmt(r#"..."#);
+```
+For example: `snowflake().verified_stmt(r#"SELECT * FROM my_table"#)`. Use `one_statement_parses_to` instead of `verified_stmt` when you expect the SQL created by the AST to differ than the input SQL. For example:
+```rust
+snowflake().one_statement_parses_to(
+    "SELECT * FROM my_table t",
+    "SELECT * FROM my_table AS t",
+)
+```
+
+## Analyzing Parsing Issues
+You can try to simplify the SQL statement to identify the root cause of the parsing issue. This may involve removing certain clauses or components of the SQL statement to see if it can be parsed successfully. Additionally, you can compare the problematic SQL statement with similar statements that are parsed correctly to identify any differences that may be causing the issue.
+
+## Pre Commit Checks
+Run the following commands before you commit to ensure the change will pass the CI process:
+```bash
+cargo test --all-features
+cargo fmt --all
+cargo clippy --all-targets --all-features -- -D warnings
+```
+
+## Pull Request Guidelines
+1. PR title should follow this format: `<DIALECT>: <SHORT DESCRIPTION>`, For example, `Showflake: Add support for casting to VARIANT`.
+2. Make the PR comment short, provide an example of what was not working and a short description of the fix. Be succint.

dev/release/rat_exclude_files.txt

@@ -6,3 +6,4 @@ dev/release/rat_exclude_files.txt
 sqlparser_bench/img/flamegraph.svg
 **Cargo.lock
 filtered_rat.txt
+AGENTS.md

src/ast/ddl.rs

@@ -3207,6 +3207,7 @@ impl fmt::Display for CreateTable {
                 Some(HiveIOFormat::FileFormat { format }) if !self.external => {
                     write!(f, " STORED AS {format}")?
                 }
+                Some(HiveIOFormat::Using { format }) => write!(f, " USING {format}")?,
                 _ => (),
             }
             if let Some(serde_properties) = serde_properties.as_ref() {

src/ast/mod.rs

@@ -30,7 +30,7 @@ use helpers::{
 };
 
 use core::cmp::Ordering;
-use core::ops::Deref;
+use core::ops::{Deref, DerefMut};
 use core::{
     fmt::{self, Display},
     hash,
@@ -200,6 +200,45 @@ fn format_statement_list(f: &mut fmt::Formatter, statements: &[Statement]) -> fm
     write!(f, ";")
 }
 
+/// A item `T` enclosed in a pair of parentheses
+#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
+pub struct Parens<T> {
+    /// the opening parenthesis token, i.e. `(`
+    pub opening_token: AttachedToken,
+    /// content enclosed in parentheses
+    pub content: T,
+    /// the closing parenthesis token, i.e. `)`
+    pub closing_token: AttachedToken,
+}
+
+impl<T> Parens<T> {
+    /// Constructor wrapping `content` into `Parens` with an empty span;
+    /// useful for testing purposes.
+    pub fn with_empty_span(content: T) -> Self {
+        Self {
+            opening_token: AttachedToken::empty(),
+            content,
+            closing_token: AttachedToken::empty(),
+        }
+    }
+}
+
+impl<T> Deref for Parens<T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        &self.content
+    }
+}
+
+impl<T> DerefMut for Parens<T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.content
+    }
+}
+
 /// An identifier, decomposed into its value or character data and the quote style.
 #[derive(Debug, Clone)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
@@ -4109,6 +4148,17 @@ pub enum Statement {
         show_options: ShowStatementOptions,
     },
     /// ```sql
+    /// SHOW CATALOGS
+    /// ```
+    ShowCatalogs {
+        /// `true` when terse output format was requested.
+        terse: bool,
+        /// `true` when history information was requested.
+        history: bool,
+        /// Additional options for `SHOW CATALOGS`.
+        show_options: ShowStatementOptions,
+    },
+    /// ```sql
     /// SHOW DATABASES
     /// ```
     ShowDatabases {
@@ -5744,6 +5794,19 @@ impl fmt::Display for Statement {
                 )?;
                 Ok(())
             }
+            Statement::ShowCatalogs {
+                terse,
+                history,
+                show_options,
+            } => {
+                write!(
+                    f,
+                    "SHOW {terse}CATALOGS{history}{show_options}",
+                    terse = if *terse { "TERSE " } else { "" },
+                    history = if *history { " HISTORY" } else { "" },
+                )?;
+                Ok(())
+            }
             Statement::ShowProcessList { full } => {
                 write!(
                     f,
@@ -8634,6 +8697,15 @@ pub enum HiveIOFormat {
         /// The file format used for storage.
         format: FileFormat,
     },
+    /// `USING <format>` syntax used by Spark SQL.
+    ///
+    /// Example: `CREATE TABLE t (i INT) USING PARQUET`
+    ///
+    /// See <https://spark.apache.org/docs/latest/sql-ref-syntax-ddl-create-table-datasource.html>
+    Using {
+        /// The data source or format name, e.g. `parquet`, `delta`, `csv`.
+        format: Ident,
+    },
 }
 
 #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Default)]

src/ast/query.rs

@@ -1522,6 +1522,8 @@ pub enum TableFactor {
         name: ObjectName,
         /// Arguments passed to the function.
         args: Vec<FunctionArg>,
+        /// Whether `WITH ORDINALITY` was specified to include ordinality.
+        with_ordinality: bool,
         /// Optional alias for the result of the function.
         alias: Option<TableAlias>,
     },
@@ -2277,13 +2279,17 @@ impl fmt::Display for TableFactor {
                 lateral,
                 name,
                 args,
+                with_ordinality,
                 alias,
             } => {
                 if *lateral {
                     write!(f, "LATERAL ")?;
                 }
                 write!(f, "{name}")?;
                 write!(f, "({})", display_comma_separated(args))?;
+                if *with_ordinality {
+                    write!(f, " WITH ORDINALITY")?;
+                }
                 if let Some(alias) = alias {
                     write!(f, " {alias}")?;
                 }
@@ -2527,6 +2533,12 @@ pub struct TableAlias {
     pub name: Ident,
     /// Optional column aliases declared in parentheses after the table alias.
     pub columns: Vec<TableAliasColumnDef>,
+    /// Optional PartiQL index alias declared with `AT`. For example:
+    /// ```sql
+    /// SELECT element, index FROM bar AS b, b.data.scalar_array AS element AT index
+    /// ```
+    /// See: <https://docs.aws.amazon.com/redshift/latest/dg/query-super.html>
+    pub at: Option<Ident>,
 }
 
 impl fmt::Display for TableAlias {
@@ -2535,6 +2547,9 @@ impl fmt::Display for TableAlias {
         if !self.columns.is_empty() {
             write!(f, " ({})", display_comma_separated(&self.columns))?;
         }
+        if let Some(at) = &self.at {
+            write!(f, " AT {at}")?;
+        }
         Ok(())
     }
 }
@@ -2770,6 +2785,13 @@ impl fmt::Display for Join {
                 self.relation,
                 suffix(constraint)
             )),
+            JoinOperator::ArrayJoin => f.write_fmt(format_args!("ARRAY JOIN {}", self.relation)),
+            JoinOperator::LeftArrayJoin => {
+                f.write_fmt(format_args!("LEFT ARRAY JOIN {}", self.relation))
+            }
+            JoinOperator::InnerArrayJoin => {
+                f.write_fmt(format_args!("INNER ARRAY JOIN {}", self.relation))
+            }
         }
     }
 }
@@ -2824,6 +2846,14 @@ pub enum JoinOperator {
     ///
     /// See <https://dev.mysql.com/doc/refman/8.4/en/join.html>.
     StraightJoin(JoinConstraint),
+    /// ClickHouse: `ARRAY JOIN` for unnesting arrays inline.
+    ///
+    /// See <https://clickhouse.com/docs/en/sql-reference/statements/select/array-join>.
+    ArrayJoin,
+    /// ClickHouse: `LEFT ARRAY JOIN` for unnesting arrays inline (preserves rows with empty arrays).
+    LeftArrayJoin,
+    /// ClickHouse: `INNER ARRAY JOIN` for unnesting arrays inline (filters rows with empty arrays).
+    InnerArrayJoin,
 }
 
 #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)]
@@ -3628,7 +3658,7 @@ pub struct Values {
     /// <https://dev.mysql.com/doc/refman/9.2/en/insert.html>
     pub value_keyword: bool,
     /// The list of rows, each row is a list of expressions.
-    pub rows: Vec<Vec<Expr>>,
+    pub rows: Vec<Parens<Vec<Expr>>>,
 }
 
 impl fmt::Display for Values {

src/ast/spans.rs

@@ -41,13 +41,13 @@ use super::{
     MatchRecognizePattern, Measure, Merge, MergeAction, MergeClause, MergeInsertExpr,
     MergeInsertKind, MergeUpdateExpr, NamedParenthesizedList, NamedWindowDefinition, ObjectName,
     ObjectNamePart, Offset, OnConflict, OnConflictAction, OnInsert, OpenStatement, OrderBy,
-    OrderByExpr, OrderByKind, OutputClause, Partition, PartitionBoundValue, PivotValueSource,
-    ProjectionSelect, Query, RaiseStatement, RaiseStatementValue, ReferentialAction,
-    RenameSelectItem, ReplaceSelectElement, ReplaceSelectItem, Select, SelectInto, SelectItem,
-    SetExpr, SqlOption, Statement, Subscript, SymbolDefinition, TableAlias, TableAliasColumnDef,
-    TableConstraint, TableFactor, TableObject, TableOptionsClustered, TableWithJoins, Update,
-    UpdateTableFromKind, Use, Values, ViewColumnDef, WhileStatement, WildcardAdditionalOptions,
-    With, WithFill,
+    OrderByExpr, OrderByKind, OutputClause, Parens, Partition, PartitionBoundValue,
+    PivotValueSource, ProjectionSelect, Query, RaiseStatement, RaiseStatementValue,
+    ReferentialAction, RenameSelectItem, ReplaceSelectElement, ReplaceSelectItem, Select,
+    SelectInto, SelectItem, SetExpr, SqlOption, Statement, Subscript, SymbolDefinition, TableAlias,
+    TableAliasColumnDef, TableConstraint, TableFactor, TableObject, TableOptionsClustered,
+    TableWithJoins, Update, UpdateTableFromKind, Use, Values, ViewColumnDef, WhileStatement,
+    WildcardAdditionalOptions, With, WithFill,
 };
 
 /// Given an iterator of spans, return the [Span::union] of all spans.
@@ -106,6 +106,12 @@ impl Spanned for TokenWithSpan {
     }
 }
 
+impl<T> Spanned for Parens<T> {
+    fn span(&self) -> Span {
+        self.opening_token.0.span.union(&self.closing_token.0.span)
+    }
+}
+
 impl Spanned for Query {
     fn span(&self) -> Span {
         let Query {
@@ -239,10 +245,11 @@ impl Spanned for Values {
             rows,
         } = self;
 
-        union_spans(
-            rows.iter()
-                .map(|row| union_spans(row.iter().map(|expr| expr.span()))),
-        )
+        match &rows[..] {
+            [] => Span::empty(),
+            [f] => f.span(),
+            [f, .., l] => f.span().union(&l.span()),
+        }
     }
 }
 
@@ -478,6 +485,7 @@ impl Spanned for Statement {
             Statement::AlterConnector { .. } => Span::empty(),
             Statement::DropPolicy { .. } => Span::empty(),
             Statement::DropConnector { .. } => Span::empty(),
+            Statement::ShowCatalogs { .. } => Span::empty(),
             Statement::ShowDatabases { .. } => Span::empty(),
             Statement::ShowProcessList { .. } => Span::empty(),
             Statement::ShowSchemas { .. } => Span::empty(),
@@ -1990,6 +1998,7 @@ impl Spanned for TableFactor {
                 lateral: _,
                 name,
                 args,
+                with_ordinality: _,
                 alias,
             } => union_spans(
                 name.0
@@ -2178,8 +2187,13 @@ impl Spanned for TableAlias {
             explicit: _,
             name,
             columns,
+            at,
         } = self;
-        union_spans(core::iter::once(name.span).chain(columns.iter().map(Spanned::span)))
+        union_spans(
+            core::iter::once(name.span)
+                .chain(columns.iter().map(Spanned::span))
+                .chain(at.iter().map(|at| at.span)),
+        )
     }
 }
 
@@ -2239,6 +2253,9 @@ impl Spanned for JoinOperator {
             JoinOperator::Anti(join_constraint) => join_constraint.span(),
             JoinOperator::Semi(join_constraint) => join_constraint.span(),
             JoinOperator::StraightJoin(join_constraint) => join_constraint.span(),
+            JoinOperator::ArrayJoin => Span::empty(),
+            JoinOperator::LeftArrayJoin => Span::empty(),
+            JoinOperator::InnerArrayJoin => Span::empty(),
         }
     }
 }

src/dialect/clickhouse.rs

@@ -64,6 +64,14 @@ impl Dialect for ClickHouseDialect {
         true
     }
 
+    fn supports_partition_by_after_order_by(&self) -> bool {
+        true
+    }
+
+    fn supports_array_join_syntax(&self) -> bool {
+        true
+    }
+
     // ClickHouse uses this for some FORMAT expressions in `INSERT` context, e.g. when inserting
     // with FORMAT JSONEachRow a raw JSON key-value expression is valid and expected.
     //

src/dialect/generic.rs

@@ -45,6 +45,14 @@ impl Dialect for GenericDialect {
         true
     }
 
+    fn supports_partition_by_after_order_by(&self) -> bool {
+        true
+    }
+
+    fn supports_array_join_syntax(&self) -> bool {
+        true
+    }
+
     fn supports_group_by_expr(&self) -> bool {
         true
     }

src/dialect/hive.rs

@@ -72,4 +72,11 @@ impl Dialect for HiveDialect {
     fn supports_group_by_with_modifier(&self) -> bool {
         true
     }
+
+    // TODO: The parsing of the FROM keyword seems wrong, as it happens within the CTE.
+    // See https://github.com/apache/datafusion-sqlparser-rs/issues/2236 for more details.
+    /// See <https://hive.apache.org/docs/latest/language/common-table-expression/>
+    fn supports_from_first_insert(&self) -> bool {
+        true
+    }
 }