From 4df2961dd0403ecce5d39125a91c3c7e95de9be4 Mon Sep 17 00:00:00 2001 From: Corey Fritz Date: Tue, 21 Apr 2026 08:35:53 -0600 Subject: [PATCH 1/4] feat(bigquery): parse WITH CONNECTION on CREATE EXTERNAL TABLE --- src/ast/ddl.rs | 6 ++ src/ast/helpers/stmt_create_table.rs | 11 ++++ src/ast/spans.rs | 1 + src/parser/mod.rs | 25 ++++++++- tests/sqlparser_bigquery.rs | 84 ++++++++++++++++++++++++++++ tests/sqlparser_duckdb.rs | 1 + tests/sqlparser_mssql.rs | 2 + tests/sqlparser_postgres.rs | 1 + 8 files changed, 129 insertions(+), 2 deletions(-) diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index 67aefb392..c9facd28a 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -3021,6 +3021,9 @@ pub struct CreateTable { /// Snowflake "EXTERNAL_VOLUME" clause for Iceberg tables /// pub external_volume: Option, + /// BigQuery `WITH CONNECTION` clause for external tables + /// + pub with_connection: Option, /// Snowflake "BASE_LOCATION" clause for Iceberg tables /// pub base_location: Option, @@ -3250,6 +3253,9 @@ impl fmt::Display for CreateTable { if let Some(cluster_by) = self.cluster_by.as_ref() { write!(f, " CLUSTER BY {cluster_by}")?; } + if let Some(with_connection) = &self.with_connection { + write!(f, " WITH CONNECTION {with_connection}")?; + } if let options @ CreateTableOptions::Options(_) = &self.table_options { write!(f, " {options}")?; } diff --git a/src/ast/helpers/stmt_create_table.rs b/src/ast/helpers/stmt_create_table.rs index ab2feb693..21bf2ec5e 100644 --- a/src/ast/helpers/stmt_create_table.rs +++ b/src/ast/helpers/stmt_create_table.rs @@ -157,6 +157,9 @@ pub struct CreateTableBuilder { pub base_location: Option, /// Optional external volume identifier. pub external_volume: Option, + /// BigQuery `WITH CONNECTION` clause for external tables. + /// + pub with_connection: Option, /// Optional catalog name. pub catalog: Option, /// Optional catalog synchronization option. @@ -235,6 +238,7 @@ impl CreateTableBuilder { with_tags: None, base_location: None, external_volume: None, + with_connection: None, catalog: None, catalog_sync: None, storage_serialization_policy: None, @@ -488,6 +492,11 @@ impl CreateTableBuilder { self.external_volume = external_volume; self } + /// Set the BigQuery `WITH CONNECTION` clause for external tables. + pub fn with_connection(mut self, with_connection: Option) -> Self { + self.with_connection = with_connection; + self + } /// Set the catalog name for the table. pub fn catalog(mut self, catalog: Option) -> Self { self.catalog = catalog; @@ -605,6 +614,7 @@ impl CreateTableBuilder { with_tags: self.with_tags, base_location: self.base_location, external_volume: self.external_volume, + with_connection: self.with_connection, catalog: self.catalog, catalog_sync: self.catalog_sync, storage_serialization_policy: self.storage_serialization_policy, @@ -686,6 +696,7 @@ impl From for CreateTableBuilder { with_tags: table.with_tags, base_location: table.base_location, external_volume: table.external_volume, + with_connection: table.with_connection, catalog: table.catalog, catalog_sync: table.catalog_sync, storage_serialization_policy: table.storage_serialization_policy, diff --git a/src/ast/spans.rs b/src/ast/spans.rs index 0dc834ba0..0d8e46d4e 100644 --- a/src/ast/spans.rs +++ b/src/ast/spans.rs @@ -589,6 +589,7 @@ impl Spanned for CreateTable { with_storage_lifecycle_policy: _, // todo, Snowflake specific with_tags: _, // todo, Snowflake specific external_volume: _, // todo, Snowflake specific + with_connection: _, // todo, BigQuery external table connection base_location: _, // todo, Snowflake specific catalog: _, // todo, Snowflake specific catalog_sync: _, // todo, Snowflake specific diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 668c520e5..198f5c924 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -6416,8 +6416,28 @@ impl<'a> Parser<'a> { None }; let location = hive_formats.as_ref().and_then(|hf| hf.location.clone()); - let table_properties = self.parse_options(Keyword::TBLPROPERTIES)?; - let table_options = if !table_properties.is_empty() { + + // BigQuery external tables support `WITH CONNECTION ` and `OPTIONS(...)` + // clauses after the (optional) column list. + let with_connection = if self.parse_keywords(&[Keyword::WITH, Keyword::CONNECTION]) { + Some(self.parse_object_name(false)?) + } else { + None + }; + // BigQuery uses OPTIONS(...); Hive uses TBLPROPERTIES(...). They are + // mutually exclusive in practice, and `parse_options` returns an empty + // vec when the keyword isn't present, so trying OPTIONS first and + // falling back to TBLPROPERTIES preserves the existing Hive path + // without accepting both in the same statement. + let options = self.parse_options(Keyword::OPTIONS)?; + let table_properties = if options.is_empty() { + self.parse_options(Keyword::TBLPROPERTIES)? + } else { + vec![] + }; + let table_options = if !options.is_empty() { + CreateTableOptions::Options(options) + } else if !table_properties.is_empty() { CreateTableOptions::TableProperties(table_properties) } else if let Some(options) = self.maybe_parse_options(Keyword::OPTIONS)? { CreateTableOptions::Options(options) @@ -6430,6 +6450,7 @@ impl<'a> Parser<'a> { .hive_distribution(hive_distribution) .hive_formats(hive_formats) .table_options(table_options) + .with_connection(with_connection) .or_replace(or_replace) .if_not_exists(if_not_exists) .external(true) diff --git a/tests/sqlparser_bigquery.rs b/tests/sqlparser_bigquery.rs index 4bdb54f74..2393d1690 100644 --- a/tests/sqlparser_bigquery.rs +++ b/tests/sqlparser_bigquery.rs @@ -2203,6 +2203,90 @@ fn parse_big_query_declare() { ); } +#[test] +fn parse_bigquery_create_external_table_with_connection_and_options() { + let sql = concat!( + "CREATE OR REPLACE EXTERNAL TABLE `proj.ds.tbl` ", + "WITH CONNECTION `projects/proj/locations/us/connections/c` ", + r#"OPTIONS(format = "ICEBERG", uris = ["gs://b/m.json"])"#, + ); + let stmts = bigquery().parse_sql_statements(sql).unwrap(); + assert_eq!(stmts.len(), 1); + let Statement::CreateTable(ct) = &stmts[0] else { + panic!("expected CreateTable, got {:?}", stmts[0]); + }; + assert!(ct.external); + assert!(ct.or_replace); + assert!(ct.columns.is_empty()); + let with_connection = ct.with_connection.as_ref().expect("with_connection set"); + assert_eq!( + with_connection.to_string(), + "`projects/proj/locations/us/connections/c`" + ); + let CreateTableOptions::Options(opts) = &ct.table_options else { + panic!("expected OPTIONS, got {:?}", ct.table_options); + }; + assert_eq!(opts.len(), 2); +} + +#[test] +fn parse_bigquery_create_external_table_with_connection_variants() { + // WITH CONNECTION alone — no OPTIONS, no explicit column list. + // Display normalizes the bare form by adding an empty column list, so + // use `one_statement_parses_to` to assert the normalized output. + let stmt = bigquery().one_statement_parses_to( + "CREATE EXTERNAL TABLE t WITH CONNECTION c", + "CREATE EXTERNAL TABLE t () WITH CONNECTION c", + ); + let Statement::CreateTable(ct) = stmt else { + panic!("expected CreateTable"); + }; + assert!(ct.external); + assert!(!ct.or_replace); + assert!(ct.columns.is_empty()); + assert_eq!( + ct.with_connection + .as_ref() + .expect("with_connection set") + .to_string(), + "c" + ); + assert!(matches!(ct.table_options, CreateTableOptions::None)); + + // With explicit columns + OPTIONS. Exercises the Display round-trip of + // the `(columns) WITH CONNECTION OPTIONS(...)` ordering so a + // future refactor that reshuffles clause order will fail this test. + let sql = concat!( + "CREATE EXTERNAL TABLE t (a INT64, b STRING) ", + r#"WITH CONNECTION c OPTIONS(uris = ["gs://x"])"#, + ); + let stmt = bigquery().verified_stmt(sql); + let Statement::CreateTable(ct) = stmt else { + panic!("expected CreateTable"); + }; + assert_eq!(ct.columns.len(), 2); + assert_eq!( + ct.with_connection + .as_ref() + .expect("with_connection set") + .to_string(), + "c" + ); + let CreateTableOptions::Options(opts) = &ct.table_options else { + panic!("expected OPTIONS, got {:?}", ct.table_options); + }; + assert_eq!(opts.len(), 1); + + // Baseline: no WITH CONNECTION. The new parser branch must not produce + // a spurious with_connection when the keyword pair is absent. + let stmt = + bigquery().verified_stmt("CREATE EXTERNAL TABLE t (a INT64) OPTIONS(uris = [\"gs://x\"])"); + let Statement::CreateTable(ct) = stmt else { + panic!("expected CreateTable"); + }; + assert!(ct.with_connection.is_none()); +} + fn bigquery() -> TestedDialects { TestedDialects::new(vec![Box::new(BigQueryDialect {})]) } diff --git a/tests/sqlparser_duckdb.rs b/tests/sqlparser_duckdb.rs index df6268580..f1b492c03 100644 --- a/tests/sqlparser_duckdb.rs +++ b/tests/sqlparser_duckdb.rs @@ -780,6 +780,7 @@ fn test_duckdb_union_datatype() { with_tags: Default::default(), base_location: Default::default(), external_volume: Default::default(), + with_connection: Default::default(), catalog: Default::default(), catalog_sync: Default::default(), storage_serialization_policy: Default::default(), diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 1e053da78..7168f0ad2 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -1999,6 +1999,7 @@ fn parse_create_table_with_valid_options() { with_tags: None, base_location: None, external_volume: None, + with_connection: None, catalog: None, catalog_sync: None, storage_serialization_policy: None, @@ -2173,6 +2174,7 @@ fn parse_create_table_with_identity_column() { with_tags: None, base_location: None, external_volume: None, + with_connection: None, catalog: None, catalog_sync: None, storage_serialization_policy: None, diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 86315b1ef..c94695319 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -6702,6 +6702,7 @@ fn parse_trigger_related_functions() { with_tags: None, base_location: None, external_volume: None, + with_connection: None, catalog: None, catalog_sync: None, storage_serialization_policy: None, From b9b1edc899dc6b1c319563d24df0229e4b5e45ba Mon Sep 17 00:00:00 2001 From: Corey Fritz Date: Wed, 29 Apr 2026 09:24:58 -0600 Subject: [PATCH 2/4] Revert OPTIONS/TBLPROPERTIES reorder in parse_create_external_table The pre-existing maybe_parse_options(OPTIONS) fallback already handled BigQuery OPTIONS(...) on external tables; the reorder made it unreachable without changing behavior. --- src/parser/mod.rs | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 198f5c924..26265b90b 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -6417,27 +6417,13 @@ impl<'a> Parser<'a> { }; let location = hive_formats.as_ref().and_then(|hf| hf.location.clone()); - // BigQuery external tables support `WITH CONNECTION ` and `OPTIONS(...)` - // clauses after the (optional) column list. let with_connection = if self.parse_keywords(&[Keyword::WITH, Keyword::CONNECTION]) { Some(self.parse_object_name(false)?) } else { None }; - // BigQuery uses OPTIONS(...); Hive uses TBLPROPERTIES(...). They are - // mutually exclusive in practice, and `parse_options` returns an empty - // vec when the keyword isn't present, so trying OPTIONS first and - // falling back to TBLPROPERTIES preserves the existing Hive path - // without accepting both in the same statement. - let options = self.parse_options(Keyword::OPTIONS)?; - let table_properties = if options.is_empty() { - self.parse_options(Keyword::TBLPROPERTIES)? - } else { - vec![] - }; - let table_options = if !options.is_empty() { - CreateTableOptions::Options(options) - } else if !table_properties.is_empty() { + let table_properties = self.parse_options(Keyword::TBLPROPERTIES)?; + let table_options = if !table_properties.is_empty() { CreateTableOptions::TableProperties(table_properties) } else if let Some(options) = self.maybe_parse_options(Keyword::OPTIONS)? { CreateTableOptions::Options(options) From 85c935382ec24d86fff0686bb74c5974315e9eb9 Mon Sep 17 00:00:00 2001 From: Corey Fritz Date: Wed, 29 Apr 2026 09:25:01 -0600 Subject: [PATCH 3/4] Use verified_stmt for WITH CONNECTION tests; drop AST assertions --- tests/sqlparser_bigquery.rs | 85 +++++++------------------------------ 1 file changed, 15 insertions(+), 70 deletions(-) diff --git a/tests/sqlparser_bigquery.rs b/tests/sqlparser_bigquery.rs index 2393d1690..bda4e14f8 100644 --- a/tests/sqlparser_bigquery.rs +++ b/tests/sqlparser_bigquery.rs @@ -2205,86 +2205,31 @@ fn parse_big_query_declare() { #[test] fn parse_bigquery_create_external_table_with_connection_and_options() { - let sql = concat!( - "CREATE OR REPLACE EXTERNAL TABLE `proj.ds.tbl` ", - "WITH CONNECTION `projects/proj/locations/us/connections/c` ", - r#"OPTIONS(format = "ICEBERG", uris = ["gs://b/m.json"])"#, - ); - let stmts = bigquery().parse_sql_statements(sql).unwrap(); - assert_eq!(stmts.len(), 1); - let Statement::CreateTable(ct) = &stmts[0] else { - panic!("expected CreateTable, got {:?}", stmts[0]); - }; - assert!(ct.external); - assert!(ct.or_replace); - assert!(ct.columns.is_empty()); - let with_connection = ct.with_connection.as_ref().expect("with_connection set"); - assert_eq!( - with_connection.to_string(), - "`projects/proj/locations/us/connections/c`" + bigquery().one_statement_parses_to( + concat!( + "CREATE OR REPLACE EXTERNAL TABLE `proj.ds.tbl` ", + "WITH CONNECTION `projects/proj/locations/us/connections/c` ", + r#"OPTIONS(format = "ICEBERG", uris = ["gs://b/m.json"])"#, + ), + concat!( + "CREATE OR REPLACE EXTERNAL TABLE `proj`.`ds`.`tbl` () ", + "WITH CONNECTION `projects/proj/locations/us/connections/c` ", + r#"OPTIONS(format = "ICEBERG", uris = ["gs://b/m.json"])"#, + ), ); - let CreateTableOptions::Options(opts) = &ct.table_options else { - panic!("expected OPTIONS, got {:?}", ct.table_options); - }; - assert_eq!(opts.len(), 2); } #[test] fn parse_bigquery_create_external_table_with_connection_variants() { - // WITH CONNECTION alone — no OPTIONS, no explicit column list. - // Display normalizes the bare form by adding an empty column list, so - // use `one_statement_parses_to` to assert the normalized output. - let stmt = bigquery().one_statement_parses_to( + bigquery().one_statement_parses_to( "CREATE EXTERNAL TABLE t WITH CONNECTION c", "CREATE EXTERNAL TABLE t () WITH CONNECTION c", ); - let Statement::CreateTable(ct) = stmt else { - panic!("expected CreateTable"); - }; - assert!(ct.external); - assert!(!ct.or_replace); - assert!(ct.columns.is_empty()); - assert_eq!( - ct.with_connection - .as_ref() - .expect("with_connection set") - .to_string(), - "c" - ); - assert!(matches!(ct.table_options, CreateTableOptions::None)); - - // With explicit columns + OPTIONS. Exercises the Display round-trip of - // the `(columns) WITH CONNECTION OPTIONS(...)` ordering so a - // future refactor that reshuffles clause order will fail this test. - let sql = concat!( + bigquery().verified_stmt(concat!( "CREATE EXTERNAL TABLE t (a INT64, b STRING) ", r#"WITH CONNECTION c OPTIONS(uris = ["gs://x"])"#, - ); - let stmt = bigquery().verified_stmt(sql); - let Statement::CreateTable(ct) = stmt else { - panic!("expected CreateTable"); - }; - assert_eq!(ct.columns.len(), 2); - assert_eq!( - ct.with_connection - .as_ref() - .expect("with_connection set") - .to_string(), - "c" - ); - let CreateTableOptions::Options(opts) = &ct.table_options else { - panic!("expected OPTIONS, got {:?}", ct.table_options); - }; - assert_eq!(opts.len(), 1); - - // Baseline: no WITH CONNECTION. The new parser branch must not produce - // a spurious with_connection when the keyword pair is absent. - let stmt = - bigquery().verified_stmt("CREATE EXTERNAL TABLE t (a INT64) OPTIONS(uris = [\"gs://x\"])"); - let Statement::CreateTable(ct) = stmt else { - panic!("expected CreateTable"); - }; - assert!(ct.with_connection.is_none()); + )); + bigquery().verified_stmt(r#"CREATE EXTERNAL TABLE t (a INT64) OPTIONS(uris = ["gs://x"])"#); } fn bigquery() -> TestedDialects { From 1ae133b4fb3eb9f06e6746757ea7bea382ddade7 Mon Sep 17 00:00:00 2001 From: Corey Fritz Date: Wed, 29 Apr 2026 09:25:05 -0600 Subject: [PATCH 4/4] Use markdown-link form for WITH CONNECTION doc comments --- src/ast/ddl.rs | 4 ++-- src/ast/helpers/stmt_create_table.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index c9facd28a..fe1afc2b3 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -3021,8 +3021,8 @@ pub struct CreateTable { /// Snowflake "EXTERNAL_VOLUME" clause for Iceberg tables /// pub external_volume: Option, - /// BigQuery `WITH CONNECTION` clause for external tables - /// + /// `WITH CONNECTION` clause. + /// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_external_table_statement) pub with_connection: Option, /// Snowflake "BASE_LOCATION" clause for Iceberg tables /// diff --git a/src/ast/helpers/stmt_create_table.rs b/src/ast/helpers/stmt_create_table.rs index 21bf2ec5e..fe91e0360 100644 --- a/src/ast/helpers/stmt_create_table.rs +++ b/src/ast/helpers/stmt_create_table.rs @@ -157,8 +157,8 @@ pub struct CreateTableBuilder { pub base_location: Option, /// Optional external volume identifier. pub external_volume: Option, - /// BigQuery `WITH CONNECTION` clause for external tables. - /// + /// `WITH CONNECTION` clause. + /// [BigQuery](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_external_table_statement) pub with_connection: Option, /// Optional catalog name. pub catalog: Option, @@ -492,7 +492,7 @@ impl CreateTableBuilder { self.external_volume = external_volume; self } - /// Set the BigQuery `WITH CONNECTION` clause for external tables. + /// Set the `WITH CONNECTION` clause. pub fn with_connection(mut self, with_connection: Option) -> Self { self.with_connection = with_connection; self