Add sqlite reader and adjust SQL queries to work there (#182)

* Add sqlite reader

* Portable GREATEST/LEAST and various sqlite tweaks

* Drop remaining __ggsql_stat_ columns not consumed by remappings

* Use chrono ToSql/FromSql in sqlite reader

* Add sqlite flags

* Use portable SQL

* Minor tweaks to naming and features

* Add SQL dialect function overriding for duckdb

* cargo fmt

* Fixup grammar: subquery as function argument

* Use dialect GREATEST() in histogram

* cargo fmt

Author: georgestagg

.cargo/config.toml

@@ -1,3 +1,6 @@
+[env]
+LIBSQLITE3_FLAGS = "-DSQLITE_ENABLE_MATH_FUNCTIONS"
+
 [target.wasm32-unknown-unknown]
 rustflags = [
   "-C", "link-args=-z stack-size=16777216",

Cargo.toml

@@ -37,8 +37,7 @@ polars-ops = { version = "0.52", features = ["pivot"] }
 duckdb = { version = "~1.4", features = ["bundled", "vtab-arrow"] }
 arrow = { version = "56", default-features = false, features = ["ipc"] }
 postgres = "0.19"
-sqlx = { version = "0.8", features = ["postgres"] }
-rusqlite = "0.32"
+rusqlite = { version = "0.38", features = ["bundled", "chrono", "functions", "window"] }
 
 # Writers
 plotters = "0.3"

ggsql-python/src/lib.rs

@@ -127,6 +127,8 @@ struct PyReaderBridge {
     obj: Py<PyAny>,
 }
 
+static ANSI_DIALECT: ggsql::reader::AnsiDialect = ggsql::reader::AnsiDialect;
+
 impl Reader for PyReaderBridge {
     fn execute_sql(&self, sql: &str) -> ggsql::Result<DataFrame> {
         Python::attach(|py| {
@@ -161,6 +163,10 @@ impl Reader for PyReaderBridge {
             Ok(())
         })
     }
+
+    fn dialect(&self) -> &dyn ggsql::reader::SqlDialect {
+        &ANSI_DIALECT
+    }
 }
 
 // ============================================================================

src/Cargo.toml

@@ -38,7 +38,6 @@ polars-ops.workspace = true
 duckdb = { workspace = true, optional = true }
 arrow = { workspace = true, optional = true }
 postgres = { workspace = true, optional = true }
-sqlx = { workspace = true, optional = true }
 rusqlite = { workspace = true, optional = true }
 
 # Writers
@@ -77,15 +76,16 @@ proptest.workspace = true
 ureq = "3"
 
 [features]
-default = ["duckdb", "sqlite", "vegalite", "ipc", "builtin-data"]
+default = ["duckdb", "sqlite", "vegalite", "ipc", "parquet", "builtin-data"]
 ipc = ["polars/ipc"]
 duckdb = ["dep:duckdb", "dep:arrow"]
 polars-sql = ["polars/sql"]
-builtin-data = ["polars/parquet"]
+parquet = ["polars/parquet"]
 postgres = ["dep:postgres"]
 sqlite = ["dep:rusqlite"]
 vegalite = []
 ggplot2 = []
+builtin-data = []
 python = ["dep:pyo3"]
 rest-api = ["dep:axum", "dep:tokio", "dep:tower-http", "dep:tracing", "dep:tracing-subscriber", "duckdb", "vegalite"]
 all-readers = ["duckdb", "postgres", "sqlite", "polars-sql"]

src/execute/casting.rs

@@ -4,7 +4,8 @@
 //! scale requirements and updating type info accordingly.
 
 use crate::plot::scale::coerce_dtypes;
-use crate::plot::{CastTargetType, Layer, ParameterValue, Plot, SqlTypeNames};
+use crate::plot::{CastTargetType, Layer, ParameterValue, Plot};
+use crate::reader::SqlDialect;
 use crate::{naming, DataSource};
 use polars::prelude::{DataType, TimeUnit};
 use std::collections::{HashMap, HashSet};
@@ -57,7 +58,7 @@ pub fn literal_to_sql(lit: &ParameterValue) -> String {
 pub fn determine_type_requirements(
     spec: &Plot,
     layer_type_info: &[Vec<TypeInfo>],
-    type_names: &SqlTypeNames,
+    dialect: &dyn SqlDialect,
 ) -> Vec<Vec<TypeRequirement>> {
     use crate::plot::scale::TransformKind;
 
@@ -123,7 +124,7 @@ pub fn determine_type_requirements(
 
             // Check if this specific column needs casting
             if let Some(cast_target) = scale_type.required_cast_type(col_dtype, &target_dtype) {
-                if let Some(sql_type) = type_names.for_target(cast_target) {
+                if let Some(sql_type) = dialect.type_name_for(cast_target) {
                     // Don't add duplicate requirements for same column
                     if !requirements.iter().any(|r| r.column == col_name) {
                         requirements.push(TypeRequirement {
@@ -155,7 +156,7 @@ pub fn determine_type_requirements(
                     };
 
                     if needs_int_cast {
-                        if let Some(sql_type) = type_names.for_target(CastTargetType::Integer) {
+                        if let Some(sql_type) = dialect.type_name_for(CastTargetType::Integer) {
                             // Don't add duplicate requirements for same column
                             if !requirements.iter().any(|r| r.column == col_name) {
                                 requirements.push(TypeRequirement {

src/execute/layer.rs

@@ -4,9 +4,9 @@
 //! transformations, stat transforms, and post-query operations.
 
 use crate::plot::{
-    AestheticValue, DefaultAestheticValue, Layer, ParameterValue, Scale, Schema, SqlTypeNames,
-    StatResult,
+    AestheticValue, DefaultAestheticValue, Layer, ParameterValue, Scale, Schema, StatResult,
 };
+use crate::reader::SqlDialect;
 use crate::{naming, DataFrame, GgsqlError, Result};
 use polars::prelude::DataType;
 use std::collections::{HashMap, HashSet};
@@ -150,6 +150,17 @@ pub fn apply_remappings_post_query(df: DataFrame, layer: &Layer) -> Result<DataF
         }
     }
 
+    // Drop any remaining __ggsql_stat_* columns that weren't consumed by remappings.
+    let stat_cols: Vec<String> = df
+        .get_column_names()
+        .into_iter()
+        .filter(|name| naming::is_stat_column(name))
+        .map(|name| name.to_string())
+        .collect();
+    if !stat_cols.is_empty() {
+        df = df.drop_many(stat_cols);
+    }
+
     Ok(df)
 }
 
@@ -183,14 +194,14 @@ pub fn literal_to_series(name: &str, lit: &ParameterValue, len: usize) -> polars
 /// * `layer` - The layer configuration
 /// * `schema` - The layer's schema (used for column dtype lookup)
 /// * `scales` - All resolved scales
-/// * `type_names` - SQL type names for the database backend
+/// * `dialect` - SQL dialect for the database backend
 pub fn apply_pre_stat_transform(
     query: &str,
     layer: &Layer,
     full_schema: &Schema,
     aesthetic_schema: &Schema,
     scales: &[Scale],
-    type_names: &SqlTypeNames,
+    dialect: &dyn SqlDialect,
 ) -> String {
     let mut transform_exprs: Vec<(String, String)> = vec![];
     let mut transformed_columns: HashSet<String> = HashSet::new();
@@ -226,7 +237,7 @@ pub fn apply_pre_stat_transform(
                 // Get pre-stat SQL transformation from scale type (if applicable)
                 // Each scale type's pre_stat_transform_sql() returns None if not applicable
                 if let Some(sql) =
-                    scale_type.pre_stat_transform_sql(&aes_col_name, &col_dtype, scale, type_names)
+                    scale_type.pre_stat_transform_sql(&aes_col_name, &col_dtype, scale, dialect)
                 {
                     transformed_columns.insert(aes_col_name.clone());
                     transform_exprs.push((aes_col_name, sql));
@@ -336,7 +347,7 @@ pub fn build_layer_base_query(
 /// * `base_query` - The base query from build_layer_base_query
 /// * `schema` - The layer's schema (with min/max from base_query)
 /// * `scales` - All resolved scales
-/// * `type_names` - SQL type names for the database backend
+/// * `dialect` - SQL dialect for the database backend
 /// * `execute_query` - Function to execute queries (needed for some stat transforms)
 ///
 /// # Returns
@@ -347,7 +358,7 @@ pub fn apply_layer_transforms<F>(
     base_query: &str,
     schema: &Schema,
     scales: &[Scale],
-    type_names: &SqlTypeNames,
+    dialect: &dyn SqlDialect,
     execute_query: &F,
 ) -> Result<String>
 where
@@ -387,7 +398,7 @@ where
         schema,
         &aesthetic_schema,
         scales,
-        type_names,
+        dialect,
     );
 
     // Build group_by columns from partition_by
@@ -416,6 +427,7 @@ where
         &group_by,
         &layer.parameters,
         execute_query,
+        dialect,
     )?;
 
     // Apply literal default remappings from geom defaults (e.g., y2 => 0.0 for bar baseline).
@@ -518,12 +530,6 @@ where
             if stat_rename_exprs.is_empty() {
                 transformed_query
             } else {
-                let stat_col_names: Vec<String> = stat_columns
-                    .iter()
-                    .map(|s| naming::stat_column(s))
-                    .collect();
-                let exclude_clause = format!("EXCLUDE ({})", stat_col_names.join(", "));
-
                 // If the transformed query uses CTEs (WITH ... SELECT ...),
                 // we can't wrap it in a subquery because Polars SQL doesn't
                 // support CTEs inside subqueries. Instead, split into CTE
@@ -536,16 +542,14 @@ where
                         .and_then(super::cte::split_with_query)
                 {
                     format!(
-                        "{}, __ggsql_stat__ AS ({}) SELECT * {}, {} FROM __ggsql_stat__",
+                        "{}, __ggsql_stat__ AS ({}) SELECT *, {} FROM __ggsql_stat__",
                         cte_prefix,
                         trailing_select,
-                        exclude_clause,
                         stat_rename_exprs.join(", ")
                     )
                 } else {
                     format!(
-                        "SELECT * {}, {} FROM ({}) AS __ggsql_stat__",
-                        exclude_clause,
+                        "SELECT *, {} FROM ({}) AS __ggsql_stat__",
                         stat_rename_exprs.join(", "),
                         transformed_query
                     )

src/execute/mod.rs

@@ -144,7 +144,10 @@ fn validate(layers: &[Layer], layer_schemas: &[Schema]) -> Result<()> {
 
 /// Check if an aesthetic value is a null sentinel (explicit removal marker)
 fn is_null_sentinel(value: &AestheticValue) -> bool {
-    matches!(value, AestheticValue::Literal(crate::plot::ParameterValue::Null))
+    matches!(
+        value,
+        AestheticValue::Literal(crate::plot::ParameterValue::Null)
+    )
 }
 
 /// Merge global mappings into layer aesthetics and expand wildcards
@@ -887,7 +890,7 @@ pub struct PreparedData {
 /// * `reader` - A Reader implementation for executing SQL
 pub fn prepare_data_with_reader<R: Reader>(query: &str, reader: &R) -> Result<PreparedData> {
     let execute_query = |sql: &str| reader.execute_sql(sql);
-    let type_names = reader.sql_type_names();
+    let dialect = reader.dialect();
 
     // Parse once and create SourceTree
     let source_tree = parser::SourceTree::new(query)?;
@@ -1025,7 +1028,7 @@ pub fn prepare_data_with_reader<R: Reader>(query: &str, reader: &R) -> Result<Pr
 
     // Determine which columns need type casting
     let type_requirements =
-        casting::determine_type_requirements(&specs[0], &layer_type_info, &type_names);
+        casting::determine_type_requirements(&specs[0], &layer_type_info, dialect);
 
     // Update type info with post-cast dtypes
     // This ensures subsequent schema extraction and scale resolution see the correct types
@@ -1094,7 +1097,7 @@ pub fn prepare_data_with_reader<R: Reader>(query: &str, reader: &R) -> Result<Pr
             &layer_base_queries[idx],
             &layer_schemas[idx],
             &scales,
-            &type_names,
+            dialect,
             &execute_query,
         )?;
         layer_queries.push(layer_query);

src/execute/schema.rs

@@ -145,7 +145,7 @@ pub fn extract_series_value(
 
 /// Fetch only column types (no min/max) from a query.
 ///
-/// Uses LIMIT 0 to get schema without reading data.
+/// Uses LIMIT 1 to get schema while minimally reading data.
 /// Returns `(name, dtype, is_discrete)` tuples for each column.
 ///
 /// This is the first phase of the split schema extraction approach:
@@ -157,7 +157,7 @@ where
     F: Fn(&str) -> Result<DataFrame>,
 {
     let schema_query = format!(
-        "SELECT * FROM ({}) AS {} LIMIT 0",
+        "SELECT * FROM ({}) AS {} LIMIT 1",
         query,
         naming::SCHEMA_ALIAS
     );

src/parser/builder.rs

@@ -3350,7 +3350,10 @@ mod tests {
 
         let literal_node = source.find_node(&root, "(literal_value) @lit").unwrap();
         let parsed = parse_literal_value(&literal_node, &source).unwrap();
-        assert!(matches!(parsed, AestheticValue::Literal(ParameterValue::Null)));
+        assert!(matches!(
+            parsed,
+            AestheticValue::Literal(ParameterValue::Null)
+        ));
     }
 
     // ========================================

src/plot/layer/geom/bar.rs

@@ -7,6 +7,7 @@ use super::types::get_column_name;
 use super::{DefaultAesthetics, DefaultParam, DefaultParamValue, GeomTrait, GeomType, StatResult};
 use crate::naming;
 use crate::plot::types::{DefaultAestheticValue, ParameterValue};
+use crate::reader::SqlDialect;
 use crate::{DataFrame, GgsqlError, Mappings, Result};
 
 use super::types::Schema;
@@ -81,6 +82,7 @@ impl GeomTrait for Bar {
         group_by: &[String],
         _parameters: &HashMap<String, ParameterValue>,
         _execute_query: &dyn Fn(&str) -> Result<DataFrame>,
+        _dialect: &dyn SqlDialect,
     ) -> Result<StatResult> {
         stat_bar_count(query, schema, aesthetics, group_by)
     }