refactor(python): remove pyarrow dependency, use IPC for data transfer

- Replace pyo3-polars PyDataFrame with IPC byte serialization
- Use polars native IPC reader/writer for Python-Rust data transfer
- Remove pyarrow from dependencies (saves ~117MB install size)
- Add proper Altair chart type detection (LayerChart, FacetChart, etc.)
- Wheel size: 5.8MB → 6.1MB (but eliminates 117MB pyarrow dep)

The IPC approach adds <2ms overhead for 1M rows, negligible compared
to Altair's JSON parsing which dominates execution time.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: cpsievert

Cargo.toml

@@ -29,7 +29,7 @@ tree-sitter = "0.25"
 csscolorparser = "0.8.1"
 
 # Data processing
-polars = { version = "0.52", features = ["lazy", "sql"] }
+polars = { version = "0.52", features = ["lazy", "sql", "ipc"] }
 
 # Readers
 duckdb = { version = "1.1", features = ["bundled"] }

ggsql-python/Cargo.toml

@@ -11,8 +11,7 @@ crate-type = ["cdylib"]
 
 [dependencies]
 pyo3 = { version = "0.26", features = ["extension-module"] }
-pyo3-polars = { version = "0.25", features = ["dtype-decimal", "dtype-struct"] }
-polars.workspace = true
+polars = { workspace = true, features = ["ipc"] }
 ggsql = { path = "../src", default-features = false, features = ["vegalite"] }
 
 [features]

ggsql-python/README.md

@@ -148,7 +148,6 @@ pytest tests/ -v
 - altair >= 5.0
 - narwhals >= 2.15
 - polars >= 1.0
-- pyarrow >= 14.0
 
 ## License
 

ggsql-python/pyproject.toml

@@ -18,7 +18,6 @@ dependencies = [
     "altair>=5.0",
     "narwhals>=2.15.0",
     "polars>=1.0",
-    "pyarrow>=14.0",
 ]
 
 [project.optional-dependencies]

ggsql-python/python/ggsql/__init__.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
-from typing import Any
+import io
+import json
+from typing import Any, Union
 
 import altair
 import narwhals as nw
@@ -11,12 +13,23 @@
 __all__ = ["split_query", "render_altair"]
 __version__ = "0.1.0"
 
+# Type alias for any Altair chart type
+AltairChart = Union[
+    altair.Chart,
+    altair.LayerChart,
+    altair.FacetChart,
+    altair.ConcatChart,
+    altair.HConcatChart,
+    altair.VConcatChart,
+    altair.RepeatChart,
+]
+
 
 def render_altair(
     df: IntoFrame,
     viz: str,
     **kwargs: Any,
-) -> altair.Chart:
+) -> AltairChart:
     """Render a DataFrame with a VISUALISE spec to an Altair chart.
 
     Parameters
@@ -27,13 +40,13 @@ def render_altair(
     viz
         VISUALISE spec string (e.g., "VISUALISE x, y DRAW point")
     **kwargs
-        Additional keyword arguments passed to `altair.Chart.from_json()`.
+        Additional keyword arguments passed to `from_json()`.
         Common options include `validate=False` to skip schema validation.
 
     Returns
     -------
-    altair.Chart
-        An Altair chart object.
+    AltairChart
+        An Altair chart object (Chart, LayerChart, FacetChart, etc.).
     """
     df = nw.from_native(df, pass_through=True)
 
@@ -43,8 +56,29 @@ def render_altair(
     if not isinstance(df, nw.DataFrame):
         raise TypeError("df must be a narwhals DataFrame or compatible type")
 
+    # Convert to polars and serialize to IPC bytes
     pl_df = df.to_polars()
+    buffer = io.BytesIO()
+    pl_df.write_ipc(buffer)
+    ipc_bytes = buffer.getvalue()
+
+    vegalite_json = _render(ipc_bytes, viz, writer="vegalite")
 
-    vegalite_json = _render(pl_df, viz, writer="vegalite")
+    # Parse to determine the correct Altair class
+    spec = json.loads(vegalite_json)
 
-    return altair.Chart.from_json(vegalite_json, **kwargs)
+    # Determine the correct Altair class based on spec structure
+    if "layer" in spec:
+        return altair.LayerChart.from_json(vegalite_json, **kwargs)
+    elif "facet" in spec or "spec" in spec:
+        return altair.FacetChart.from_json(vegalite_json, **kwargs)
+    elif "concat" in spec:
+        return altair.ConcatChart.from_json(vegalite_json, **kwargs)
+    elif "hconcat" in spec:
+        return altair.HConcatChart.from_json(vegalite_json, **kwargs)
+    elif "vconcat" in spec:
+        return altair.VConcatChart.from_json(vegalite_json, **kwargs)
+    elif "repeat" in spec:
+        return altair.RepeatChart.from_json(vegalite_json, **kwargs)
+    else:
+        return altair.Chart.from_json(vegalite_json, **kwargs)

ggsql-python/src/lib.rs

@@ -3,16 +3,16 @@
 #![allow(clippy::useless_conversion)]
 
 use pyo3::prelude::*;
-use pyo3_polars::PyDataFrame;
+use pyo3::types::PyBytes;
 use std::collections::{HashMap, HashSet};
+use std::io::Cursor;
 
 use ggsql::naming::GLOBAL_DATA_KEY;
 use ggsql::parser::parse_query;
 use ggsql::writer::{VegaLiteWriter, Writer};
 use ggsql::AestheticValue;
 
-// Re-export polars from pyo3_polars to ensure type compatibility
-use polars::prelude::DataFrame;
+use polars::prelude::{DataFrame, IpcReader, SerReader};
 
 #[pyfunction]
 fn split_query(query: &str) -> PyResult<(String, String)> {
@@ -21,10 +21,14 @@ fn split_query(query: &str) -> PyResult<(String, String)> {
 }
 
 #[pyfunction]
-#[pyo3(signature = (py_df, viz, *, writer = "vegalite"))]
-fn render(py_df: PyDataFrame, viz: &str, writer: &str) -> PyResult<String> {
-    // Convert PyDataFrame to Polars DataFrame (pyo3-polars' bundled polars)
-    let df: DataFrame = py_df.into();
+#[pyo3(signature = (ipc_bytes, viz, *, writer = "vegalite"))]
+fn render(ipc_bytes: &Bound<'_, PyBytes>, viz: &str, writer: &str) -> PyResult<String> {
+    // Read DataFrame from IPC bytes
+    let bytes = ipc_bytes.as_bytes();
+    let cursor = Cursor::new(bytes);
+    let df: DataFrame = IpcReader::new(cursor)
+        .finish()
+        .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, _>(format!("Failed to read IPC data: {}", e)))?;
 
     // Parse the visualization spec
     // The viz string should be a complete VISUALISE statement
@@ -73,8 +77,6 @@ fn render(py_df: PyDataFrame, viz: &str, writer: &str) -> PyResult<String> {
     spec.compute_aesthetic_labels();
 
     // Create data map with the DataFrame as global data
-    // Note: We use pyo3-polars' bundled polars DataFrame, which may differ from ggsql's
-    // The VegaLiteWriter only needs DataFrame for serialization to JSON
     let mut data_map: HashMap<String, DataFrame> = HashMap::new();
     data_map.insert(GLOBAL_DATA_KEY.to_string(), df);
 

ggsql-python/tests/test_ggsql.py

@@ -87,6 +87,69 @@ def test_chart_can_be_serialized(self):
         assert len(json_str) > 0
 
 
+class TestRenderAltairChartTypeDetection:
+    """Tests for correct Altair chart type detection based on spec structure."""
+
+    def test_simple_chart_returns_layer_chart(self):
+        """Simple DRAW specs produce LayerChart (ggsql always wraps in layer)."""
+        df = pl.DataFrame({"x": [1, 2, 3], "y": [10, 20, 30]})
+        chart = ggsql.render_altair(df, "VISUALISE x, y DRAW point")
+        # ggsql wraps all charts in a layer
+        assert isinstance(chart, altair.LayerChart)
+
+    def test_layered_chart_can_round_trip(self):
+        """LayerChart can be converted to dict and back."""
+        df = pl.DataFrame({"x": [1, 2, 3], "y": [10, 20, 30]})
+        chart = ggsql.render_altair(df, "VISUALISE x, y DRAW point")
+
+        # Convert to dict and back
+        spec = chart.to_dict()
+        assert "layer" in spec
+
+        # Should be able to recreate from dict
+        recreated = altair.LayerChart.from_dict(spec)
+        assert isinstance(recreated, altair.LayerChart)
+
+    def test_faceted_chart_returns_facet_chart(self):
+        """FACET WRAP specs produce FacetChart."""
+        df = pl.DataFrame({
+            "x": [1, 2, 3, 4, 5, 6],
+            "y": [10, 20, 30, 40, 50, 60],
+            "group": ["A", "A", "A", "B", "B", "B"],
+        })
+        # Need validate=False because ggsql produces v6 specs
+        chart = ggsql.render_altair(df, "VISUALISE x, y FACET WRAP group DRAW point", validate=False)
+        assert isinstance(chart, altair.FacetChart)
+
+    def test_faceted_chart_can_round_trip(self):
+        """FacetChart can be converted to dict and back."""
+        df = pl.DataFrame({
+            "x": [1, 2, 3, 4, 5, 6],
+            "y": [10, 20, 30, 40, 50, 60],
+            "group": ["A", "A", "A", "B", "B", "B"],
+        })
+        chart = ggsql.render_altair(df, "VISUALISE x, y FACET WRAP group DRAW point", validate=False)
+
+        # Convert to dict (skip validation for ggsql specs)
+        spec = chart.to_dict(validate=False)
+        assert "facet" in spec or "spec" in spec
+
+        # Should be able to recreate from dict (with validation disabled)
+        recreated = altair.FacetChart.from_dict(spec, validate=False)
+        assert isinstance(recreated, altair.FacetChart)
+
+    def test_chart_with_color_encoding(self):
+        """Charts with color encoding still return correct type."""
+        df = pl.DataFrame({
+            "x": [1, 2, 3, 4],
+            "y": [10, 20, 30, 40],
+            "category": ["A", "B", "A", "B"],
+        })
+        chart = ggsql.render_altair(df, "VISUALISE x, y, category AS color DRAW point")
+        # Should still be a LayerChart (ggsql wraps in layer)
+        assert isinstance(chart, altair.LayerChart)
+
+
 class TestRenderAltairErrorHandling:
     """Tests for error handling in render_altair()."""