Support for pandas dataframes, and mapping sequences to Python tuples

Pandas

Author: ghislainfourny

README.md

@@ -42,10 +42,16 @@ Any feedback or error reports are very welcome.
 
 ## Type mapping
 
-When passing Python values to JSONiq or getting them from a JSONiq queries, the mapping is as follows: 
+Any expression in JSONiq returns a sequence of items. Any variable in JSONiq is bound to a sequence of items.
+Items can be objects, arrays, or atomic values (strings, integers, booleans, nulls, dates, binary, durations, doubles, decimal numbers, etc).
+A sequence of items can be a sequence of just one item, but it can also be empty, or it can be as large as to contain millions, billions or even trillions of items. Obviously, for sequence longer than a billion items, it is a better idea to use a cluster than a laptop.
+A relational table (or more generally a data frame) corresponds to a sequence of object items sharing the same schema. However, sequences of items are more general than tables or data frames and support heterogeneity seamlessly.
+
+When passing Python values to JSONiq or getting them from a JSONiq queries, the mapping to and from Python is as follows: 
 
 | Python | JSONiq |
 |-------|-------|
+|tuple|sequence of items|
 |dict|object|
 |list|array|
 |str|string|
@@ -73,6 +79,7 @@ You can directly copy paste the code below to a Python file and execute it with
 
 ```
 from jsoniq import RumbleSession
+import pandas as pd
 
 # The syntax to start a session is similar to that of Spark.
 # A RumbleSession is a SparkSession that additionally knows about RumbleDB.
@@ -155,16 +162,16 @@ print(seq.json());
 ###### Binding JSONiq variables to Python values ###########
 ############################################################
 
-# It is possible to bind a JSONiq variable to a list of native Python values
+# It is possible to bind a JSONiq variable to a tuple of native Python values
 # and then use it in a query.
 # JSONiq, variables are bound to sequences of items, just like the results of JSONiq
 # queries are sequence of items.
-# A Python list will be seamlessly converted to a sequence of items by the library.
+# A Python tuple will be seamlessly converted to a sequence of items by the library.
 # Currently we only support strs, ints, floats, booleans, None, lists, and dicts.
 # But if you need more (like date, bytes, etc) we will add them without any problem.
 # JSONiq has a rich type system.
  
-rumble.bind('$c', [1,2,3,4, 5, 6])
+rumble.bind('$c', (1,2,3,4, 5, 6))
 print(rumble.jsoniq("""
 for $v in $c
 let $parity := $v mod 2
@@ -176,7 +183,7 @@ return { switch($parity)
 }
 """).json())
 
-rumble.bind('$c', [[1,2,3],[4,5,6]])
+rumble.bind('$c', ([1,2,3],[4,5,6]))
 print(rumble.jsoniq("""
 for $i in $c
 return [
@@ -185,18 +192,34 @@ return [
 ]
 """).json())
 
-rumble.bind('$c', [{"foo":[1,2,3]},{"foo":[4,{"bar":[1,False, None]},6]}])
+rumble.bind('$c', ({"foo":[1,2,3]},{"foo":[4,{"bar":[1,False, None]},6]}))
 print(rumble.jsoniq('{ "results" : $c.foo[[2]] }').json())
 
-# It is possible to bind only one value. The it must be provided as a singleton list.
+# It is possible to bind only one value. The it must be provided as a singleton tuple.
 # This is because in JSONiq, an item is the same a sequence of one item.
-rumble.bind('$c', [42])
+rumble.bind('$c', (42,))
 print(rumble.jsoniq('for $i in 1 to $c return $i*$i').json())
 
 # For convenience and code readability, you can also use bindOne().
 rumble.bindOne('$c', 42)
 print(rumble.jsoniq('for $i in 1 to $c return $i*$i').json())
 
+##########################################################
+##### Binding JSONiq variables to pandas DataFrames ######
+##### Getting the output as a Pandas DataFrame      ######
+##########################################################
+
+# Creating a dummy pandas dataframe
+data = {'Name': ['Alice', 'Bob', 'Charlie'],
+        'Age': [30,25,35]};
+pdf = pd.DataFrame(data);
+
+# Binding a pandas dataframe
+rumble.bind('$a',pdf);
+seq = rumble.jsoniq('$a.Name')
+# Getting the output as a pandas dataframe
+print(seq.pdf())
+
 
 ################################################
 ##### Using Pyspark DataFrames with JSONiq #####
@@ -324,6 +347,13 @@ Even more queries can be found [here](https://colab.research.google.com/github/R
 
 # Last updates
 
+## Version 0.1.0 alpha 13
+- Allow to bind JSONiq variables to pandas dataframes
+- Allow to retrieve the output of a JSONiq query as a pandas dataframe (if the output is available as a dataframe, i.e., availableOutputs() returns a list that contains "DataFrame")
+- Clean up the mapping to strictly map tuples to sequence of items, and lists ot array items. This will avoid confusion between arrays and sequences.
+- As a consequence, json() now returns a tuple, not a list.
+- Calling bind() with a single list will return an informative error. Use bind() with a tuple instead, or call bindOne() to interpret the list as a sequence of one array item.
+
 ## Version 0.1.0 alpha 12
 - Allow to bind JSONiq variables to Python values (mapping Python lists to sequences of items). This makes it possible to manipulate Python values directly with JSONiq and even without any knowledge of Spark at all.
 - renamed bindDataFrameAsVariable() to bind(), which can be used both with DataFrames and Python lists.

pyproject.toml

@@ -4,11 +4,12 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "jsoniq"
-version = "0.1.0a12"
+version = "0.2.0a1"
 description = "Python edition of RumbleDB, a JSONiq engine"
 requires-python = ">=3.11"
 dependencies = [
-    "pyspark==4.0"
+    "pyspark==4.0",
+    "pandas==2.3"
 ]
 authors = [
   {name = "Ghislain Fourny", email = "ghislain.fourny@inf.ethz.ch"},
@@ -23,6 +24,8 @@ classifiers = [
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
   "Programming Language :: Python :: 3.13",
+  "Typing :: Typed",
+  "License :: OSI Approved :: Apache Software License"
 ]
 
 [tool.setuptools.packages.find]

requirements.txt

@@ -1 +1,2 @@
-pyspark==4.0.0
+pyspark==4.0
+pandas==2.3

src/jsoniq/sequence.py

@@ -10,7 +10,7 @@ def __init__(self, sequence, sparksession):
         self._sparksession = sparksession
 
     def json(self):
-        return [json.loads(l.serializeAsJSON()) for l in self._jsequence.items()]
+        return tuple([json.loads(l.serializeAsJSON()) for l in self._jsequence.items()])
 
     def rdd(self):
         rdd = self._jsequence.getAsPickledStringRDD();
@@ -20,6 +20,9 @@ def rdd(self):
     def df(self):
         return DataFrame(self._jsequence.getAsDataFrame(), self._sparksession)
 
+    def pdf(self):
+        return self.df().toPandas()
+
     def nextJSON(self):
         return self._jsequence.next().serializeAsJSON()
 

src/jsoniq/session.py

@@ -4,6 +4,7 @@
 import platform
 import os
 import re
+import pandas as pd
 import importlib.resources as pkg_resources
 
 with pkg_resources.path("jsoniq.jars", "rumbledb-1.24.0.jar") as jar_path:
@@ -84,6 +85,8 @@ def __getattr__(self, name):
     _builder = Builder()
 
     def convert(self, value):
+        if isinstance(value, tuple):
+            return [ self.convert(v) for v in value]
         if isinstance(value, bool):
             return self._sparksession._jvm.org.rumbledb.items.ItemFactory.getInstance().createBooleanItem(value)
         elif isinstance(value, str):
@@ -114,18 +117,30 @@ def bind(self, name: str, valueToBind):
         if not name.startswith("$"):
             raise ValueError("Variable name must start with a dollar symbol ('$').")
         name = name[1:]
-        if isinstance(valueToBind, list):
-            items = [ self.convert(value) for value in valueToBind]
-            conf.setExternalVariableValue(name, items)
-            return self
-        if(hasattr(valueToBind, "_get_object_id")):
+        if isinstance(valueToBind, SequenceOfItems):
+            outputs = valueToBind.availableOutputs()
+            if isinstance(outputs, list) and "DataFrame" in outputs:
+                conf.setExternalVariableValue(name, valueToBind.df());
+            # TODO support binding a variable to an RDD
+            #elif isinstance(outputs, list) and "RDD" in outputs:
+            #    conf.setExternalVariableValue(name, valueToBind.getAsRDD());
+            else:
+                conf.setExternalVariableValue(name, valueToBind.items());
+        elif isinstance(valueToBind, pd.DataFrame):
+            pysparkdf = self._sparksession.createDataFrame(valueToBind)
+            conf.setExternalVariableValue(name, pysparkdf._jdf);
+        elif isinstance(valueToBind, tuple):
+            conf.setExternalVariableValue(name, self.convert(valueToBind))
+        elif isinstance(valueToBind, list):
+            raise ValueError("To avoid confusion, a sequence of items must be provided as a Python tuple, not as a Python list. Lists are mapped to single array items, while tuples are mapped to sequences of items. If you want to interpret the list as a sequence of items (one item for each list member), then you need to change this list to a tuple by wrapping it into a tuple() call. If you want to bind the variable to one array item, then you need to wrap the provided list inside a singleton tuple and try again, or you can also call bindOne() instead.")
+        elif(hasattr(valueToBind, "_get_object_id")):
             conf.setExternalVariableValue(name, valueToBind);
         else:
             conf.setExternalVariableValue(name, valueToBind._jdf);
         return self;
 
     def bindOne(self, name: str, value):
-        return self.bind(name, [value])
+        return self.bind(name, (value,))
 
     def bindDataFrameAsVariable(self, name: str, df):
         conf = self._jrumblesession.getConfiguration();