From ca740940353f87eeb7a1d826c4bb4548ba077cc7 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 28 Sep 2026 12:36:08 +0200 Subject: [PATCH 1/9] Replace jar. --- src/jsoniq/jars/rumbledb-2.1.9.jar | 3 --- src/jsoniq/jars/rumbledb-3.0.0.jar | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 src/jsoniq/jars/rumbledb-2.1.9.jar create mode 100644 src/jsoniq/jars/rumbledb-3.0.0.jar diff --git a/src/jsoniq/jars/rumbledb-2.1.9.jar b/src/jsoniq/jars/rumbledb-2.1.9.jar deleted file mode 100644 index a5009bd..0000000 --- a/src/jsoniq/jars/rumbledb-2.1.9.jar +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:cad5d2bbc20972bfe7536dad7624cb6f1d7df7efd995ab181173b199b5e3ad53 -size 79750237 diff --git a/src/jsoniq/jars/rumbledb-3.0.0.jar b/src/jsoniq/jars/rumbledb-3.0.0.jar new file mode 100644 index 0000000..ca62062 --- /dev/null +++ b/src/jsoniq/jars/rumbledb-3.0.0.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daba67e04cddf3aaa4f6294fe93e19e5139a03938e532ad004f9e9b6693a90b1 +size 82146180 From d282b7953b5d410e8dc4703d3a98f412cb2a5c4e Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 28 Sep 2026 12:43:22 +0200 Subject: [PATCH 2/9] Increment version. --- src/jsoniq/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jsoniq/session.py b/src/jsoniq/session.py index dcacaad..e2e6c8f 100644 --- a/src/jsoniq/session.py +++ b/src/jsoniq/session.py @@ -7,7 +7,7 @@ import pandas as pd from importlib.resources import files, as_file -with as_file(files("jsoniq.jars").joinpath("rumbledb-2.1.9.jar")) as jar_path: +with as_file(files("jsoniq.jars").joinpath("rumbledb-3.0.0.jar")) as jar_path: if (os.name == 'nt'): jar_path_str = str(jar_path) else: From a7495879bebef00eb851b039e012cbd8e4dc910a Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 28 Sep 2026 14:57:33 +0200 Subject: [PATCH 3/9] Adapt API. --- README.md | 15 +++++ pyproject.toml | 2 +- src/jsoniq/configuration.py | 39 +++++++++++ src/jsoniq/session.py | 66 ++++++++++--------- tests/test_rumbledb_3_api.py | 124 +++++++++++++++++++++++++++++++++++ tests/test_sample.py | 16 +++-- 6 files changed, 224 insertions(+), 38 deletions(-) create mode 100644 src/jsoniq/configuration.py create mode 100644 tests/test_rumbledb_3_api.py diff --git a/README.md b/README.md index 08cafd1..0268759 100644 --- a/README.md +++ b/README.md @@ -340,6 +340,21 @@ Even more queries can be found [here](https://colab.research.google.com/github/R # Latest updates +## Version 3.0.0 +- Upgraded to RumbleDB 3.0.0 and its immutable configuration and external bindings APIs. +- Preserved `getRumbleConf().setResultSizeCap(...)`, `setMaterializationCap(...)`, and `setShowErrorInfo(...)` through a Python adapter. Configuration changes apply to subsequent queries; existing sequences retain their compilation settings. +- Fixed object conversion and binding query results as DataFrames. Keyword bindings are scoped to a query and restore persistent bindings even when query compilation fails. + +Configuration can also be changed using RumbleDB 3.0's dot-separated paths: + +```python +rumble.getRumbleConf().set("runtime.resultsSizeCap", 100) +rumble.getRumbleConf().set("runtime.materializationCap", 100000) +rumble.getRumbleConf().set("debug.showErrorInfo", True) +``` + +The result size cap controls `first()` and notebook display. `json()` retrieves all items, subject to the separate materialization cap. + ## Version 2.1.9 - Fixed a bug in the inferred conversion to DataFrames of output involving arrays of objects. diff --git a/pyproject.toml b/pyproject.toml index 13497eb..2088b3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "jsoniq" -version = "2.1.9" +version = "3.0.0" description = "Python edition of RumbleDB, a JSONiq engine" requires-python = ">=3.11" dependencies = [ diff --git a/src/jsoniq/configuration.py b/src/jsoniq/configuration.py new file mode 100644 index 0000000..5820ef0 --- /dev/null +++ b/src/jsoniq/configuration.py @@ -0,0 +1,39 @@ +class RumbleConfiguration: + """Session configuration adapter for RumbleDB's immutable Java configuration. + + Changes apply to subsequent queries. Already-created sequences keep the + configuration with which they were compiled. + """ + + def __init__(self, session): + self._session = session + + def set(self, path, value): + configuration = self._session._jrumblesession.getConfiguration() + builder = configuration.toBuilder() + configuration = getattr(builder, "with")(path, value).build() + self._session._jrumblesession = ( + self._session._sparksession._jvm.org.rumbledb.api.Rumble(configuration) + ) + return self + + def getResultSizeCap(self): + return self.getInt("runtime.resultsSizeCap") + + def setResultSizeCap(self, value): + return self.set("runtime.resultsSizeCap", value) + + def getMaterializationCap(self): + return self.getInt("runtime.materializationCap") + + def setMaterializationCap(self, value): + return self.set("runtime.materializationCap", value) + + def getShowErrorInfo(self): + return self.getBoolean("debug.showErrorInfo") + + def setShowErrorInfo(self, value): + return self.set("debug.showErrorInfo", value) + + def __getattr__(self, name): + return getattr(self._session._jrumblesession.getConfiguration(), name) diff --git a/src/jsoniq/session.py b/src/jsoniq/session.py index e2e6c8f..282436e 100644 --- a/src/jsoniq/session.py +++ b/src/jsoniq/session.py @@ -1,5 +1,7 @@ from pyspark.sql import SparkSession +from py4j.java_collections import JavaList from .sequence import SequenceOfItems +from .configuration import RumbleConfiguration import sys import platform import os @@ -34,9 +36,11 @@ class RumbleSession(object, metaclass=MetaRumbleSession): def __init__(self, spark_session: SparkSession): self._sparksession = spark_session self._jrumblesession = spark_session._jvm.org.rumbledb.api.Rumble(spark_session._jsparkSession) + self._configuration = RumbleConfiguration(self) + self._bindings = {} def getRumbleConf(self): - return self._jrumblesession.getConfiguration() + return self._configuration class Builder: def __init__(self): @@ -213,36 +217,33 @@ def convert(self, value): java_list = self._sparksession._jvm.java.util.ArrayList() java_list.add(self.convert(v)) java_map[k] = java_list - return self._sparksession._jvm.org.rumbledb.items.ItemFactory.getInstance().createObjectItem(java_map, False) + return self._sparksession._jvm.org.rumbledb.items.ItemFactory.getInstance().createObjectItemFromValueLists(java_map, False) else: raise ValueError("Cannot yet convert value of type " + str(type(value)) + " to a RumbleDB item. Please open an issue and we will look into it!") def unbind(self, name: str): - conf = self._jrumblesession.getConfiguration(); if not name.startswith("$"): raise ValueError("Variable name must start with a dollar symbol ('$').") name = name[1:] - conf.resetExternalVariableValue(name); + self._bindings.pop(name, None) def bind(self, name: str, valueToBind): - conf = self._jrumblesession.getConfiguration(); if not name.startswith("$"): raise ValueError("Variable name must start with a dollar symbol ('$').") name = name[1:] if isinstance(valueToBind, SequenceOfItems): outputs = valueToBind.availableOutputs() - if isinstance(outputs, list) and "DataFrame" in outputs: - conf.setExternalVariableValue(name, valueToBind.df()); - # TODO support binding a variable to an RDD - #elif isinstance(outputs, list) and "RDD" in outputs: - # conf.setExternalVariableValue(name, valueToBind.getAsRDD()); + if "DataFrame" in outputs: + self._bindings[name] = ("bindDataFrame", valueToBind.df()._jdf) else: - conf.setExternalVariableValue(name, valueToBind.items()); + self._bindings[name] = ("bindItems", valueToBind.items()) elif isinstance(valueToBind, pd.DataFrame): pysparkdf = self._sparksession.createDataFrame(valueToBind) - conf.setExternalVariableValue(name, pysparkdf._jdf); + self._bindings[name] = ("bindDataFrame", pysparkdf._jdf) elif isinstance(valueToBind, tuple): - conf.setExternalVariableValue(name, self.convert(valueToBind)) + self._bindings[name] = ("bindItems", self.convert(valueToBind)) + elif isinstance(valueToBind, JavaList): + self._bindings[name] = ("bindItems", valueToBind) elif isinstance(valueToBind, list): raise ValueError(""" To avoid confusion, a sequence of items must be provided as a Python tuple, not as a Python list. @@ -255,45 +256,48 @@ def bind(self, name: str, valueToBind): Example: [1,2,3] should then be rewritten as ([1,2,3],) for the sequence of one (array) item [1,2,3]. """) elif isinstance(valueToBind, dict): - conf.setExternalVariableValue(name, self.convert((valueToBind, ))) + self._bindings[name] = ("bindItems", self.convert((valueToBind,))) elif isinstance(valueToBind, str): - conf.setExternalVariableValue(name, self.convert((valueToBind, ))) + self._bindings[name] = ("bindItems", self.convert((valueToBind,))) elif isinstance(valueToBind, int): - conf.setExternalVariableValue(name, self.convert((valueToBind, ))) + self._bindings[name] = ("bindItems", self.convert((valueToBind,))) elif isinstance(valueToBind, float): - conf.setExternalVariableValue(name, self.convert((valueToBind, ))) + self._bindings[name] = ("bindItems", self.convert((valueToBind,))) elif isinstance(valueToBind, bool): - conf.setExternalVariableValue(name, self.convert((valueToBind, ))) + self._bindings[name] = ("bindItems", self.convert((valueToBind,))) elif valueToBind is None: - conf.setExternalVariableValue(name, self.convert((valueToBind, ))) + self._bindings[name] = ("bindItems", self.convert((valueToBind,))) elif(hasattr(valueToBind, "_get_object_id")): - conf.setExternalVariableValue(name, valueToBind); + self._bindings[name] = ("bindDataFrame", valueToBind) else: - conf.setExternalVariableValue(name, valueToBind._jdf); + self._bindings[name] = ("bindDataFrame", valueToBind._jdf) return self; def bindOne(self, name: str, value): return self.bind(name, (value,)) def bindDataFrameAsVariable(self, name: str, df): - conf = self._jrumblesession.getConfiguration(); if not name.startswith("$"): raise ValueError("Variable name must start with a dollar symbol ('$').") name = name[1:] if(hasattr(df, "_get_object_id")): - conf.setExternalVariableValue(name, df); + self._bindings[name] = ("bindDataFrame", df) else: - conf.setExternalVariableValue(name, df._jdf); + self._bindings[name] = ("bindDataFrame", df._jdf) return self; def jsoniq(self, str, **kwargs): - for key, value in kwargs.items(): - self.bind(f"${key}", value); - sequence = self._jrumblesession.runQuery(str); - seq = SequenceOfItems(sequence, self); - for key, value in kwargs.items(): - self.unbind(f"${key}"); - return seq; + previous_bindings = self._bindings.copy() + try: + for key, value in kwargs.items(): + self.bind(f"${key}", value) + bindings = self._sparksession._jvm.org.rumbledb.api.ExternalBindings() + for name, (method, value) in self._bindings.items(): + getattr(bindings, method)(name, value) + sequence = self._jrumblesession.runQuery(str, bindings) + return SequenceOfItems(sequence, self) + finally: + self._bindings = previous_bindings def __getattr__(self, item): return getattr(self._sparksession, item) diff --git a/tests/test_rumbledb_3_api.py b/tests/test_rumbledb_3_api.py new file mode 100644 index 0000000..82fc8b5 --- /dev/null +++ b/tests/test_rumbledb_3_api.py @@ -0,0 +1,124 @@ +import json +from unittest.mock import Mock, patch + +import pandas as pd +import pytest +from py4j.protocol import Py4JJavaError + +from jsoniq import RumbleSession + + +@pytest.fixture(scope="module") +def rumble(): + session = ( + RumbleSession.builder.master("local[2]") + .config("spark.ui.enabled", "false") + .getOrCreate() + ) + yield session + session.stop() + RumbleSession._rumbleSession = None + + +def test_configuration_updates_preserve_session_and_other_settings(rumble): + conf = rumble.getRumbleConf() + spark = rumble._sparksession._jsparkSession + rumble.bind("$saved", 42) + try: + conf.setResultSizeCap(2) + conf.setShowErrorInfo(True) + assert conf.getResultSizeCap() == 2 + assert conf.getShowErrorInfo() is True + assert rumble.getRumbleConf() is conf + assert rumble._sparksession._jsparkSession == spark + assert rumble.jsoniq("$saved").json() == (42,) + assert len(rumble.jsoniq("1 to 5").first()) == 2 + assert rumble.jsoniq("1 to 5").json() == (1, 2, 3, 4, 5) + finally: + conf.setResultSizeCap(10) + conf.setShowErrorInfo(False) + rumble.unbind("$saved") + + +def test_materialization_cap_applies_to_new_queries(rumble): + conf = rumble.getRumbleConf() + original = conf.getMaterializationCap() + existing = rumble.jsoniq("1 to 3") + try: + conf.setMaterializationCap(2) + assert existing.json() == (1, 2, 3) + with pytest.raises(Py4JJavaError, match="Cannot materialize"): + rumble.jsoniq("1 to 3").json() + finally: + conf.setMaterializationCap(original) + + +@pytest.mark.parametrize("value", [42, 1.5, True, None, "hello", {"a": [1, False, None]}, (1, 2, 3), ([1, 2],)]) +def test_python_value_bindings(rumble, value): + expected = value if isinstance(value, tuple) else (value,) + assert rumble.jsoniq("$value", value=value).json() == expected + with pytest.raises(Py4JJavaError): + rumble.jsoniq("$value") + + +def test_keyword_bindings_restore_persistent_values_even_on_failure(rumble): + rumble.bind("$value", 7) + try: + result = rumble.jsoniq("$value", value=9) + assert rumble.jsoniq("$value").json() == (7,) + assert result.json() == (9,) + with pytest.raises(Py4JJavaError): + rumble.jsoniq("1 +", value=11) + assert rumble.jsoniq("$value").json() == (7,) + with pytest.raises(ValueError): + rumble.jsoniq("$value", value=11, invalid=[1]) + assert rumble.jsoniq("$value").json() == (7,) + finally: + rumble.unbind("$value") + + +def test_dataframe_and_sequence_bindings(rumble): + frame = rumble.createDataFrame([(1,), (2,)], ["n"]) + assert sorted(rumble.jsoniq("$rows.n", rows=frame).json()) == [1, 2] + assert sorted(rumble.jsoniq("$rows.n", rows=frame._jdf).json()) == [1, 2] + assert sorted(rumble.jsoniq("$rows.n", rows=pd.DataFrame({"n": [1, 2]})).json()) == [1, 2] + rumble.bindDataFrameAsVariable("$rows", frame) + try: + sequence = rumble.jsoniq("$rows") + assert sorted(rumble.jsoniq("$copy.n", copy=sequence).json()) == [1, 2] + finally: + rumble.unbind("$rows") + local = rumble.jsoniq('(1, "two")') + assert rumble.jsoniq("$copy", copy=local).json() == (1, "two") + assert rumble.jsoniq("$copy", copy=local.items()).json() == (1, "two") + + +def test_item_iteration_and_rdd(rumble): + sequence = rumble.jsoniq("1 to 3") + assert [json.loads(item.serializeAsJSON()) for item in sequence.take(2)] == [1, 2] + sequence.open() + try: + assert sequence.hasNext() + assert sequence.nextJSON() == "1" + finally: + sequence.close() + assert rumble.jsoniq("1 to 3").rdd().collect() == [1, 2, 3] + + +def test_notebook_extension_and_display(rumble, capsys): + pytest.importorskip("IPython") + from jsoniqmagic import JSONiqMagic, load_ipython_extension + + shell = Mock() + with patch.object(RumbleSession.Builder, "getOrCreate", return_value=rumble): + load_ipython_extension(shell) + shell.register_magics.assert_called_once_with(JSONiqMagic) + assert rumble.getRumbleConf().getResultSizeCap() == 10 + rumble.getRumbleConf().setResultSizeCap(2) + try: + JSONiqMagic().run("", "1 to 3") + output = capsys.readouterr().out + assert "Displaying the first 2 items" in output + assert output.endswith("1\n2\n") + finally: + rumble.getRumbleConf().setResultSizeCap(10) diff --git a/tests/test_sample.py b/tests/test_sample.py index d1dc3f0..35c5e80 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -1,4 +1,6 @@ from jsoniq import RumbleSession +from pathlib import Path +from tempfile import TemporaryDirectory from unittest import TestCase import json import pandas as pd @@ -111,7 +113,8 @@ def test1(self): default return "?" : $v } """) - self.assertTrue(json.dumps(seq.json()) == json.dumps(({'even': [ 2, 4, 6 ] }, { 'odd': [ 1, 3, 5 ]},))) + # Without order by, the test should accept either group order. + self.assertCountEqual(seq.json(), ({'even': [2, 4, 6]}, {'odd': [1, 3, 5]})) rumble.bind('$c', ([1,2,3],[4,5,6])) seq = rumble.jsoniq(""" @@ -278,11 +281,12 @@ def test1(self): # RumbleDB was already tested with up to 64 AWS machines and 100s of TBs of data. # Of course the examples below are so small that it makes more sense to process the results locally with Python, # but this shows how GBs or TBs of data obtained from JSONiq can be written back to disk. - seq = rumble.jsoniq("$a.Name"); - seq.write().mode("overwrite").json("outputjson"); - seq.write().mode("overwrite").parquet("outputparquet"); + with TemporaryDirectory() as output_dir: + seq = rumble.jsoniq("$a.Name"); + seq.write().mode("overwrite").json((Path(output_dir) / "outputjson").as_posix()); + seq.write().mode("overwrite").parquet((Path(output_dir) / "outputparquet").as_posix()); - seq = rumble.jsoniq("1+1"); - seq.write().mode("overwrite").text("outputtext"); + seq = rumble.jsoniq("1+1"); + seq.write().mode("overwrite").text((Path(output_dir) / "outputtext").as_posix()); self.assertTrue(True) From 2afd4e469bb5d79591c6c32090025d57f8eada9a Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 28 Sep 2026 15:01:41 +0200 Subject: [PATCH 4/9] Use Java API. --- README.md | 2 +- src/jsoniq/configuration.py | 18 ------------------ src/jsoniq/sequence.py | 2 +- src/jsoniqmagic/__init__.py | 2 +- src/jsoniqmagic/magic.py | 18 +++++++++--------- tests/test_rumbledb_3_api.py | 24 ++++++++++++------------ tests/test_sample.py | 2 +- 7 files changed, 25 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 0268759..8c68454 100644 --- a/README.md +++ b/README.md @@ -342,7 +342,7 @@ Even more queries can be found [here](https://colab.research.google.com/github/R ## Version 3.0.0 - Upgraded to RumbleDB 3.0.0 and its immutable configuration and external bindings APIs. -- Preserved `getRumbleConf().setResultSizeCap(...)`, `setMaterializationCap(...)`, and `setShowErrorInfo(...)` through a Python adapter. Configuration changes apply to subsequent queries; existing sequences retain their compilation settings. +- Configuration reads use the Java API directly, such as `getInt("runtime.resultsSizeCap")` and `getBoolean("debug.showErrorInfo")`. The Python `set(path, value)` helper rebuilds the immutable Java configuration. Configuration changes apply to subsequent queries; existing sequences retain their compilation settings. - Fixed object conversion and binding query results as DataFrames. Keyword bindings are scoped to a query and restore persistent bindings even when query compilation fails. Configuration can also be changed using RumbleDB 3.0's dot-separated paths: diff --git a/src/jsoniq/configuration.py b/src/jsoniq/configuration.py index 5820ef0..53b4807 100644 --- a/src/jsoniq/configuration.py +++ b/src/jsoniq/configuration.py @@ -17,23 +17,5 @@ def set(self, path, value): ) return self - def getResultSizeCap(self): - return self.getInt("runtime.resultsSizeCap") - - def setResultSizeCap(self, value): - return self.set("runtime.resultsSizeCap", value) - - def getMaterializationCap(self): - return self.getInt("runtime.materializationCap") - - def setMaterializationCap(self, value): - return self.set("runtime.materializationCap", value) - - def getShowErrorInfo(self): - return self.getBoolean("debug.showErrorInfo") - - def setShowErrorInfo(self, value): - return self.set("debug.showErrorInfo", value) - def __getattr__(self, name): return getattr(self._session._jrumblesession.getConfiguration(), name) diff --git a/src/jsoniq/sequence.py b/src/jsoniq/sequence.py index 2bb729d..e1bcd08 100644 --- a/src/jsoniq/sequence.py +++ b/src/jsoniq/sequence.py @@ -40,7 +40,7 @@ def take(self, n): return self._rumblesession.lastResult def first(self): - self._rumblesession.lastResult = tuple(self.getFirstItemsAsList(self._rumblesession.getRumbleConf().getResultSizeCap())) + self._rumblesession.lastResult = tuple(self.getFirstItemsAsList(self._rumblesession.getRumbleConf().getInt("runtime.resultsSizeCap"))) return self._rumblesession.lastResult def json(self): diff --git a/src/jsoniqmagic/__init__.py b/src/jsoniqmagic/__init__.py index 2c4ccce..5ed792c 100644 --- a/src/jsoniqmagic/__init__.py +++ b/src/jsoniqmagic/__init__.py @@ -5,5 +5,5 @@ def load_ipython_extension(ipython): rumble = RumbleSession.builder.getOrCreate(); - rumble.getRumbleConf().setResultSizeCap(10); + rumble.getRumbleConf().set("runtime.resultsSizeCap", 10); ipython.register_magics(JSONiqMagic) \ No newline at end of file diff --git a/src/jsoniqmagic/magic.py b/src/jsoniqmagic/magic.py index c607f4c..3da7d52 100644 --- a/src/jsoniqmagic/magic.py +++ b/src/jsoniqmagic/magic.py @@ -70,13 +70,13 @@ def run(self, line, cell=None, timed=False): try: df = response.df(); except Py4JJavaError as e: - if rumble.getRumbleConf().getShowErrorInfo() : + if rumble.getRumbleConf().getBoolean("debug.showErrorInfo") : raise e; else: print(e.java_exception.getMessage()) return except Exception as e: - if rumble.getRumbleConf().getShowErrorInfo() : + if rumble.getRumbleConf().getBoolean("debug.showErrorInfo") : raise e; else: print("Query unsuccessful.") @@ -95,7 +95,7 @@ def run(self, line, cell=None, timed=False): try: pdf = response.pdf() except Py4JJavaError as e: - if rumble.getRumbleConf().getShowErrorInfo() : + if rumble.getRumbleConf().getBoolean("debug.showErrorInfo") : raise e; else: print(e.java_exception.getMessage()) @@ -118,7 +118,7 @@ def run(self, line, cell=None, timed=False): try: response.applyPUL() except Py4JJavaError as e: - if rumble.getRumbleConf().getShowErrorInfo() : + if rumble.getRumbleConf().getBoolean("debug.showErrorInfo") : raise e; else: print(e.java_exception.getMessage()) @@ -139,9 +139,9 @@ def run(self, line, cell=None, timed=False): if (args.json or (not args.pandas_data_frame and not args.pyspark_data_frame)): try: - capplusone = response.take(rumble.getRumbleConf().getResultSizeCap() + 1) + capplusone = response.take(rumble.getRumbleConf().getInt("runtime.resultsSizeCap") + 1) except Py4JJavaError as e: - if rumble.getRumbleConf().getShowErrorInfo() : + if rumble.getRumbleConf().getBoolean("debug.showErrorInfo") : raise e; else: print(e.java_exception.getMessage()) @@ -156,10 +156,10 @@ def run(self, line, cell=None, timed=False): print("Query unsuccessful.") print("Usual reasons: firewall, misconfigured proxy.") return - if len(capplusone) > rumble.getRumbleConf().getResultSizeCap(): + if len(capplusone) > rumble.getRumbleConf().getInt("runtime.resultsSizeCap"): count = response.count() - print("The query output %s items, which is too many to display. Displaying the first %s items:" % (count, rumble.getRumbleConf().getResultSizeCap())) - for e in capplusone[:rumble.getRumbleConf().getResultSizeCap()]: + print("The query output %s items, which is too many to display. Displaying the first %s items:" % (count, rumble.getRumbleConf().getInt("runtime.resultsSizeCap"))) + for e in capplusone[:rumble.getRumbleConf().getInt("runtime.resultsSizeCap")]: print(json.dumps(json.loads(e.serializeAsJSON()), indent=2)) end = time.time() diff --git a/tests/test_rumbledb_3_api.py b/tests/test_rumbledb_3_api.py index 82fc8b5..bcdce17 100644 --- a/tests/test_rumbledb_3_api.py +++ b/tests/test_rumbledb_3_api.py @@ -25,32 +25,32 @@ def test_configuration_updates_preserve_session_and_other_settings(rumble): spark = rumble._sparksession._jsparkSession rumble.bind("$saved", 42) try: - conf.setResultSizeCap(2) - conf.setShowErrorInfo(True) - assert conf.getResultSizeCap() == 2 - assert conf.getShowErrorInfo() is True + conf.set("runtime.resultsSizeCap", 2) + conf.set("debug.showErrorInfo", True) + assert conf.getInt("runtime.resultsSizeCap") == 2 + assert conf.getBoolean("debug.showErrorInfo") is True assert rumble.getRumbleConf() is conf assert rumble._sparksession._jsparkSession == spark assert rumble.jsoniq("$saved").json() == (42,) assert len(rumble.jsoniq("1 to 5").first()) == 2 assert rumble.jsoniq("1 to 5").json() == (1, 2, 3, 4, 5) finally: - conf.setResultSizeCap(10) - conf.setShowErrorInfo(False) + conf.set("runtime.resultsSizeCap", 10) + conf.set("debug.showErrorInfo", False) rumble.unbind("$saved") def test_materialization_cap_applies_to_new_queries(rumble): conf = rumble.getRumbleConf() - original = conf.getMaterializationCap() + original = conf.getInt("runtime.materializationCap") existing = rumble.jsoniq("1 to 3") try: - conf.setMaterializationCap(2) + conf.set("runtime.materializationCap", 2) assert existing.json() == (1, 2, 3) with pytest.raises(Py4JJavaError, match="Cannot materialize"): rumble.jsoniq("1 to 3").json() finally: - conf.setMaterializationCap(original) + conf.set("runtime.materializationCap", original) @pytest.mark.parametrize("value", [42, 1.5, True, None, "hello", {"a": [1, False, None]}, (1, 2, 3), ([1, 2],)]) @@ -113,12 +113,12 @@ def test_notebook_extension_and_display(rumble, capsys): with patch.object(RumbleSession.Builder, "getOrCreate", return_value=rumble): load_ipython_extension(shell) shell.register_magics.assert_called_once_with(JSONiqMagic) - assert rumble.getRumbleConf().getResultSizeCap() == 10 - rumble.getRumbleConf().setResultSizeCap(2) + assert rumble.getRumbleConf().getInt("runtime.resultsSizeCap") == 10 + rumble.getRumbleConf().set("runtime.resultsSizeCap", 2) try: JSONiqMagic().run("", "1 to 3") output = capsys.readouterr().out assert "Displaying the first 2 items" in output assert output.endswith("1\n2\n") finally: - rumble.getRumbleConf().setResultSizeCap(10) + rumble.getRumbleConf().set("runtime.resultsSizeCap", 10) diff --git a/tests/test_sample.py b/tests/test_sample.py index 35c5e80..84ce899 100644 --- a/tests/test_sample.py +++ b/tests/test_sample.py @@ -13,7 +13,7 @@ def test1(self): # All attributes and methods of SparkSession are also available on RumbleSession. rumble = RumbleSession.builder.getOrCreate(); - rumble.getRumbleConf().setResultSizeCap(100); + rumble.getRumbleConf().set("runtime.resultsSizeCap", 100); # Just to improve readability when invoking Spark methods # (such as spark.sql() or spark.createDataFrame()). From 2694a94a1c956417e9e3dd448c8c8ad58243457b Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 28 Sep 2026 15:17:49 +0200 Subject: [PATCH 5/9] Adapt README. --- README.md | 166 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 99 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 8c68454..c287508 100644 --- a/README.md +++ b/README.md @@ -81,15 +81,22 @@ after installing the library. You can directly copy paste the code below to a Python file and execute it with Python. -``` +```python from jsoniq import RumbleSession import pandas as pd # The syntax to start a session is similar to that of Spark. # A RumbleSession is a SparkSession that additionally knows about RumbleDB. -# All attributes and methods of SparkSession are also available on RumbleSession. +# SparkSession methods such as createDataFrame() and sql() are also available on RumbleSession. + +rumble = RumbleSession.builder.getOrCreate() -rumble = RumbleSession.builder.getOrCreate(); +# Configure subsequent queries using RumbleDB 3.0's dot-separated paths. +# The result size cap controls first() and notebook display; json() uses the materialization cap. +rumble.getRumbleConf().set("runtime.resultsSizeCap", 100) +rumble.getRumbleConf().set("runtime.materializationCap", 100000) +rumble.getRumbleConf().set("debug.showErrorInfo", True) +print(rumble.getRumbleConf().getInt("runtime.resultsSizeCap")) # Just to improve readability when invoking Spark methods # (such as spark.sql() or spark.createDataFrame()). @@ -105,12 +112,16 @@ spark = rumble # of items, here the sequence with just the integer item 2. items = rumble.jsoniq('1+1') -# A sequence of items can simply be converted to a list of Python/JSON values with json(). -# Since there is only one value in the sequence output by this query, -# we get a singleton list with the integer 2. -# Generally though, the results may contain zero, one, two, or more items. -python_list = items.json() -print(python_list) +# json() collects the complete result as a tuple of native Python values. +# Here the result is (2,). JSON arrays become lists and JSON objects become dicts. +# Collection is subject to runtime.materializationCap. +python_values = items.json() +print(python_values) + +# take(n) and first() return Java Item objects, not native Python values. +# first() uses runtime.resultsSizeCap; take(n) uses the requested count. +print([item.serializeAsJSON() for item in rumble.jsoniq("1 to 5").take(2)]) +print([item.serializeAsJSON() for item in rumble.jsoniq("1 to 5").first()]) ############################################ ##### More complex, standalone queries ##### @@ -147,9 +158,9 @@ let $join := "sold" : $sale.product } return [$join] -"""); +""") -print(seq.json()); +print(seq.json()) seq = rumble.jsoniq(""" for $product in json-lines("http://rumbledb.org/samples/products-small.json", 10) @@ -159,8 +170,8 @@ return { "store" : $store-number, "products" : [ distinct-values($product.product) ] } -"""); -print(seq.json()); +""") +print(seq.json()) ############################################################ ###### Binding JSONiq variables to Python values ########### @@ -168,17 +179,17 @@ print(seq.json()); # It is possible to bind a JSONiq variable to a tuple of native Python values # and then use it in a query. -# JSONiq, variables are bound to sequences of items, just like the results of JSONiq -# queries are sequence of items. +# In JSONiq, variables and query results are sequences of items. # A Python tuple will be seamlessly converted to a sequence of items by the library. -# Currently we only support strs, ints, floats, booleans, None, lists, and dicts. -# But if you need more (like date, bytes, etc) we will add them without any problem. -# JSONiq has a rich type system. - +# Scalars can be strings, ints, floats, booleans, None, or dicts. +# Lists represent array items: wrap a list in a singleton tuple to bind one array. +# Keyword bindings apply only to this query; they do not persist on the session. + print(rumble.jsoniq(""" for $v in $c let $parity := $v mod 2 group by $parity +order by $parity return { switch($parity) case 0 return "even" case 1 return "odd" @@ -196,13 +207,22 @@ return [ print(rumble.jsoniq('{ "results" : $c.foo[[2]] }', c=({"foo":[1,2,3]},{"foo":[4,{"bar":[1,False, None]},6]})).json()) -# It is possible to bind only one value. The it must be provided as a singleton tuple. -# This is because in JSONiq, an item is the same a sequence of one item. +# A singleton tuple binds a sequence containing one item. print(rumble.jsoniq('for $i in 1 to $c return $i*$i', c=(42,)).json()) -# For convenience and code readability, you can also use bindOne(). +# A scalar keyword argument also binds one item, without requiring a tuple. print(rumble.jsoniq('for $i in 1 to $c return $i*$i', c=42).json()) +# Use bind() or bindOne() for a binding shared by subsequent queries. +# A keyword argument temporarily overrides it; unbind() removes it. +rumble.bindOne("$limit", 3) +try: + print(rumble.jsoniq("1 to $limit").json()) # (1, 2, 3) + print(rumble.jsoniq("1 to $limit", limit=2).json()) # (1, 2) + print(rumble.jsoniq("1 to $limit").json()) # (1, 2, 3) +finally: + rumble.unbind("$limit") + ########################################################## ##### Binding JSONiq variables to pandas DataFrames ###### ##### Getting the output as a Pandas DataFrame ###### @@ -210,8 +230,8 @@ print(rumble.jsoniq('for $i in 1 to $c return $i*$i', c=42).json()) # Creating a dummy pandas dataframe data = {'Name': ['Alice', 'Bob', 'Charlie'], - 'Age': [30,25,35]}; -pdf = pd.DataFrame(data); + 'Age': [30,25,35]} +pdf = pd.DataFrame(data) # Binding a pandas dataframe seq = rumble.jsoniq('$a.Name', a=pdf) @@ -231,22 +251,23 @@ print(seq.pdf()) # DataFrame as a sequence of object items. # Create a data frame also similar to Spark (but using the rumble object). -data = [("Alice", 30), ("Bob", 25), ("Charlie", 35)]; -columns = ["Name", "Age"]; -df = spark.createDataFrame(data, columns); +data = [("Alice", 30), ("Bob", 25), ("Charlie", 35)] +columns = ["Name", "Age"] +df = spark.createDataFrame(data, columns) # You can bind JSONiq variables to pyspark DataFrames as follows. You can bind as many variables as you want. # Since variable $a is bound to a DataFrame, it is automatically declared as an external variable # and can be used in the query. In JSONiq, it is logically a sequence of objects. -res = rumble.jsoniq('$a.Name', a=df); +res = rumble.jsoniq('$a.Name', a=df) # There are several ways to collect the outputs, depending on the user needs but also # on the query supplied. # This returns a list containing one or several of "DataFrame", "RDD", "PUL", "Local" # If DataFrame is in the list, df() can be invoked. # If RDD is in the list, rdd() can be invoked. -# If Local is the list, items() or json() can be invokved, as well as the local iterator API. -modes = res.availableOutputs(); +# If Local is in the list, items(), json(), and the local iterator API are available. +# If PUL is in the list, applyPUL() can apply the pending updates. +modes = res.availableOutputs() for mode in modes: print(mode) @@ -256,29 +277,33 @@ for mode in modes: # If the output of the JSONiq query is structured (i.e., RumbleDB was able to detect a schema), # then we can extract a regular data frame that can be further processed with spark.sql() or rumble.jsoniq(). -df = res.df(); -df.show(); +df = res.df() +df.show() # We are continuously working on the detection of schemas and RumbleDB will get better at it with them. # JSONiq is a very powerful language and can also produce heterogeneous output "by design". Then you need -# to use rdd() instead of df(), or to collect the list of JSON values (see further down). Remember +# to use rdd() instead of df(), or to collect the tuple of native Python values (see further down). Remember # that availableOutputs() tells you what is at your disposal. # A DataFrame output by JSONiq can be reused as input to a Spark SQL query. # (Remember that rumble is a wrapper around a SparkSession object, so you can use rumble.sql() just like spark.sql()) -df.createTempView("myview") -df2 = spark.sql("SELECT * FROM myview").toDF("name"); -df2.show(); +df.createOrReplaceTempView("myview") +df2 = spark.sql("SELECT * FROM myview").toDF("name") +df2.show() # A DataFrame output by Spark SQL can be reused as input to a JSONiq query. -seq2 = rumble.jsoniq("for $i in 1 to 5 return $b", b=df2); -df3 = seq2.df(); -df3.show(); +seq2 = rumble.jsoniq("for $i in 1 to 5 return $b", b=df2) +df3 = seq2.df() +df3.show() # And a DataFrame output by JSONiq can be reused as input to another JSONiq query. -seq3 = rumble.jsoniq("$b[position() lt 3]", b=df3); -df4 = seq3.df(); -df4.show(); +seq3 = rumble.jsoniq("$b[position() lt 3]", b=df3) +df4 = seq3.df() +df4.show() + +# A SequenceOfItems result can also be bound directly, preserving DataFrame execution +# when available. No explicit conversion with df() is needed. +print(rumble.jsoniq("$rows.name", rows=seq3).json()) ######################### ##### Local access ###### @@ -286,36 +311,40 @@ df4.show(); # This materializes the rows as items. # The items are accessed with the RumbleDB Item API. -list = res.items(); -for result in list: +java_items = res.items() +for result in java_items: print(result.getStringValue()) # This streams through the items one by one -res.open(); -while (res.hasNext()): - print(res.next().getStringValue()); -res.close(); +res.open() +try: + while res.hasNext(): + print(res.next().getStringValue()) +finally: + res.close() ################################################################################################################ ###### Native Python/JSON Access for bypassing the Item API (but losing on the richer JSONiq type system) ###### ################################################################################################################ -# This method directly gets the result as JSON (dict, list, strings, ints, etc). -jlist = res.json(); -for str in jlist: - print(str); +# This returns a tuple of native Python values (dicts, lists, strings, ints, etc.). +python_values = res.json() +for value in python_values: + print(value) -# This streams through the JSON values one by one. -res.open(); -while(res.hasNext()): - print(res.nextJSON()); -res.close(); +# nextJSON() returns each item as a serialized JSON string. +res.open() +try: + while res.hasNext(): + print(res.nextJSON()) +finally: + res.close() # This gets an RDD of JSON values that can be processed by Python -rdd = res.rdd(); -print(rdd.count()); -for str in rdd.take(10): - print(str); +rdd = res.rdd() +print(rdd.count()) +for value in rdd.take(10): + print(value) ################################################### ###### Write back to the disk (or data lake) ###### @@ -326,12 +355,15 @@ for str in rdd.take(10): # RumbleDB was already tested with up to 64 AWS machines and 100s of TBs of data. # Of course the examples below are so small that it makes more sense to process the results locally with Python, # but this shows how GBs or TBs of data obtained from JSONiq can be written back to disk. -seq = rumble.jsoniq("$a.Name", a=spark.createDataFrame(data, columns)); -seq.write().mode("overwrite").json("outputjson"); -seq.write().mode("overwrite").parquet("outputparquet"); +seq = rumble.jsoniq("$a.Name", a=spark.createDataFrame(data, columns)) +# Select the format before setting the mode, so it applies to the final writer. +seq.write().format("json").mode("overwrite").save("outputjson") +seq.write().format("parquet").mode("overwrite").save("outputparquet") + +seq = rumble.jsoniq("1+1") +seq.write().format("xml-json-hybrid").mode("overwrite").save("outputtext") -seq = rumble.jsoniq("1+1"); -seq.write().mode("overwrite").text("outputtext"); +spark.stop() ``` # How to learn JSONiq, and more query examples @@ -443,7 +475,7 @@ The result size cap controls `first()` and notebook display. `json()` retrieves - Now also accepts pandas version 2.2. ## Version 0.2.0 alpha 2 -- You can change the result size cap through to the now accessible Rumble configuration (for example rumble .getRumbleConf().setResultSizeCap(10)). This controls how many items can be retrieved at most with a json() call. You can increase it to whichever number you would like if you reach the cap. +- Added access to the Rumble configuration to change the result size cap. For the current API and cap semantics, see the Version 3.0.0 notes above. - Add the JSONiq magic to execute JSONiq queries directly in a notebook cell, using the RumbleDB instance shipped with the library. - RumbleSession.builder.getOrCreate() now correctly reuses an existing session instead of creating a new object. It preserves the configuration. From d8c363202632967ee4d3bf63390b8aa69061557b Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 28 Sep 2026 15:35:35 +0200 Subject: [PATCH 6/9] Add explanations for obsolete methods. --- src/jsoniq/configuration.py | 559 ++++++++++++++++++++++++++++++++++++ 1 file changed, 559 insertions(+) diff --git a/src/jsoniq/configuration.py b/src/jsoniq/configuration.py index 53b4807..523f551 100644 --- a/src/jsoniq/configuration.py +++ b/src/jsoniq/configuration.py @@ -17,5 +17,564 @@ def set(self, path, value): ) return self + # Migration errors for methods declared in the legacy RumbleRuntimeConfiguration. + # Source: https://rumbledb.org/docs/latest/api/org/rumbledb/config/RumbleRuntimeConfiguration.html + # Java overloads share a Python method; inherited java.lang.Object methods are not included. + + def applyUpdates(self): + raise NotImplementedError( + 'applyUpdates() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("runtime.shouldApplyUpdates") instead.' + ) + + def dataFrameExecution(self): + raise NotImplementedError( + 'dataFrameExecution() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("runtime.useDataFrameExecution") instead.' + ) + + def dateWithTimezone(self): + raise NotImplementedError( + 'dateWithTimezone() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("semantics.datesWithTimeZone") instead.' + ) + + def doStaticAnalysis(self): + raise NotImplementedError( + 'doStaticAnalysis() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("analysis.enableStaticTyping") instead.' + ) + + def functionInlining(self): + raise NotImplementedError( + 'functionInlining() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("optimization.useFunctionInlining") instead.' + ) + + def getAllowedURIPrefixes(self): + raise NotImplementedError( + 'getAllowedURIPrefixes() was removed in RumbleDB 3.0.0. URI restrictions were removed ' + 'along with the server feature in RumbleDB 3.0. Use the Python library to run ' + 'RumbleDB in notebooks: from jsoniq import RumbleSession; rumble = ' + 'RumbleSession.builder.getOrCreate(); rumble.jsoniq("1 + 1").json().' + ) + + def getDataFrameExecutionModeDetection(self): + raise NotImplementedError( + 'getDataFrameExecutionModeDetection() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("runtime.detectDataFrameExecutionMode") instead.' + ) + + @staticmethod + def getDefaultConfiguration(): + raise NotImplementedError( + 'getDefaultConfiguration() was removed in RumbleDB 3.0.0. ' + 'Use rumble._sparksession._jvm.org.rumbledb.api.RumbleConfiguration() to create a default Java configuration; this does not change the active session.' + ) + + def getExternalVariableValue(self, name): + raise NotImplementedError( + 'getExternalVariableValue() was removed in RumbleDB 3.0.0. Bind a sequence with ' + 'rumble.bind("$name", tuple(items)); for a single value (including an array), use ' + 'rumble.bindOne("$name", value). For a Java List, use rumble.bind("$name", ' + 'java_items). To evaluate a persistent binding, use rumble.jsoniq("$name").items() ' + 'for Java items or .json() for Python values; retain the original value if you need ' + 'it without evaluation.' + ) + + def getExternalVariableValueReadFromDataFrame(self, name): + raise NotImplementedError( + 'getExternalVariableValueReadFromDataFrame() was removed in RumbleDB 3.0.0. Use ' + 'rumble.bind("$name", dataframe) or rumble.jsoniq(query, name=dataframe) to bind a ' + 'DataFrame. Retain the original DataFrame in Python; rumble.jsoniq("$name").df() ' + 'evaluates a persistent binding as a DataFrame.' + ) + + def getExternalVariableValueReadFromFile(self, name): + raise NotImplementedError( + 'getExternalVariableValueReadFromFile() was removed in RumbleDB 3.0.0. ' + 'Retain path in Python; configuration no longer stores file-binding paths. ' + 'To read and bind a JSON document instead:\n' + 'import json\n' + 'with open(path) as source:\n' + ' rumble.bindOne("$name", json.load(source))\n' + 'For a text file:\n' + 'with open(path) as source:\n' + ' rumble.bindOne("$name", source.read())\n' + 'bindOne() preserves a JSON array as one item. ' + 'For large JSON Lines files, use rumble.bind("$name", ' + 'rumble.jsoniq("json-lines($path)", path=path)); other formats can use their JSONiq ' + 'input function or a Spark DataFrame.' + ) + + def getExternalVariablesReadFromDataFrames(self): + raise NotImplementedError( + 'getExternalVariablesReadFromDataFrames() was removed in RumbleDB 3.0.0. Bind a ' + 'DataFrame with rumble.bind("$name", dataframe), or use rumble.jsoniq(query, ' + 'name=dataframe) for one query. There is no public replacement for listing binding ' + 'names; keep those names in your Python application.' + ) + + def getExternalVariablesReadFromListsOfItems(self): + raise NotImplementedError( + 'getExternalVariablesReadFromListsOfItems() was removed in RumbleDB 3.0.0. Bind a ' + 'sequence of Python items with rumble.bind("$name", tuple(items)), or use ' + 'rumble.jsoniq(query, name=tuple(items)) for one query. For a Java List, pass ' + 'it directly to rumble.bind("$name", java_items). There is no public replacement for ' + 'listing binding names; keep those names in your Python application.' + ) + + def getHost(self): + raise NotImplementedError( + 'getHost() was removed in RumbleDB 3.0.0. The server feature was removed in RumbleDB ' + '3.0. Use the Python library to run RumbleDB in notebooks: from jsoniq import ' + 'RumbleSession; rumble = RumbleSession.builder.getOrCreate(); rumble.jsoniq("1 + ' + '1").json().' + ) + + def getInputFormat(self, variableName=None): + raise NotImplementedError( + 'getInputFormat() was removed in RumbleDB 3.0.0. Neither the global nor the ' + 'per-variable format getter has a Python replacement. Keep the selected format ' + 'in your Python application; it determines how you parse input before binding. ' + 'For JSON from standard input:\n' + 'import json, sys\n' + 'rumble.bindOne("$input", json.load(sys.stdin))\n' + 'For text from standard input:\n' + 'import sys\n' + 'rumble.bindOne("$input", sys.stdin.read())\n' + 'bindOne() preserves a JSON array as one item. Replace "$input" with your ' + 'variable name. Input formats now belong to individual engine bindings; the ' + 'public Java ExternalBindings API does not expose them yet.' + ) + + def getLaxJSONNullValidation(self): + raise NotImplementedError( + 'getLaxJSONNullValidation() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("semantics.laxJSONNullValidation") instead.' + ) + + def getLogPath(self): + raise NotImplementedError( + 'getLogPath() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getString("output.logPath") instead.' + ) + + def getMaterializationCap(self): + raise NotImplementedError( + 'getMaterializationCap() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getInt("runtime.materializationCap") instead.' + ) + + def getNativeSQLPredicates(self): + raise NotImplementedError( + 'getNativeSQLPredicates() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("runtime.useNativeSQLPredicates") instead.' + ) + + def getNumberOfOutputPartitions(self): + raise NotImplementedError( + 'getNumberOfOutputPartitions() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getInt("output.numberOfOutputPartitions") instead.' + ) + + def getOutputPath(self): + raise NotImplementedError( + 'getOutputPath() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getString("output.outputPath") instead.' + ) + + def getOverwrite(self): + raise NotImplementedError( + 'getOverwrite() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("output.allowOverwrite") instead.' + ) + + def getPort(self): + raise NotImplementedError( + 'getPort() was removed in RumbleDB 3.0.0. The server feature was removed in RumbleDB ' + '3.0. Use the Python library to run RumbleDB in notebooks: from jsoniq import ' + 'RumbleSession; rumble = RumbleSession.builder.getOrCreate(); rumble.jsoniq("1 + ' + '1").json().' + ) + + def getQuery(self): + raise NotImplementedError( + 'getQuery() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getString("input.query") instead.' + ) + + def getQueryLanguage(self): + raise NotImplementedError( + 'getQueryLanguage() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getString("semantics.queryLanguage") instead.' + ) + + def getQueryPath(self): + raise NotImplementedError( + 'getQueryPath() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getString("input.queryPath") instead.' + ) + + def getResultSizeCap(self): + raise NotImplementedError( + 'getResultSizeCap() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getInt("runtime.resultsSizeCap") instead.' + ) + + def getSerializationParameters(self): + raise NotImplementedError( + 'getSerializationParameters() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().get("output.serializationParameters") instead. This returns configured parameters as plain Java values, not the former SerializationParameters object.' + ) + + def getShellFilter(self): + raise NotImplementedError( + 'getShellFilter() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getString("output.shellFilter") instead.' + ) + + def getShowErrorInfo(self): + raise NotImplementedError( + 'getShowErrorInfo() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("debug.showErrorInfo") instead.' + ) + + def getStaticBaseUri(self): + raise NotImplementedError( + 'getStaticBaseUri() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getString("semantics.staticBaseUri") instead.' + ) + + def getUnparsedExternalVariableValue(self, name): + raise NotImplementedError( + 'getUnparsedExternalVariableValue() was removed in RumbleDB 3.0.0. Keep the original ' + 'text in Python. To bind a string, use rumble.bindOne("$name", text); to parse and ' + 'bind a JSON value, use import json; rumble.bindOne("$name", json.loads(text)). These ' + 'are distinct operations. For Java lexical bindings, create ' + 'org.rumbledb.api.ExternalBindings and call bindLiteral(name, text).' + ) + + def getXmlVersion(self): + raise NotImplementedError( + 'getXmlVersion() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getString("semantics.xmlVersion") instead.' + ) + + def init(self): + raise NotImplementedError( + 'init() was removed in RumbleDB 3.0.0. ' + 'No direct replacement is needed: configuration is initialized when the session is created. Use rumble.getRumbleConf().set(path, value) to change a setting.' + ) + + def isCheckReturnTypeOfBuiltinFunctions(self): + raise NotImplementedError( + 'isCheckReturnTypeOfBuiltinFunctions() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("analysis.checkReturnTypeOfBuiltinFunctions") instead.' + ) + + def isLocal(self): + raise NotImplementedError( + 'isLocal() was removed in RumbleDB 3.0.0. ' + 'Use rumble.sparkContext._jsc.sc().isLocal() instead.' + ) + + def isPrintIteratorTree(self): + raise NotImplementedError( + 'isPrintIteratorTree() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("debug.printIteratorTree") instead.' + ) + + def isServer(self): + raise NotImplementedError( + 'isServer() was removed in RumbleDB 3.0.0. The server feature was removed in RumbleDB ' + '3.0. Use the Python library to run RumbleDB in notebooks: from jsoniq import ' + 'RumbleSession; rumble = RumbleSession.builder.getOrCreate(); rumble.jsoniq("1 + ' + '1").json().' + ) + + def isShell(self): + raise NotImplementedError( + 'isShell() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getString("mode") == "REPL" instead.' + ) + + def nativeExecution(self): + raise NotImplementedError( + 'nativeExecution() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("runtime.useNativeExecution") instead.' + ) + + def optimizeGeneralComparisonToValueComparison(self): + raise NotImplementedError( + 'optimizeGeneralComparisonToValueComparison() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("optimization.optimizeGeneralComparisonToValueComparison") instead.' + ) + + def optimizeParentPointers(self): + raise NotImplementedError( + 'optimizeParentPointers() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("optimization.optimizeParentPointers") instead.' + ) + + def optimizeStepExperimental(self): + raise NotImplementedError( + 'optimizeStepExperimental() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("optimization.optimizeStepsExperimental") instead.' + ) + + def optimizeSteps(self): + raise NotImplementedError( + 'optimizeSteps() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("optimization.optimizeSteps") instead.' + ) + + def parallelExecution(self): + raise NotImplementedError( + 'parallelExecution() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("runtime.useParallelExecution") instead.' + ) + + def printInferredTypes(self): + raise NotImplementedError( + 'printInferredTypes() was removed in RumbleDB 3.0.0. ' + 'Use rumble.getRumbleConf().getBoolean("analysis.printInferredTypes") instead.' + ) + + def read(self, kryo, input): + raise NotImplementedError( + 'read() was removed in RumbleDB 3.0.0. ' + 'No direct replacement: Kryo read/write hooks are not exposed by the public configuration API. Store settings in your application and restore them with rumble.getRumbleConf().set(path, value).' + ) + + def readFromStandardInput(self, variableName): + raise NotImplementedError( + 'readFromStandardInput() was removed in RumbleDB 3.0.0. This method used to ' + 'report whether a variable came from standard input; retain that source ' + 'information in Python. To actually read and bind JSON from standard input:\n' + 'import json, sys\n' + 'rumble.bindOne("$input", json.load(sys.stdin))\n' + 'For text from standard input:\n' + 'import sys\n' + 'rumble.bindOne("$input", sys.stdin.read())\n' + 'bindOne() preserves a JSON array as one item. Replace "$input" with your ' + 'variable name. The engine has input binding types, including ' + 'StandardInputBinding, but the public Java ' + 'ExternalBindings API does not expose them yet.' + ) + + def resetExternalVariableValue(self, name): + raise NotImplementedError( + 'resetExternalVariableValue() was removed in RumbleDB 3.0.0. Use ' + 'rumble.unbind("$name") instead, with a dollar-prefixed Python string instead of a ' + 'Java Name. Alternatively, use rumble.jsoniq(query, name=value) for a query-scoped ' + 'binding that is removed automatically after compilation and restores any previous ' + 'persistent binding.' + ) + + def setAllowedURIPrefixes(self, value): + raise NotImplementedError( + 'setAllowedURIPrefixes() was removed in RumbleDB 3.0.0. URI restrictions were removed ' + 'along with the server feature in RumbleDB 3.0. Use the Python library to run ' + 'RumbleDB in notebooks: from jsoniq import RumbleSession; rumble = ' + 'RumbleSession.builder.getOrCreate(); rumble.jsoniq("1 + 1").json().' + ) + + def setApplyUpdates(self, value): + raise NotImplementedError( + 'setApplyUpdates() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("runtime.shouldApplyUpdates", {value!r}) instead.' + ) + + def setCheckReturnTypeOfBuiltinFunctions(self, value): + raise NotImplementedError( + 'setCheckReturnTypeOfBuiltinFunctions() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("analysis.checkReturnTypeOfBuiltinFunctions", {value!r}) instead.' + ) + + def setDataFrameExecution(self, value): + raise NotImplementedError( + 'setDataFrameExecution() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("runtime.useDataFrameExecution", {value!r}) instead.' + ) + + def setDataFrameExecutionModeDetection(self, value): + raise NotImplementedError( + 'setDataFrameExecutionModeDetection() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("runtime.detectDataFrameExecutionMode", {value!r}) instead.' + ) + + def setDateWithTimezone(self, value): + raise NotImplementedError( + 'setDateWithTimezone() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("semantics.datesWithTimeZone", {value!r}) instead.' + ) + + def setExternalVariableValue(self, name, value): + raise NotImplementedError( + 'setExternalVariableValue() was removed in RumbleDB 3.0.0. Use rumble.bind("$name", ' + 'tuple(items)) for a Python sequence, rumble.bindOne("$name", value) for one item ' + '(including an array), or rumble.bind("$name", dataframe) for a DataFrame. A Java ' + 'List can be passed directly as rumble.bind("$name", java_items). For one ' + 'query, use rumble.jsoniq(query, name=value), with tuples for sequences and (array,) ' + 'for one array item. Replace the Java Name with a dollar-prefixed Python string for ' + 'bind()/bindOne(); keyword argument names omit the dollar sign.' + ) + + def setFunctionInlining(self, value): + raise NotImplementedError( + 'setFunctionInlining() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("optimization.useFunctionInlining", {value!r}) instead.' + ) + + def setInputFormat(self, value): + raise NotImplementedError( + 'setInputFormat() was removed in RumbleDB 3.0.0. Select the parser in Python ' + 'instead of setting a global format. For JSON from standard input:\n' + 'import json, sys\n' + 'rumble.bindOne("$input", json.load(sys.stdin))\n' + 'For text from standard input:\n' + 'import sys\n' + 'rumble.bindOne("$input", sys.stdin.read())\n' + 'For a file, open it with a with statement and pass the file object to ' + 'json.load(source) or source.read() instead of sys.stdin. bindOne() preserves ' + 'a JSON array as one item. The engine has FileBinding and StandardInputBinding ' + 'with InputFormat.JSON or InputFormat.TEXT, but the public Java ExternalBindings ' + 'API does not expose them yet. For CLI context-item input, use ' + '--context-item-input - --context-item-input-format json (or text).' + ) + + def setLaxJSONNullValidation(self, value): + raise NotImplementedError( + 'setLaxJSONNullValidation() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("semantics.laxJSONNullValidation", {value!r}) instead.' + ) + + def setLogPath(self, value): + raise NotImplementedError( + 'setLogPath() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("output.logPath", {value!r}) instead.' + ) + + def setMaterializationCap(self, value): + raise NotImplementedError( + 'setMaterializationCap() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("runtime.materializationCap", {value!r}) instead.' + ) + + def setNativeExecution(self, value): + raise NotImplementedError( + 'setNativeExecution() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("runtime.useNativeExecution", {value!r}) instead.' + ) + + def setNativeSQLPredicates(self, value): + raise NotImplementedError( + 'setNativeSQLPredicates() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("runtime.useNativeSQLPredicates", {value!r}) instead.' + ) + + def setNumberOfOutputPartitions(self, value): + raise NotImplementedError( + 'setNumberOfOutputPartitions() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("output.numberOfOutputPartitions", {value!r}) instead.' + ) + + def setOptimizeGeneralComparisonToValueComparison(self, value): + raise NotImplementedError( + 'setOptimizeGeneralComparisonToValueComparison() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("optimization.optimizeGeneralComparisonToValueComparison", {value!r}) instead.' + ) + + def setOptimizeParentPointers(self, value): + raise NotImplementedError( + 'setOptimizeParentPointers() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("optimization.optimizeParentPointers", {value!r}) instead.' + ) + + def setOptimizeSteps(self, value): + raise NotImplementedError( + 'setOptimizeSteps() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("optimization.optimizeSteps", {value!r}) instead.' + ) + + def setOptimizeStepsExperimental(self, value): + raise NotImplementedError( + 'setOptimizeStepsExperimental() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("optimization.optimizeStepsExperimental", {value!r}) instead.' + ) + + def setOutputPath(self, value): + raise NotImplementedError( + 'setOutputPath() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("output.outputPath", {value!r}) instead.' + ) + + def setParallelExecution(self, value): + raise NotImplementedError( + 'setParallelExecution() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("runtime.useParallelExecution", {value!r}) instead.' + ) + + def setPrintIteratorTree(self, value): + raise NotImplementedError( + 'setPrintIteratorTree() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("debug.printIteratorTree", {value!r}) instead.' + ) + + def setQueryLanguage(self, value): + raise NotImplementedError( + 'setQueryLanguage() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("semantics.queryLanguage", {value!r}) instead.' + ) + + def setQueryPath(self, value): + raise NotImplementedError( + 'setQueryPath() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("input.queryPath", {value!r}) instead.' + ) + + def setResultSizeCap(self, value): + raise NotImplementedError( + 'setResultSizeCap() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("runtime.resultsSizeCap", {value!r}) instead.' + ) + + def setShellFilter(self, value): + raise NotImplementedError( + 'setShellFilter() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("output.shellFilter", {value!r}) instead.' + ) + + def setShowErrorInfo(self, value): + raise NotImplementedError( + 'setShowErrorInfo() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("debug.showErrorInfo", {value!r}) instead.' + ) + + def setStaticBaseUri(self, value): + raise NotImplementedError( + 'setStaticBaseUri() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("semantics.staticBaseUri", {value!r}) instead.' + ) + + def setXmlVersion(self, value): + raise NotImplementedError( + 'setXmlVersion() was removed in RumbleDB 3.0.0. ' + f'Use rumble.getRumbleConf().set("semantics.xmlVersion", {value!r}) instead.' + ) + + def toString(self): + raise NotImplementedError( + 'toString() was removed in RumbleDB 3.0.0. ' + 'No direct replacement for the old configuration summary. Use rumble.getRumbleConf().get(path) to inspect a section, for example rumble.getRumbleConf().get("runtime").' + ) + + def write(self, kryo, output): + raise NotImplementedError( + 'write() was removed in RumbleDB 3.0.0. ' + 'No direct replacement: Kryo read/write hooks are not exposed by the public configuration API. Read settings with rumble.getRumbleConf().get(path) and serialize those values in your application.' + ) + def __getattr__(self, name): return getattr(self._session._jrumblesession.getConfiguration(), name) From d5eba4472642f3cd4e6e2c7ebdd53fbd08370eee Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 28 Sep 2026 15:37:21 +0200 Subject: [PATCH 7/9] Revert edit. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c287508..113c066 100644 --- a/README.md +++ b/README.md @@ -475,7 +475,7 @@ The result size cap controls `first()` and notebook display. `json()` retrieves - Now also accepts pandas version 2.2. ## Version 0.2.0 alpha 2 -- Added access to the Rumble configuration to change the result size cap. For the current API and cap semantics, see the Version 3.0.0 notes above. +- You can change the result size cap through to the now accessible Rumble configuration (for example rumble .getRumbleConf().setResultSizeCap(10)). This controls how many items can be retrieved at most with a json() call. You can increase it to whichever number you would like if you reach the cap. - Add the JSONiq magic to execute JSONiq queries directly in a notebook cell, using the RumbleDB instance shipped with the library. - RumbleSession.builder.getOrCreate() now correctly reuses an existing session instead of creating a new object. It preserves the configuration. From 9fdd30a6fe66573a54e25d49c46f304121520c8a Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Mon, 28 Sep 2026 15:43:34 +0200 Subject: [PATCH 8/9] Add back condition. --- src/jsoniq/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jsoniq/session.py b/src/jsoniq/session.py index 282436e..1fa52b4 100644 --- a/src/jsoniq/session.py +++ b/src/jsoniq/session.py @@ -233,7 +233,7 @@ def bind(self, name: str, valueToBind): name = name[1:] if isinstance(valueToBind, SequenceOfItems): outputs = valueToBind.availableOutputs() - if "DataFrame" in outputs: + if isinstance(outputs, (list, JavaList)) and "DataFrame" in outputs: self._bindings[name] = ("bindDataFrame", valueToBind.df()._jdf) else: self._bindings[name] = ("bindItems", valueToBind.items()) From 41a7c72f350b4f4a4c172e0c9ad5fe7635387719 Mon Sep 17 00:00:00 2001 From: Ghislain Fourny Date: Wed, 30 Sep 2026 16:13:17 +0200 Subject: [PATCH 9/9] Add -s parameter to notebooks to use serialization for the output. --- README.md | 2 +- src/jsoniqmagic/magic.py | 26 ++++++++++++++++++++++++++ tests/test_rumbledb_3_api.py | 3 +++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 113c066..8e212a3 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ The resulting sequence of items can be retrieved as a list of JSON values, as a It is also possible to write the sequence of items to the local disk, to HDFS, to S3, etc in a way similar to how DataFrames are written back by Pyspark. -The library also contains a jsoniq magic that allows you to directly write JSONiq queries in a Jupyter notebook and see the results automatically output on the screen. +The library also contains a jsoniq magic that allows you to directly write JSONiq queries in a Jupyter notebook and see the results automatically output on the screen. In notebooks, you can use `%%jsoniq -s` (or `--serialize`) to serialize the result sequence to text output, according to the XSLT and XQuery Serialization 3.1 specification by W3C. The method and serialization options can all be specified in the query with option declarations, following the XQuery/JSONiq standard. `%%jsoniq -j` shows the results in JSON lines format, while `%%jsoniq -pdf` shows a pandas data frame, and `%%jsoniq -df` shows a Spark data frame. The design goal is that it is possible to chain DataFrames between JSONiq and Spark SQL queries seamlessly. For example, JSONiq can be used to clean up very messy data and turn it into a clean DataFrame, which can then be processed with Spark SQL, spark.ml, etc. diff --git a/src/jsoniqmagic/magic.py b/src/jsoniqmagic/magic.py index 3da7d52..9685d27 100644 --- a/src/jsoniqmagic/magic.py +++ b/src/jsoniqmagic/magic.py @@ -21,6 +21,9 @@ class JSONiqMagic(Magics): @argument( '-j', '--json', action='store_true', help='Prints the output as JSON.' ) + @argument( + '-s', '--serialize', action='store_true', help='Prints the serialized result sequence without additional quoting.' + ) @argument( '-u', '--apply-updates', action='store_true', help='Applies updates if a PUL is output.' ) @@ -49,6 +52,29 @@ def run(self, line, cell=None, timed=False): print("Usual reasons: firewall, misconfigured proxy.") return + if args.serialize: + try: + serialized = response.serialize() + except Py4JJavaError as e: + if rumble.getRumbleConf().getBoolean("debug.showErrorInfo"): + raise + print(e.java_exception.getMessage()) + return + except Exception as e: + print("Query unsuccessful.") + print("Usual reasons: firewall, misconfigured proxy.") + print("Error message:") + print(e.args[0]) + return + except: + print("Query unsuccessful.") + print("Usual reasons: firewall, misconfigured proxy.") + return + print(serialized) + if args.timed: + print("Response time: %s ms" % (time.time() - start)) + return + schema_str = """ No DataFrame available as no schema was detected. If you still believe the output is structured enough, you could add a schema and validate expression explicitly to your query. diff --git a/tests/test_rumbledb_3_api.py b/tests/test_rumbledb_3_api.py index bcdce17..d14edb9 100644 --- a/tests/test_rumbledb_3_api.py +++ b/tests/test_rumbledb_3_api.py @@ -120,5 +120,8 @@ def test_notebook_extension_and_display(rumble, capsys): output = capsys.readouterr().out assert "Displaying the first 2 items" in output assert output.endswith("1\n2\n") + serialized = rumble.jsoniq("1 to 3").serialize() + JSONiqMagic().jsoniq("-s", "1 to 3") + assert capsys.readouterr().out == serialized + "\n" finally: rumble.getRumbleConf().set("runtime.resultsSizeCap", 10)