From 8789d18acec946667fe7119f28e012c5c220a5ba Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Mon, 26 Oct 2026 01:11:42 +0000 Subject: [PATCH 1/6] fix(mongodb): store SET values as sent in the numeric ADD pipeline update An unconditional UpdateItem on a MongoDB table with no index and no stream is built as a native update while the server records that the table has no index. With a numeric ADD it is an aggregation pipeline, and each SET value went into its `$set` stage as it was. In a pipeline that value is an expression: - A string that starts with `$` is a field path or a variable, so `SET #s = :dollar ADD #n :one` with `:dollar` = `{"S": "$item_data"}` stored the item_data document in `s.S`; the update and every later read of the item answered InternalServerError, and so did every Query of its partition and every Scan of its table. `$$NOW` stored a date and `$$REMOVE` left `s` empty; `$`, a member named `$k` and a member whose name holds `.` made MongoDB refuse the update. - A document value merges into the embedded document it replaces, so a SET of a map over a map kept the old members, and a SET that changed the attribute's type stored two type tags and answered InternalServerError. Each SET value in the pipeline stage is now wrapped in `$literal`, which stores it as a constant. Amazon DynamoDB stores every SET value as sent and replaces the attribute it overwrites, with or without the ADD (measured 2026-10-25 at 23:15 UTC and 2026-10-26 at 00:55 UTC, each case twice). --- crates/storage-mongodb/src/data_engine.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index bb75560d3..38a0230ff 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -1705,10 +1705,13 @@ impl MongoEngine { // Each numeric ADD is ` = toString(toDecimal(field // or 0) + delta)`; `$unset` is expressed as ` = // "$$REMOVE"`; SET actions are literal assignments. `_v` - // is bumped in the same stage. + // is bumped in the same stage. `$literal` makes each SET value a + // constant: a `$` string or member name is not read as an + // expression, and a document value replaces the attribute instead + // of merging into the embedded document it replaces. let mut stage: Document = Document::new(); for (k, v) in &set_doc { - stage.insert(k, v.clone()); + stage.insert(k, doc! { "$literal": v.clone() }); } for k in unset_doc.keys() { stage.insert(k, "$$REMOVE"); From f1bf895b390ce21293b567091bffeffca0f9db36 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Mon, 26 Oct 2026 01:11:42 +0000 Subject: [PATCH 2/6] test(update): SET values that hold `$` strings, with and without a numeric ADD Dual-target wire tests from the 2026-10-25 23:15 UTC and 2026-10-26 00:55 UTC measurements: strings `$item_data`, `$$ROOT`, `$` and `$$REMOVE`, a string set and a list holding them, maps whose values hold them, maps with members named `$k`, `$literal` and names holding `.`, and the other scalar types, each stored as sent; a SET over an attribute the item already holds, including a type change; and a REMOVE in the same update as the ADD. Each test uses a table of its own and writes an item before its update, so on MongoDB the updates take the native update path. --- tests/test_update_item_dollar_values.py | 148 ++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 tests/test_update_item_dollar_values.py diff --git a/tests/test_update_item_dollar_values.py b/tests/test_update_item_dollar_values.py new file mode 100644 index 000000000..5732f83ce --- /dev/null +++ b/tests/test_update_item_dollar_values.py @@ -0,0 +1,148 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""UpdateItem SET of values that hold `$` strings or `$` member names. + +Amazon DynamoDB stores every SET value as sent, with or without a numeric ADD +in the same update: strings such as ``$item_data``, ``$$ROOT``, ``$`` and +``$$REMOVE``, string sets and lists holding them, maps whose values hold +them, maps with members named ``$k`` or ``$literal``, and maps with members +whose names hold ``.`` (measured with the ADD on 2026-10-25 at 23:15 UTC and without +it on 2026-10-26 at 00:55 UTC, us-east-1, each case twice). A SET over an +attribute the item already holds replaces it, and a REMOVE in the same update +as the ADD removes its attribute (2026-10-26 at 00:55 UTC). + +On MongoDB an unconditional update on a table with no index and no stream is +built as a native update while the server records that the table has no +index (60 seconds after a write). With a numeric ADD it is an aggregation +pipeline, where a `$` string reads as a field path. Each test gets a table of +its own and writes an item before its update, so the update runs on that +path. + +Dual-target against Amazon DynamoDB and extenddb. +""" + +from __future__ import annotations + +import pytest + +from conftest import scoped_table + +CASES = { + "s_dollar_field": {"S": "$item_data"}, + "s_dollar_dollar_root": {"S": "$$ROOT"}, + "s_dollar_only": {"S": "$"}, + "s_dollar_remove": {"S": "$$REMOVE"}, + "s_literal_text": {"S": '{"$literal": 1}'}, + "ss_dollar": {"SS": ["$a", "$$ROOT", "b"]}, + "l_dollar": {"L": [{"S": "$a"}, {"N": "1"}, {"M": {"k": {"S": "$b"}}}]}, + "m_dollar_value": {"M": {"k": {"S": "$item_data.pk"}}}, + "m_dollar_member_name": {"M": {"$k": {"S": "v"}}}, + "m_member_named_literal": {"M": {"$literal": {"S": "v"}}}, + "m_dotted_member_names": { + "M": {"a.b": {"S": "v"}, "$c.d": {"S": "$e"}, "f": {"M": {"g.h": {"N": "2"}}}} + }, + "n_value": {"N": "5"}, + "bool_value": {"BOOL": True}, + "null_value": {"NULL": True}, + "b_value": {"B": b"$x"}, +} + + +@pytest.fixture +def table(dynamodb_client): + with scoped_table(dynamodb_client) as name: + yield name + + +def _normalize(value): + """String sets come back in no fixed order.""" + if "SS" in value: + return {"SS": sorted(value["SS"])} + return value + + +@pytest.mark.parametrize("with_add", [True, False], ids=["set_and_add", "set_alone"]) +@pytest.mark.parametrize("case", sorted(CASES)) +def test_set_stores_the_value_as_sent(dynamodb_client, table, case, with_add): + value = CASES[case] + key = {"pk": {"S": case}} + dynamodb_client.put_item(TableName=table, Item={**key, "v": {"S": "x"}}) + names = {"#s": "s"} + values = {":s": value} + expression = "SET #s = :s" + if with_add: + names["#n"] = "n" + values[":one"] = {"N": "1"} + expression += " ADD #n :one" + dynamodb_client.update_item( + TableName=table, + Key=key, + UpdateExpression=expression, + ExpressionAttributeNames=names, + ExpressionAttributeValues=values, + ) + item = dynamodb_client.get_item(TableName=table, Key=key, ConsistentRead=True)["Item"] + want = {**key, "v": {"S": "x"}, "s": value} + if with_add: + want["n"] = {"N": "1"} + assert {k: _normalize(v) for k, v in item.items()} == { + k: _normalize(v) for k, v in want.items() + } + + +# A SET over an attribute the item already holds replaces it, with or without a +# numeric ADD in the same update (measured on 2026-10-26 at 00:55 UTC). In a +# MongoDB pipeline a document value merges into the embedded document it +# replaces unless it is a constant. +OVER = { + "over_s_to_n": ({"S": "x"}, {"N": "5"}), + "over_m_to_m": ({"M": {"a": {"S": "1"}}}, {"M": {"b": {"S": "2"}}}), + "over_m_nested": ( + {"M": {"a": {"M": {"x": {"S": "1"}}}}}, + {"M": {"a": {"M": {"y": {"S": "2"}}}}}, + ), + "over_l_to_m": ({"L": [{"S": "1"}]}, {"M": {"b": {"S": "2"}}}), +} + + +@pytest.mark.parametrize("with_add", [True, False], ids=["set_and_add", "set_alone"]) +@pytest.mark.parametrize("case", sorted(OVER)) +def test_set_replaces_the_value_the_item_holds(dynamodb_client, table, case, with_add): + old, new = OVER[case] + key = {"pk": {"S": case}} + dynamodb_client.put_item(TableName=table, Item={**key, "s": old}) + names = {"#s": "s"} + values = {":s": new} + expression = "SET #s = :s" + if with_add: + names["#n"] = "n" + values[":one"] = {"N": "1"} + expression += " ADD #n :one" + dynamodb_client.update_item( + TableName=table, + Key=key, + UpdateExpression=expression, + ExpressionAttributeNames=names, + ExpressionAttributeValues=values, + ) + item = dynamodb_client.get_item(TableName=table, Key=key, ConsistentRead=True)["Item"] + want = {**key, "s": new} + if with_add: + want["n"] = {"N": "1"} + assert item == want + + +def test_remove_in_the_same_update_as_add_removes_the_attribute(dynamodb_client, table): + """Measured on 2026-10-26 at 00:55 UTC: r is removed, s and n are written.""" + key = {"pk": {"S": "remove_with_add"}} + dynamodb_client.put_item(TableName=table, Item={**key, "r": {"S": "gone"}}) + dynamodb_client.update_item( + TableName=table, + Key=key, + UpdateExpression="SET #s = :s REMOVE #r ADD #n :one", + ExpressionAttributeNames={"#s": "s", "#r": "r", "#n": "n"}, + ExpressionAttributeValues={":s": {"S": "$$REMOVE"}, ":one": {"N": "1"}}, + ) + item = dynamodb_client.get_item(TableName=table, Key=key, ConsistentRead=True)["Item"] + assert item == {**key, "s": {"S": "$$REMOVE"}, "n": {"N": "1"}} From dcaf5dc5edf636396334124f3226cc66be32f4f8 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:17:30 +0000 Subject: [PATCH 3/6] test(update): SET a map whose only member name holds a dot m_dotted_member_names holds the dotted member names `a.b` and `g.h` together with `$c.d` and `$e`, so its value contains a `$`. The new case sends `{"M": {"a.b": {"S": "v"}}}`, a dotted member name in a value with no `$`. In the numeric ADD pipeline stage MongoDB refuses that map unless it is inside `$literal` (error 40183, "cannot use dotted field name 'a.b' in a sub object"). The case pins that every SET value is wrapped, including a value that holds no `$`: with the fix reverted, or with only the values that hold a `$` wrapped, its update with the ADD answers InternalServerError. --- tests/test_update_item_dollar_values.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_update_item_dollar_values.py b/tests/test_update_item_dollar_values.py index 5732f83ce..093601578 100644 --- a/tests/test_update_item_dollar_values.py +++ b/tests/test_update_item_dollar_values.py @@ -42,6 +42,11 @@ "m_dotted_member_names": { "M": {"a.b": {"S": "v"}, "$c.d": {"S": "$e"}, "f": {"M": {"g.h": {"N": "2"}}}} }, + # Unlike m_dotted_member_names, this map holds no `$`. In the pipeline stage + # MongoDB 7.0.45 refuses it as it is (error 40183, "cannot use dotted field + # name 'a.b' in a sub object") and stores it as sent inside `$literal` + # (measured on 2026-10-27 with the stage this server builds). + "m_dotted_member_name_only": {"M": {"a.b": {"S": "v"}}}, "n_value": {"N": "5"}, "bool_value": {"BOOL": True}, "null_value": {"NULL": True}, From bc76c68360bd4f460e383a127f09f6f4f20fbd06 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:17:30 +0000 Subject: [PATCH 4/6] test(mongodb): assert the ADD pipeline stage wraps SET values try_build_native_update reads no field of the engine, so it is now an associated function and a unit test can call it without a MongoDB connection. The new test parses `SET #s = :dollar ADD #n :one` with `:dollar` = `{"S": "$item_data"}`, builds the update, and checks that the pipeline's `$set` stage holds `{"$literal": {"S": "$item_data"}}` under `item_data.s`. The wire tests in tests/test_update_item_dollar_values.py check the stored value, and the session path stores it as sent as well, so they would pass if this update stopped taking the pipeline. The unit test fails when the stage holds the bare value. --- crates/storage-mongodb/src/data_engine.rs | 27 +++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index 38a0230ff..5f10c450a 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -805,7 +805,7 @@ impl MongoEngine { && !return_old && stream.is_none() && self.gsi_cache_get_fresh(&key_info.table_id) == Some(false) - && let Some(mongo_update) = self.try_build_native_update(actions, maps) + && let Some(mongo_update) = Self::try_build_native_update(actions, maps) { let (fast_filter, took_fast) = match mongo_update { NativeUpdate::Doc(d) => { @@ -1630,7 +1630,6 @@ impl MongoEngine { /// because mongo serializes doc-scoped write locks around the /// pipeline's read + compute + write, no OCC retry needed. fn try_build_native_update( - &self, actions: &[UpdateAction], maps: &ExpressionMaps, ) -> Option { @@ -4431,4 +4430,28 @@ transaction_read_concern = "majority""#, assert_eq!(native_attribute_name(&path, &maps), Some("name".to_owned())); } + + #[test] + fn native_update_pipeline_sets_each_value_as_a_literal() { + let names = std::collections::HashMap::from([ + ("s".to_owned(), "s".to_owned()), + ("n".to_owned(), "n".to_owned()), + ]); + let values = std::collections::HashMap::from([ + ("dollar".to_owned(), AttributeValue::S("$item_data".into())), + ("one".to_owned(), AttributeValue::N("1".into())), + ]); + let maps = ExpressionMaps::new(names, values); + let tokens = expression::tokenize("SET #s = :dollar ADD #n :one").unwrap(); + let actions = expression::parse_update(&tokens).unwrap(); + + let Some(NativeUpdate::Pipeline { pipeline, .. }) = + MongoEngine::try_build_native_update(&actions, &maps) + else { + panic!("a SET with a numeric ADD builds a pipeline update"); + }; + let stage = pipeline[0].get_document("$set").unwrap(); + let want = doc! { "$literal": { "S": "$item_data" } }; + assert_eq!(stage.get("item_data.s"), Some(&bson::Bson::Document(want))); + } } From 0cefc279288d9c6cb04c2284ec83f22414c699e5 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:17:30 +0000 Subject: [PATCH 5/6] test(update): state when the update takes the native pipeline The module docstring said an unconditional update on a table with no index and no stream is built as a native update while the server records that the table has no index (60 seconds after a write). The update also stays on the session-scoped path when it asks for old values (ReturnValues ALL_OLD or UPDATED_OLD) or for consumed capacity: data_engine.rs requires `!return_old`, and crates/engine/src/update_item.rs sets it for both. The first write on the session-scoped path that finds no record, or one older than 60 seconds, reads the index catalog and records the answer; no write renews the record during its 60 seconds. The docstring now states both conditions and says which write in each test records that the table has no index. --- tests/test_update_item_dollar_values.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/test_update_item_dollar_values.py b/tests/test_update_item_dollar_values.py index 093601578..2592f4a46 100644 --- a/tests/test_update_item_dollar_values.py +++ b/tests/test_update_item_dollar_values.py @@ -12,12 +12,15 @@ attribute the item already holds replaces it, and a REMOVE in the same update as the ADD removes its attribute (2026-10-26 at 00:55 UTC). -On MongoDB an unconditional update on a table with no index and no stream is -built as a native update while the server records that the table has no -index (60 seconds after a write). With a numeric ADD it is an aggregation -pipeline, where a `$` string reads as a field path. Each test gets a table of -its own and writes an item before its update, so the update runs on that -path. +On MongoDB an unconditional update that asks for no old values and no +consumed capacity, on a table with no index and no stream, is built as a +native update while the server records that the table has no index. A write +on the session-scoped read-modify-write path renews that record and it lasts +60 seconds from the last such write; a native update does not renew it. With +a numeric ADD the native update is an aggregation pipeline, where a `$` +string reads as a field path. Each test gets a table of its own and writes +an item before its update. That write takes the session-scoped path and +records that the table has no index, so the update runs as the pipeline. Dual-target against Amazon DynamoDB and extenddb. """ From b6cdfce24b47f88814c44a02f924769748e00903 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:17:30 +0000 Subject: [PATCH 6/6] test(mongodb): pin the wrapper on a plain string SET value The stage test asserted only the value whose text reads as a field path. A builder that wrapped only such values passed every test while a plain string SET over a non-string attribute with the ADD answered InternalServerError on every later read. The test now carries a plain string beside it and asserts both wrappers. The wire file's docstring says when the native path is the pipeline and how long the no-index record lasts. --- crates/storage-mongodb/src/data_engine.rs | 12 +++++++++++- tests/test_update_item_dollar_values.py | 16 +++++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index 5f10c450a..dd8f4cc9f 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -4435,14 +4435,16 @@ transaction_read_concern = "majority""#, fn native_update_pipeline_sets_each_value_as_a_literal() { let names = std::collections::HashMap::from([ ("s".to_owned(), "s".to_owned()), + ("p".to_owned(), "p".to_owned()), ("n".to_owned(), "n".to_owned()), ]); let values = std::collections::HashMap::from([ ("dollar".to_owned(), AttributeValue::S("$item_data".into())), + ("plain".to_owned(), AttributeValue::S("x".into())), ("one".to_owned(), AttributeValue::N("1".into())), ]); let maps = ExpressionMaps::new(names, values); - let tokens = expression::tokenize("SET #s = :dollar ADD #n :one").unwrap(); + let tokens = expression::tokenize("SET #s = :dollar, #p = :plain ADD #n :one").unwrap(); let actions = expression::parse_update(&tokens).unwrap(); let Some(NativeUpdate::Pipeline { pipeline, .. }) = @@ -4451,7 +4453,15 @@ transaction_read_concern = "majority""#, panic!("a SET with a numeric ADD builds a pipeline update"); }; let stage = pipeline[0].get_document("$set").unwrap(); + // Every SET value is wrapped, not only the ones whose text reads as a + // field path: a plain string under a `$literal` stays a string, and + // without the wrapper a later read of a non-string attribute fails. let want = doc! { "$literal": { "S": "$item_data" } }; assert_eq!(stage.get("item_data.s"), Some(&bson::Bson::Document(want))); + let want_plain = doc! { "$literal": { "S": "x" } }; + assert_eq!( + stage.get("item_data.p"), + Some(&bson::Bson::Document(want_plain)) + ); } } diff --git a/tests/test_update_item_dollar_values.py b/tests/test_update_item_dollar_values.py index 2592f4a46..06174a959 100644 --- a/tests/test_update_item_dollar_values.py +++ b/tests/test_update_item_dollar_values.py @@ -14,13 +14,15 @@ On MongoDB an unconditional update that asks for no old values and no consumed capacity, on a table with no index and no stream, is built as a -native update while the server records that the table has no index. A write -on the session-scoped read-modify-write path renews that record and it lasts -60 seconds from the last such write; a native update does not renew it. With -a numeric ADD the native update is an aggregation pipeline, where a `$` -string reads as a field path. Each test gets a table of its own and writes -an item before its update. That write takes the session-scoped path and -records that the table has no index, so the update runs as the pipeline. +native update while the server records that the table has no index. The +first write on the session-scoped read-modify-write path that finds no such +record, or one older than 60 seconds, reads the index catalog and records the +answer; no write renews the record during its 60 seconds. With a numeric ADD +the native update is an aggregation pipeline, where a `$` string reads as a +field path. Each test gets a table of its own and writes an item before its +update. That write takes the session-scoped path and records that the table +has no index, so the update runs as a native update, which is the pipeline +when it has the ADD. Dual-target against Amazon DynamoDB and extenddb. """