From 1fbc67192a7c15b1618041cb907eeda181a40684 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Mon, 26 Oct 2026 00:59:18 +0000 Subject: [PATCH 1/4] fix(update): refuse an update whose result nests beyond 32 levels UpdateItem and the TransactWriteItems Update checked the nesting depth of ExpressionAttributeValues but not of the item the update produced, so a SET of a 2-level map at a 30-level path stored an attribute nested past the limit, on every backend. Amazon DynamoDB refuses an update whose result nests an attribute more than 32 levels (31 M or L wrappers around a scalar) and writes nothing: UpdateItem answers ValidationException `Nesting Levels have exceeded supported limits`, and a TransactWriteItems Update is cancelled with a ValidationError reason carrying that text (measured 2026-10-25 at 19:53 UTC and 2026-10-26 at 00:55 UTC, each case twice). A failing condition is reported first. apply_update_validated, which every backend's update and transactional update path calls after its condition check, now checks the top-level attributes the SET actions wrote. REMOVE, ADD and DELETE cannot deepen a value, and other attributes are not checked, so an item stored deeper than the limit before this change stays updatable through them. --- crates/core/src/expression/mod.rs | 1 + .../core/src/expression/update_evaluator.rs | 1 + crates/core/src/expression/update_nesting.rs | 128 ++++++++++++++++++ 3 files changed, 130 insertions(+) create mode 100644 crates/core/src/expression/update_nesting.rs diff --git a/crates/core/src/expression/mod.rs b/crates/core/src/expression/mod.rs index db2613a7e..e6d61d4cd 100755 --- a/crates/core/src/expression/mod.rs +++ b/crates/core/src/expression/mod.rs @@ -20,6 +20,7 @@ mod resolver; mod search_condition; mod tokenizer; mod update_evaluator; +mod update_nesting; mod update_parser; pub use ast::{ArithOp, CompareOp, Expr, PathElement, UpdateAction}; diff --git a/crates/core/src/expression/update_evaluator.rs b/crates/core/src/expression/update_evaluator.rs index 52f2a149b..ec278d8c4 100755 --- a/crates/core/src/expression/update_evaluator.rs +++ b/crates/core/src/expression/update_evaluator.rs @@ -113,6 +113,7 @@ pub fn apply_update_validated( } } apply_update(actions, item, maps)?; + super::update_nesting::validate_written_nesting_depth(actions, item, maps)?; crate::validation::validate_vector_write_changed( item, &before, diff --git a/crates/core/src/expression/update_nesting.rs b/crates/core/src/expression/update_nesting.rs new file mode 100644 index 000000000..8ec7bc417 --- /dev/null +++ b/crates/core/src/expression/update_nesting.rs @@ -0,0 +1,128 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Nesting depth of the attributes an update expression writes. + +use std::collections::BTreeMap; + +use crate::error::DynamoDbError; +use crate::types::AttributeValue; + +use super::ast::UpdateAction; +use super::resolver::{ExpressionMaps, resolve_element_name}; + +/// Most M or L wrappers an attribute value may have: 31 wrappers around a +/// scalar is 32 levels, the service's limit. +const MAX_WRAPPERS: usize = 31; + +/// Refuse an updated item whose SET targets now nest beyond the limit. +/// +/// Only the top-level attributes a SET writes are checked. REMOVE, ADD and +/// DELETE cannot deepen a value, and an attribute the update does not touch +/// is left alone, so an item stored deeper than the limit before this check +/// existed stays updatable through its other attributes. +/// +/// # Errors +/// +/// `ValidationException` `Nesting Levels have exceeded supported limits`, the +/// service's text for UpdateItem and for a TransactWriteItems Update +/// cancellation reason (measured 2026-10-25). +pub(super) fn validate_written_nesting_depth( + actions: &[UpdateAction], + item: &BTreeMap, + maps: &ExpressionMaps, +) -> Result<(), DynamoDbError> { + for action in actions { + let UpdateAction::Set { path, .. } = action else { + continue; + }; + let Some(first) = path.first() else { + continue; + }; + let name = resolve_element_name(first, maps)?; + if item + .get(name.as_ref()) + .is_some_and(|v| wrappers_exceed(v, 0)) + { + return Err(DynamoDbError::ValidationException( + "Nesting Levels have exceeded supported limits".to_owned(), + )); + } + } + Ok(()) +} + +/// Whether `value`, sitting under `above` wrappers, has more than +/// [`MAX_WRAPPERS`] wrappers on any path down to a scalar. +fn wrappers_exceed(value: &AttributeValue, above: usize) -> bool { + let children: Box> = match value { + AttributeValue::M(map) => Box::new(map.values()), + AttributeValue::L(list) => Box::new(list.iter()), + _ => return false, + }; + let here = above + 1; + here > MAX_WRAPPERS || children.into_iter().any(|v| wrappers_exceed(v, here)) +} + +#[cfg(test)] +mod tests { + use std::collections::{BTreeMap, HashMap}; + + use super::*; + use crate::expression::{parse_update, tokenize}; + + fn nmap(wrappers: usize) -> AttributeValue { + let mut v = AttributeValue::S("leaf".to_owned()); + for _ in 0..wrappers { + v = AttributeValue::M(BTreeMap::from([("k".to_owned(), v)])); + } + v + } + + fn check(expr: &str, item: &BTreeMap) -> Result<(), DynamoDbError> { + let actions = parse_update(&tokenize(expr).unwrap()).unwrap(); + // The names map is keyed without the leading `#`. + let maps = ExpressionMaps::new( + HashMap::from([("m".to_owned(), "m".to_owned())]), + HashMap::new(), + ); + validate_written_nesting_depth(&actions, item, &maps) + } + + fn is_nesting_error(result: Result<(), DynamoDbError>) -> bool { + matches!( + result, + Err(DynamoDbError::ValidationException(m)) + if m == "Nesting Levels have exceeded supported limits" + ) + } + + #[test] + fn thirty_one_wrappers_pass_and_thirty_two_are_refused() { + let at = BTreeMap::from([("m".to_owned(), nmap(31))]); + assert!(check("SET #m = :v", &at).is_ok()); + let over = BTreeMap::from([("m".to_owned(), nmap(32))]); + assert!(is_nesting_error(check("SET #m.k = :v", &over))); + } + + #[test] + fn a_list_counts_as_a_wrapper_and_every_branch_is_checked() { + let mut deep = AttributeValue::S("leaf".to_owned()); + for _ in 0..32 { + deep = AttributeValue::L(vec![AttributeValue::N("1".to_owned()), deep]); + } + let item = BTreeMap::from([("m".to_owned(), deep)]); + assert!(is_nesting_error(check("SET #m[0] = :v", &item))); + } + + #[test] + fn attributes_the_update_does_not_set_are_not_checked() { + let item = BTreeMap::from([ + ("m".to_owned(), nmap(40)), + ("x".to_owned(), AttributeValue::S("y".to_owned())), + ]); + assert!(check("SET x = :v", &item).is_ok()); + assert!(check("REMOVE #m.k", &item).is_ok()); + assert!(is_nesting_error(check("SET #m.k = :v", &item))); + } +} From 8fd65a044d0a3e7ea271cd43f23834f8f5abb7a2 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Mon, 26 Oct 2026 00:59:18 +0000 Subject: [PATCH 2/4] test(update): nesting depth of the item an update produces Dual-target wire tests from the 2026-10-25 and 2026-10-26 measurements: SET into a deep map or list, at the top level, through list_append and through if_not_exists, up to and past the limit; a failing condition next to a result that is too deep; an update of another attribute on an item at the limit; and the TransactWriteItems Update cancellation reason. --- tests/test_update_result_nesting_depth.py | 185 ++++++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 tests/test_update_result_nesting_depth.py diff --git a/tests/test_update_result_nesting_depth.py b/tests/test_update_result_nesting_depth.py new file mode 100644 index 000000000..095d9381d --- /dev/null +++ b/tests/test_update_result_nesting_depth.py @@ -0,0 +1,185 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Nesting depth of the item an UpdateItem or a TransactWriteItems Update produces. + +Amazon DynamoDB refuses an update whose result nests an attribute value more +than 32 levels (31 M or L wrappers around a scalar) and writes nothing: +UpdateItem answers ValidationException ``Nesting Levels have exceeded +supported limits``, and a TransactWriteItems Update is cancelled with a +``ValidationError`` reason carrying the same text. A failing condition is +reported first. Measured on 2026-10-25 at 19:53 UTC and 2026-10-26 at 00:55 +UTC (us-east-1), each case twice. + +Depth below is counted in wrappers: a value with w wrappers has w + 1 levels. + +Dual-target against Amazon DynamoDB and extenddb. +""" + +from __future__ import annotations + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +MESSAGE = "Nesting Levels have exceeded supported limits" +NAMES = {"#m": "m", "#k": "k"} + + +def nmap(wrappers: int, leaf: dict | None = None) -> dict: + value = leaf or {"S": "leaf"} + for _ in range(wrappers): + value = {"M": {"k": value}} + return value + + +def nlist(wrappers: int) -> dict: + value = {"S": "leaf"} + for _ in range(wrappers): + value = {"L": [value]} + return value + + +def wrappers(value: dict) -> int: + depth = 0 + while True: + if value.get("M"): + value = next(iter(value["M"].values())) + elif value.get("L"): + value = value["L"][0] + else: + return depth + depth += 1 + + +def path(j: int) -> str: + return "#m" + ".#k" * j + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + yield name + + +def _put(client, table, sk, m): + key = {"pk": {"S": "nest"}, "sk": {"S": sk}} + item = {**key, "x": {"S": "x"}} + if m is not None: + item["m"] = m + client.put_item(TableName=table, Item=item) + return key + + +def _stored_wrappers(client, table, key): + item = client.get_item(TableName=table, Key=key, ConsistentRead=True)["Item"] + return wrappers(item["m"]) if "m" in item else None + + +# (case, base value of m, update expression, names, values, wrappers stored after) +ACCEPTED = [ + ("j29_w2", nmap(30), f"SET {path(29)} = :v", NAMES, {":v": nmap(2)}, 31), + ("j30_w1", nmap(30), f"SET {path(30)} = :v", NAMES, {":v": nmap(1)}, 31), + ("j1_w30", nmap(30), f"SET {path(1)} = :v", NAMES, {":v": nmap(30)}, 31), + ("j15_w16", nmap(30), f"SET {path(15)} = :v", NAMES, {":v": nmap(16)}, 31), + ("top_w31", None, "SET #m = :v", {"#m": "m"}, {":v": nmap(31)}, 31), + ("new_j1_w30", {"M": {}}, f"SET {path(1)} = :v", NAMES, {":v": nmap(30)}, 31), + ("list_j30_w1", nlist(30), "SET #m" + "[0]" * 30 + " = :v", {"#m": "m"}, {":v": nlist(1)}, 31), + ("if_not_exists_kept", nmap(30), f"SET {path(30)} = if_not_exists({path(30)}, :v)", NAMES, + {":v": nmap(2)}, 30), + ("at_limit_scalar", nmap(31), f"SET {path(31)} = :v", NAMES, {":v": {"S": "y"}}, 31), +] + +REFUSED = [ + ("j29_w3", nmap(30), f"SET {path(29)} = :v", NAMES, {":v": nmap(3)}, 30), + ("j30_w2", nmap(30), f"SET {path(30)} = :v", NAMES, {":v": nmap(2)}, 30), + ("j1_w31", nmap(30), f"SET {path(1)} = :v", NAMES, {":v": nmap(31)}, 30), + ("j15_w17", nmap(30), f"SET {path(15)} = :v", NAMES, {":v": nmap(17)}, 30), + ("new_j1_w31", {"M": {}}, f"SET {path(1)} = :v", NAMES, {":v": nmap(31)}, 0), + ("list_j30_w2", nlist(30), "SET #m" + "[0]" * 30 + " = :v", {"#m": "m"}, {":v": nlist(2)}, 30), + ("list_append", nmap(29, {"L": [{"S": "a"}]}), f"SET {path(29)} = list_append({path(29)}, :v)", NAMES, + {":v": {"L": [nmap(2)]}}, 30), + ("if_not_exists_default", nmap(29, {"M": {}}), f"SET {path(30)} = if_not_exists(#m.#z, :v)", + {**NAMES, "#z": "z"}, {":v": nmap(2)}, 29), +] + + +@pytest.mark.parametrize(("case", "base", "expr", "names", "values", "after"), ACCEPTED, + ids=[c[0] for c in ACCEPTED]) +def test_update_up_to_the_limit_is_applied(dynamodb_client, table, case, base, expr, names, values, after): + key = _put(dynamodb_client, table, f"ok-{case}", base) + dynamodb_client.update_item( + TableName=table, Key=key, UpdateExpression=expr, + ExpressionAttributeNames=names, ExpressionAttributeValues=values, + ) + assert _stored_wrappers(dynamodb_client, table, key) == after + + +@pytest.mark.parametrize(("case", "base", "expr", "names", "values", "after"), REFUSED, + ids=[c[0] for c in REFUSED]) +def test_update_past_the_limit_is_refused(dynamodb_client, table, case, base, expr, names, values, after): + key = _put(dynamodb_client, table, f"no-{case}", base) + with pytest.raises(ClientError) as exc_info: + dynamodb_client.update_item( + TableName=table, Key=key, UpdateExpression=expr, + ExpressionAttributeNames=names, ExpressionAttributeValues=values, + ) + err = exc_info.value.response["Error"] + assert (err["Code"], err["Message"]) == ("ValidationException", MESSAGE) + assert _stored_wrappers(dynamodb_client, table, key) == after + + +def test_a_failing_condition_is_reported_before_the_depth(dynamodb_client, table): + key = _put(dynamodb_client, table, "condfail", nmap(30)) + with pytest.raises(ClientError) as exc_info: + dynamodb_client.update_item( + TableName=table, Key=key, UpdateExpression=f"SET {path(30)} = :v", + ConditionExpression="#x = :no", + ExpressionAttributeNames={**NAMES, "#x": "x"}, + ExpressionAttributeValues={":v": nmap(2), ":no": {"S": "no"}}, + ) + assert exc_info.value.response["Error"]["Code"] == "ConditionalCheckFailedException" + assert _stored_wrappers(dynamodb_client, table, key) == 30 + + +def test_an_item_at_the_limit_takes_an_update_of_another_attribute(dynamodb_client, table): + key = _put(dynamodb_client, table, "other-attr", nmap(31)) + dynamodb_client.update_item( + TableName=table, Key=key, UpdateExpression="SET #x = :v", + ExpressionAttributeNames={"#x": "x"}, ExpressionAttributeValues={":v": {"S": "y"}}, + ) + assert _stored_wrappers(dynamodb_client, table, key) == 31 + + +def test_transact_update_past_the_limit_is_cancelled(dynamodb_client, table): + key = _put(dynamodb_client, table, "tx-no", nmap(30)) + with pytest.raises(ClientError) as exc_info: + dynamodb_client.transact_write_items(TransactItems=[{"Update": { + "TableName": table, "Key": key, "UpdateExpression": f"SET {path(30)} = :v", + "ExpressionAttributeNames": NAMES, "ExpressionAttributeValues": {":v": nmap(2)}, + }}]) + resp = exc_info.value.response + assert resp["Error"]["Code"] == "TransactionCanceledException" + reasons = [(r.get("Code"), r.get("Message")) for r in resp["CancellationReasons"]] + assert reasons == [("ValidationError", MESSAGE)] + assert _stored_wrappers(dynamodb_client, table, key) == 30 + + +def test_transact_update_up_to_the_limit_is_applied(dynamodb_client, table): + key = _put(dynamodb_client, table, "tx-ok", nmap(30)) + dynamodb_client.transact_write_items(TransactItems=[{"Update": { + "TableName": table, "Key": key, "UpdateExpression": f"SET {path(29)} = :v", + "ExpressionAttributeNames": NAMES, "ExpressionAttributeValues": {":v": nmap(2)}, + }}]) + assert _stored_wrappers(dynamodb_client, table, key) == 31 From 564e3149f769fd1305081519aa8a180ac521ceb9 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Mon, 26 Oct 2026 02:03:38 +0000 Subject: [PATCH 3/4] fix(update): check ADD targets for depth, walk each attribute once ExtendDB's ADD creates the missing maps on its path, so an ADD can nest an attribute past the limit (`ADD #m.#k...#k.#a.#b.#c :one` on a value 30 wrappers deep stored 32). The check now covers the top-level attribute of each ADD target as well as each SET target. Each top-level attribute is walked once however many actions write into it, and the walk matches maps and lists directly instead of boxing an iterator per node. With 200 SET actions into one list of 4,000 maps, UpdateItem took 78 ms against 36 ms on main before this; the walk now runs once. --- crates/core/src/expression/update_nesting.rs | 52 +++++++++++++++----- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/crates/core/src/expression/update_nesting.rs b/crates/core/src/expression/update_nesting.rs index 8ec7bc417..48e848379 100644 --- a/crates/core/src/expression/update_nesting.rs +++ b/crates/core/src/expression/update_nesting.rs @@ -15,12 +15,14 @@ use super::resolver::{ExpressionMaps, resolve_element_name}; /// scalar is 32 levels, the service's limit. const MAX_WRAPPERS: usize = 31; -/// Refuse an updated item whose SET targets now nest beyond the limit. +/// Refuse an updated item whose SET or ADD targets now nest beyond the limit. /// -/// Only the top-level attributes a SET writes are checked. REMOVE, ADD and -/// DELETE cannot deepen a value, and an attribute the update does not touch -/// is left alone, so an item stored deeper than the limit before this check -/// existed stays updatable through its other attributes. +/// Only the top-level attributes a SET or an ADD writes are checked, each +/// once however many actions write into it. ADD is checked because it +/// creates the missing maps on its path. REMOVE and DELETE cannot deepen a +/// value, and an attribute the update does not touch is left alone, so an +/// item stored deeper than the limit before this check existed stays +/// updatable through its other attributes. /// /// # Errors /// @@ -32,14 +34,20 @@ pub(super) fn validate_written_nesting_depth( item: &BTreeMap, maps: &ExpressionMaps, ) -> Result<(), DynamoDbError> { + let mut written = Vec::new(); for action in actions { - let UpdateAction::Set { path, .. } = action else { + let (UpdateAction::Set { path, .. } | UpdateAction::Add { path, .. }) = action else { continue; }; let Some(first) = path.first() else { continue; }; let name = resolve_element_name(first, maps)?; + if !written.contains(&name) { + written.push(name); + } + } + for name in &written { if item .get(name.as_ref()) .is_some_and(|v| wrappers_exceed(v, 0)) @@ -55,13 +63,16 @@ pub(super) fn validate_written_nesting_depth( /// Whether `value`, sitting under `above` wrappers, has more than /// [`MAX_WRAPPERS`] wrappers on any path down to a scalar. fn wrappers_exceed(value: &AttributeValue, above: usize) -> bool { - let children: Box> = match value { - AttributeValue::M(map) => Box::new(map.values()), - AttributeValue::L(list) => Box::new(list.iter()), - _ => return false, - }; let here = above + 1; - here > MAX_WRAPPERS || children.into_iter().any(|v| wrappers_exceed(v, here)) + match value { + AttributeValue::M(map) => { + here > MAX_WRAPPERS || map.values().any(|v| wrappers_exceed(v, here)) + } + AttributeValue::L(list) => { + here > MAX_WRAPPERS || list.iter().any(|v| wrappers_exceed(v, here)) + } + _ => false, + } } #[cfg(test)] @@ -124,5 +135,22 @@ mod tests { assert!(check("SET x = :v", &item).is_ok()); assert!(check("REMOVE #m.k", &item).is_ok()); assert!(is_nesting_error(check("SET #m.k = :v", &item))); + assert!(check("DELETE #m.k :v", &item).is_ok()); + } + + #[test] + fn an_add_target_is_checked() { + let over = BTreeMap::from([("m".to_owned(), nmap(32))]); + assert!(is_nesting_error(check("ADD #m.k :v", &over))); + let at = BTreeMap::from([("m".to_owned(), nmap(31))]); + assert!(check("ADD #m.k :v", &at).is_ok()); + } + + #[test] + fn two_actions_into_one_attribute_are_checked_as_one() { + let over = BTreeMap::from([("m".to_owned(), nmap(32))]); + assert!(is_nesting_error(check("SET #m.k = :v, #m.j = :v", &over))); + let at = BTreeMap::from([("m".to_owned(), nmap(31))]); + assert!(check("SET #m.k = :v, #m.j = :v ADD #m.i :v", &at).is_ok()); } } From c05cbc66567523b84a59962b7ecbc703704227a3 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Mon, 26 Oct 2026 02:03:38 +0000 Subject: [PATCH 4/4] test(update): an ADD that would create maps past the nesting limit The service refuses `ADD path(29).#a.#b.#c :one` on a value 30 wrappers deep for its 33-element path (measured 2026-10-26 at 01:26 UTC, twice); ExtendDB refuses it for the depth of the maps the ADD would create. The test compares the error code and the stored item only. The stored-depth helper now takes the deepest branch, so the list_append case sees an appended element. --- tests/test_update_result_nesting_depth.py | 27 +++++++++++++++-------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/test_update_result_nesting_depth.py b/tests/test_update_result_nesting_depth.py index 095d9381d..b2cce5978 100644 --- a/tests/test_update_result_nesting_depth.py +++ b/tests/test_update_result_nesting_depth.py @@ -42,15 +42,9 @@ def nlist(wrappers: int) -> dict: def wrappers(value: dict) -> int: - depth = 0 - while True: - if value.get("M"): - value = next(iter(value["M"].values())) - elif value.get("L"): - value = value["L"][0] - else: - return depth - depth += 1 + """Most wrappers on any branch; an empty map or list adds none.""" + children = list(value.get("M", {}).values()) or value.get("L", []) + return 1 + max(wrappers(c) for c in children) if children else 0 def path(j: int) -> str: @@ -140,6 +134,21 @@ def test_update_past_the_limit_is_refused(dynamodb_client, table, case, base, ex assert _stored_wrappers(dynamodb_client, table, key) == after +def test_an_add_that_would_create_maps_past_the_limit_is_refused(dynamodb_client, table): + # The service refuses this path for its 33 elements (measured 2026-10-26 + # at 01:26 UTC) and ExtendDB for the depth of the maps ADD would create, + # so only the code and the stored item are compared. + key = _put(dynamodb_client, table, "add-deep", nmap(30)) + with pytest.raises(ClientError) as exc_info: + dynamodb_client.update_item( + TableName=table, Key=key, UpdateExpression=f"ADD {path(29)}.#a.#b.#c :one", + ExpressionAttributeNames={**NAMES, "#a": "a", "#b": "b", "#c": "c"}, + ExpressionAttributeValues={":one": {"N": "1"}}, + ) + assert exc_info.value.response["Error"]["Code"] == "ValidationException" + assert _stored_wrappers(dynamodb_client, table, key) == 30 + + def test_a_failing_condition_is_reported_before_the_depth(dynamodb_client, table): key = _put(dynamodb_client, table, "condfail", nmap(30)) with pytest.raises(ClientError) as exc_info: