From 827edb979b492f83e1a168973862844d9f0a83d8 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Mon, 19 Oct 2026 14:41:53 +0000 Subject: [PATCH] fix(limits): size items and updates the way the service does Five behaviours at the 409,600-byte item limit, each measured against Amazon DynamoDB (us-east-1, 2026-10-19) by an accept/refuse pair either side of the limit, and each wrong on main. Number byte cost. A number costs 1 + ceil(integer significant digits / 2) + ceil(fraction significant digits / 2), plus one for a sign. The two halves are rounded up separately. main counted all significant digits together, so every number with a fraction was one byte short: 1.5 costs 3 (main 2), 3.14159 costs 5 (main 4), 100.5 costs 4 (main 3). Sixteen measured literals are pinned in a unit test. Item limit on updates. The PostgreSQL and SQLite storage factories hardcoded 400_000 as the bound on the finished item of an UpdateItem or a transacted Update, while the engine applied the configured 409,600 to puts. The configured limit now reaches those two factories through ServerComponentsOptions, so on PostgreSQL and SQLite every write path enforces the one configured figure. The Cassandra and MongoDB factories do not read the option; Cassandra keeps its fixed 409,600 on the finished item and MongoDB applies no limit on updates, as before. UpdateItem is sized by the statement. The service sizes a standalone UpdateItem by the statement rather than the finished item: a base of 3; 18 per SET or ADD clause and nothing per REMOVE or DELETE; a skeleton item holding only the written paths (each map entry its name plus one, each map or list 3, each list element one more, untouched siblings left out), where a SET or ADD leaf costs the clause's value placeholders and a REMOVE or DELETE leaf costs nothing; one more for a clause through a list index; and, inside a SET operand, 26 plus the name length per function call, 34 per plus, 37 per minus, and for a document path operand 15 plus the name for its first element, 14 plus the name for a later attribute and 15 for an index, with the attribute's stored value not charged. The legacy AttributeUpdates form costs 3 plus the name and finished value of each attribute it names that remains. The key and untouched attributes are left out, so at a one-byte key a plain SET can produce a finished item of at most 409,581 bytes, and a long key or an untouched attribute buys the headroom back up to 409,600, which the finished item never exceeds. An attribute reached through an alias is charged as the attribute. A transacted Update is sized on the finished item alone: exactly 409,600 is stored and one byte over is a cancellation. Both update paths word the refusal "Item size to update has exceeded the maximum allowed size"; main used the put wording. validate_update_item_size and validate_transact_update_item_size in core carry the rules, called from the PostgreSQL and SQLite update paths; the engine marks a desugared AttributeUpdates request on the expression maps so storage sizes it as the legacy form. Empty binary index key on UpdateItem (PostgreSQL). UpdateItem setting a GSI key to an empty binary was written on PostgreSQL; SQLite and the service refuse it with "The update expression attempted to update a secondary index key to a value that is not supported. The AttributeValue for a key attribute cannot contain an empty binary value." #401 has since added those index key checks to the PostgreSQL standalone path, with the index attribute definitions from the write's own catalog read; this change keeps that version and adds no second copy. tests/test_item_size_at_limit.py carries 58 dual-target cases, 32 of them the largest accepted padding for one statement shape each (plain and multi-clause SET, same-map clauses, nested paths, list indexes with siblings, if_not_exists, list_append, nested functions, addition, subtraction, ADD, REMOVE, DELETE and six legacy AttributeUpdates forms). 58 of 58 pass against the service and against both backends with this change; 49 fail on main (PostgreSQL) and 48 (SQLite). The statement sizer also carries a direct unit test in core (validation::update_size_tests): eight cases pinning the byte figure for the common shapes, the over-limit refusal and its wording, the finished-item refusal, and the legacy form, so the arithmetic runs under cargo test without a server. docs/differences-from-dynamodb.md holds the sizing rules in one place (item bytes, number cost, the statement rule) and records the two regions (eu-south-2, me-central-1) that size an UpdateItem on the finished item instead; docs/dynamodb-limits.md states the limit and points there. The update-specific cases run on PostgreSQL and SQLite and against live DynamoDB; they are skipped on Cassandra and MongoDB, whose storage layers size updates differently as recorded in docs/differences-from-dynamodb.md. devtools/run-tests exports EXTENDDB_TEST_BACKEND so the file can gate those classes. Number sizing runs on every backend. --- crates/core/src/expression/resolver.rs | 5 + crates/core/src/types/item.rs | 114 ++++- crates/core/src/validation/mod.rs | 365 +++++++++++++++ crates/engine/src/update_item.rs | 5 +- crates/server/src/serve.rs | 1 + .../storage-postgres/src/data/transactions.rs | 10 +- .../storage-postgres/src/data/update_item.rs | 5 +- crates/storage-postgres/src/lib.rs | 4 +- .../storage-sqlite/src/data/transactions.rs | 8 +- crates/storage-sqlite/src/data/update_item.rs | 4 +- crates/storage-sqlite/src/lib.rs | 16 +- crates/storage/src/server_components.rs | 15 +- devtools/run-tests | 7 +- docs/differences-from-dynamodb.md | 31 ++ docs/dynamodb-limits.md | 4 +- tests/test_item_size_at_limit.py | 431 ++++++++++++++++++ 16 files changed, 973 insertions(+), 52 deletions(-) create mode 100644 tests/test_item_size_at_limit.py diff --git a/crates/core/src/expression/resolver.rs b/crates/core/src/expression/resolver.rs index fa14081ec..699cd4fdf 100755 --- a/crates/core/src/expression/resolver.rs +++ b/crates/core/src/expression/resolver.rs @@ -26,6 +26,10 @@ use super::ast::{CompareOp, Expr, PathElement}; pub struct ExpressionMaps { pub names: HashMap, pub values: HashMap, + /// Set by the engine when the update arrived as legacy `AttributeUpdates` + /// and was desugared into an update expression. The service sizes that + /// form against the item limit differently from an expression. + pub legacy_attribute_updates: bool, /// Pre-parsed `BigDecimal` values for numeric placeholders. /// Populated eagerly by `new()` and refreshed by `pre_parse_numerics()`. parsed_numerics: HashMap, @@ -40,6 +44,7 @@ impl ExpressionMaps { let mut maps = Self { names, values, + legacy_attribute_updates: false, parsed_numerics: HashMap::new(), }; maps.pre_parse_numerics(); diff --git a/crates/core/src/types/item.rs b/crates/core/src/types/item.rs index 0d31edb72..e363cca46 100755 --- a/crates/core/src/types/item.rs +++ b/crates/core/src/types/item.rs @@ -388,35 +388,75 @@ pub fn attribute_value_size(value: &AttributeValue) -> usize { /// Calculate the `DynamoDB` size of a number in bytes. /// -/// `DynamoDB` number sizing: approximately 1 byte per 2 significant digits + 1 byte. -/// Zero is 1 byte. Negative numbers add 1 byte. Max 21 bytes. +/// Measured against the service (us-east-1, 2026-10-19) by accept/refuse pairs +/// either side of the 409,600-byte item limit: +/// +/// ```text +/// 1 + ceil(integer significant digits / 2) + ceil(fraction significant digits / 2) +/// + 1 if negative +/// ``` +/// +/// The two halves are rounded up separately, so digits that straddle the +/// decimal point cost one byte more than the same digits without a point: +/// `1.5` is 3 bytes where `15` is 2, and `3.14159` is 5 where `123456` is 4. +/// Leading zeros of the integer half and trailing zeros of the fraction half +/// are not significant; trailing zeros of the integer half are significant +/// when a fraction follows them (`100` is 2, `100.5` is 4) and a zero between +/// two significant digits always is (`1010` is 3). The exponent is not +/// counted (`1E125` is 2). Zero is 1 byte however it is spelled, and is never +/// negative (`-0` is 1). fn dynamodb_number_size(n: &str) -> usize { - let s = n.trim_start_matches('-'); - let is_zero = s.chars().all(|c| c == '0' || c == '.'); - if is_zero { + let (negative, body) = match n.strip_prefix('-') { + Some(rest) => (true, rest), + None => (false, n.strip_prefix('+').unwrap_or(n)), + }; + let (mantissa, exponent) = match body.find(['e', 'E']) { + Some(pos) => (&body[..pos], body[pos + 1..].parse::().unwrap_or(0)), + None => (body, 0), + }; + let (int_text, frac_text) = match mantissa.find('.') { + Some(pos) => (&mantissa[..pos], &mantissa[pos + 1..]), + None => (mantissa, ""), + }; + let digits: String = int_text.chars().chain(frac_text.chars()).collect(); + if !digits.bytes().any(|b| (b'1'..=b'9').contains(&b)) { return 1; } - - let significant = if let Some(dot_pos) = s.find('.') { - let (int_part, frac_part) = s.split_at(dot_pos); - let frac = &frac_part[1..]; - let int_trimmed = int_part.trim_start_matches('0'); - let frac_trimmed = frac.trim_end_matches('0'); - if int_trimmed.is_empty() { - let frac_sig = frac.trim_start_matches('0'); - frac_sig.trim_end_matches('0').len() - } else { - format!("{int_trimmed}{frac_trimmed}").len() - } + // Where the decimal point sits in `digits` once the exponent is applied. + let point = i64::try_from(int_text.len()) + .unwrap_or(i64::MAX) + .saturating_add(exponent); + let len = i64::try_from(digits.len()).unwrap_or(i64::MAX); + let (int_part, frac_part): (String, String) = if point <= 0 { + ( + String::new(), + "0".repeat(usize::try_from(-point).unwrap_or(0)) + &digits, + ) + } else if point >= len { + ( + digits.clone() + &"0".repeat(usize::try_from(point - len).unwrap_or(0)), + String::new(), + ) } else { - let trimmed = s.trim_start_matches('0').trim_end_matches('0'); - if trimmed.is_empty() { 1 } else { trimmed.len() } + let split = usize::try_from(point).unwrap_or(0); + (digits[..split].to_owned(), digits[split..].to_owned()) }; - - let mut size = significant.div_ceil(2) + 1; - if n.starts_with('-') { - size += 1; - } + let int_part = int_part.trim_start_matches('0'); + let frac_part = frac_part.trim_end_matches('0'); + // Trailing zeros of the integer half fall away only when no fraction follows. + let int_significant = if frac_part.is_empty() { + int_part.trim_end_matches('0').len() + } else { + int_part.len() + }; + // Leading zeros of the fraction half fall away only when no integer precedes. + let frac_significant = if int_part.is_empty() { + frac_part.trim_start_matches('0').len() + } else { + frac_part.len() + }; + let size = + 1 + int_significant.div_ceil(2) + frac_significant.div_ceil(2) + usize::from(negative); size.min(21) } @@ -428,6 +468,32 @@ mod tests { AttributeValue::S(v.to_owned()) } + #[test] + fn number_size_matches_the_service_for_measured_literals() { + // Each figure measured against the service (us-east-1, 2026-10-19) by an + // accept/refuse pair either side of the 409,600-byte item limit. + for (literal, bytes) in [ + ("0", 1), + ("-0", 1), + ("1", 2), + ("15", 2), + ("100", 2), + ("1E125", 2), + ("0.15", 2), + ("1.5", 3), + ("1.2", 3), + ("1.200", 3), + ("1010", 3), + ("-42", 3), + ("1.234", 4), + ("100.5", 4), + ("123456", 4), + ("3.14159", 5), + ] { + assert_eq!(dynamodb_number_size(literal), bytes, "literal {literal}"); + } + } + #[test] fn scalar_sizes_unchanged() { assert_eq!(attribute_value_size(&s("hello")), 5); diff --git a/crates/core/src/validation/mod.rs b/crates/core/src/validation/mod.rs index 3decb4666..b34d8248d 100755 --- a/crates/core/src/validation/mod.rs +++ b/crates/core/src/validation/mod.rs @@ -1876,6 +1876,257 @@ pub fn validate_item_size(item: &Item, max_bytes: usize) -> Result<(), DynamoDbE Ok(()) } +/// The wording the service uses when an update, standalone or transacted, would +/// exceed the item size limit. +pub const ITEM_SIZE_TO_UPDATE_EXCEEDED: &str = + "Item size to update has exceeded the maximum allowed size"; + +/// Fixed cost the service charges an `UpdateItem` statement, before its clauses. +pub const UPDATE_STATEMENT_BASE_COST: usize = 3; +/// Fixed cost per `SET` or `ADD` clause, on top of the written path and operands. +pub const UPDATE_SET_CLAUSE_COST: usize = 18; +/// Fixed cost of a function call in a `SET` operand, plus the function name's length. +pub const UPDATE_FUNCTION_COST: usize = 26; +/// Fixed cost of a `+` in a `SET` operand (26 plus the length of "addition"). +pub const UPDATE_ADDITION_COST: usize = 34; +/// Fixed cost of a `-` in a `SET` operand (26 plus the length of "subtraction"). +pub const UPDATE_SUBTRACTION_COST: usize = 37; +/// Cost of the first element of a document path used as a function or arithmetic +/// operand, plus the attribute name's length. The attribute's stored value is not +/// charged. +pub const UPDATE_PATH_OPERAND_FIRST_COST: usize = 15; +/// Cost of a later attribute element of such a path, plus the name's length. +pub const UPDATE_PATH_OPERAND_NESTED_COST: usize = 14; +/// Cost of a list index element of such a path. +pub const UPDATE_PATH_OPERAND_INDEX_COST: usize = 15; +/// Extra cost of a clause whose target path ends in a list index. +pub const UPDATE_LIST_INDEX_TARGET_COST: usize = 1; + +/// Size a standalone `UpdateItem` the way the service does. +/// +/// Measured against the service (us-east-1, 2026-10-19) by accept/refuse pairs +/// either side of the item limit, for whole-attribute and document-path +/// targets, aliases, one to three clauses on the same or different attributes, +/// `if_not_exists`, `list_append`, nested functions, arithmetic, `ADD`, +/// `REMOVE`, `DELETE`, and the legacy `AttributeUpdates` form. The figure the +/// limit applies to is: +/// +/// * a base of 3; +/// * 18 per `SET` or `ADD` clause, nothing per `REMOVE` or `DELETE`; +/// * the size of a skeleton item holding only the written paths: each map +/// entry its name plus one, each map or list 3, each list element one more, +/// siblings the statement does not touch left out. A `SET` or `ADD` leaf is +/// charged the sizes of the clause's value placeholders; a `REMOVE` or +/// `DELETE` leaf is charged nothing, and the `DELETE` operand is not charged; +/// * one more for a clause whose target path ends in a list index; +/// * for operands inside a `SET` value: 26 plus the name length per function +/// call, 34 per `+` and 37 per `-`, and for a document path 15 plus the +/// name for its first element, 14 plus the name for a later attribute, 15 +/// for an index, with the attribute's stored value not charged. +/// +/// The legacy `AttributeUpdates` form is charged 3 plus, for each attribute +/// it names that is present afterwards, the name and the finished value's +/// bytes; an attribute it removes costs nothing. +/// +/// The key and untouched attributes are left out, so at a short key the largest +/// finished item an update can produce sits below the limit, and a long key or +/// an untouched attribute buys the headroom back up to the limit itself, which +/// the finished item never exceeds. +/// +/// `finished` is the whole finished item; `actions` are the statement's clauses +/// with `#name` placeholders still in place, resolved here through `maps`. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` with the update wording when +/// either the statement figure or the finished item exceeds `max_bytes`. +pub fn validate_update_item_size( + finished: &Item, + actions: &[crate::expression::UpdateAction], + maps: &crate::expression::ExpressionMaps, + max_bytes: usize, +) -> Result<(), DynamoDbError> { + let statement = if maps.legacy_attribute_updates { + legacy_update_statement_size(finished, actions, maps)? + } else { + update_statement_size(actions, maps)? + }; + if statement > max_bytes || item_size_bytes(finished) > max_bytes { + return Err(DynamoDbError::ValidationException( + ITEM_SIZE_TO_UPDATE_EXCEEDED.to_owned(), + )); + } + Ok(()) +} + +/// The bytes the service charges the operand expression of a `SET` clause: +/// value placeholders at their size, function calls, arithmetic operators and +/// document paths at their fixed costs. +fn update_operand_size( + expr: &crate::expression::Expr, + maps: &crate::expression::ExpressionMaps, +) -> Result { + use crate::expression::{Expr, PathElement, resolve_element_name}; + Ok(match expr { + Expr::Placeholder(name) => maps + .values + .get(name) + .map_or(0, crate::types::attribute_value_size), + Expr::Path(elements) => { + let mut bytes = 0; + for (i, element) in elements.iter().enumerate() { + bytes += match element { + PathElement::Index(_) => UPDATE_PATH_OPERAND_INDEX_COST, + PathElement::Attribute(_) => { + let name = resolve_element_name(element, maps)?; + name.len() + + if i == 0 { + UPDATE_PATH_OPERAND_FIRST_COST + } else { + UPDATE_PATH_OPERAND_NESTED_COST + } + } + }; + } + bytes + } + Expr::Function { name, args } => { + let mut bytes = UPDATE_FUNCTION_COST + name.len(); + for arg in args { + bytes += update_operand_size(arg, maps)?; + } + bytes + } + Expr::Arithmetic { left, op, right } => { + let operator = match op { + crate::expression::ArithOp::Add => UPDATE_ADDITION_COST, + crate::expression::ArithOp::Sub => UPDATE_SUBTRACTION_COST, + }; + operator + update_operand_size(left, maps)? + update_operand_size(right, maps)? + } + _ => 0, + }) +} + +/// A written path as a tree: map entries by name, list elements by index. +#[derive(Default)] +struct WrittenNode { + leaf_bytes: usize, + entries: std::collections::BTreeMap, + elements: std::collections::BTreeMap, +} + +fn written_node_size(node: &WrittenNode) -> usize { + let maps: usize = if node.entries.is_empty() { + 0 + } else { + 3 + node + .entries + .iter() + .map(|(name, child)| name.len() + written_node_size(child) + 1) + .sum::() + }; + let lists: usize = if node.elements.is_empty() { + 0 + } else { + 3 + node + .elements + .values() + .map(|child| written_node_size(child) + 1) + .sum::() + }; + node.leaf_bytes + maps + lists +} + +/// Build the skeleton the statement writes and size it with the clause costs. +fn update_statement_size( + actions: &[crate::expression::UpdateAction], + maps: &crate::expression::ExpressionMaps, +) -> Result { + use crate::expression::{PathElement, UpdateAction, resolve_element_name}; + let mut root = WrittenNode::default(); + let mut bytes = UPDATE_STATEMENT_BASE_COST; + for action in actions { + let (path, leaf) = match action { + UpdateAction::Set { path, value } | UpdateAction::Add { path, value } => { + bytes += UPDATE_SET_CLAUSE_COST; + (path, update_operand_size(value, maps)?) + } + UpdateAction::Remove { path } | UpdateAction::Delete { path, .. } => (path, 0), + }; + if matches!(path.last(), Some(PathElement::Index(_))) { + bytes += UPDATE_LIST_INDEX_TARGET_COST; + } + let mut node = &mut root; + for element in path { + node = match element { + PathElement::Index(i) => node.elements.entry(*i).or_default(), + PathElement::Attribute(_) => { + let name = resolve_element_name(element, maps)?; + node.entries.entry(name.into_owned()).or_default() + } + }; + } + node.leaf_bytes += leaf; + } + // The item itself is a map whose 3 bytes the base already covers. + Ok(bytes + written_node_size(&root).saturating_sub(3)) +} + +/// The legacy `AttributeUpdates` form: 3 plus, for each named attribute still +/// present in the finished item, its name and finished value bytes. Nothing per +/// action, no entry byte, nothing for an attribute the update removed. +fn legacy_update_statement_size( + finished: &Item, + actions: &[crate::expression::UpdateAction], + maps: &crate::expression::ExpressionMaps, +) -> Result { + use crate::expression::{UpdateAction, resolve_element_name}; + let mut names: Vec = Vec::new(); + for action in actions { + let path = match action { + UpdateAction::Set { path, .. } + | UpdateAction::Add { path, .. } + | UpdateAction::Remove { path } + | UpdateAction::Delete { path, .. } => path, + }; + if let Some(first) = path.first() { + let name = resolve_element_name(first, maps)?; + if !names.iter().any(|n| n == name.as_ref()) { + names.push(name.into_owned()); + } + } + } + Ok(UPDATE_STATEMENT_BASE_COST + + names + .iter() + .filter_map(|name| finished.get(name).map(|value| (name, value))) + .map(|(name, value)| name.len() + crate::types::attribute_value_size(value)) + .sum::()) +} + +/// Size a transacted `Update` the way the service does: the finished item +/// against the limit, with the update wording. Measured against the service +/// (us-east-1, 2026-10-19): a transacted Update accepts a finished item of +/// exactly the limit and cancels one byte over with this wording, so the +/// statement-sized rule of a standalone `UpdateItem` does not apply. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` with the update wording when the +/// finished item exceeds `max_bytes`. +pub fn validate_transact_update_item_size( + finished: &Item, + max_bytes: usize, +) -> Result<(), DynamoDbError> { + if item_size_bytes(finished) > max_bytes { + return Err(DynamoDbError::ValidationException( + ITEM_SIZE_TO_UPDATE_EXCEEDED.to_owned(), + )); + } + Ok(()) +} + /// Validate all number values in an item are within `DynamoDB` limits. pub fn validate_item_numbers(item: &Item) -> Result<(), DynamoDbError> { for value in item.values() { @@ -3801,3 +4052,117 @@ mod tests { assert!(validate_conditional_operator_usage(false, 1).is_ok()); } } + +#[cfg(test)] +mod update_size_tests { + //! Direct unit tests for the UpdateItem statement sizer. The byte figures + //! are the ones measured against the service in us-east-1 on 2026-10-19 and + //! exercised end to end by tests/test_item_size_at_limit.py; these assert + //! the same numbers at the function boundary so a developer can run them + //! with `cargo test` and no server. + + use std::collections::HashMap; + + use super::{ + ITEM_SIZE_TO_UPDATE_EXCEEDED, legacy_update_statement_size, update_statement_size, + validate_update_item_size, + }; + use crate::expression::{ExpressionMaps, UpdateAction, parse_update, tokenize}; + use crate::types::{AttributeValue, Item}; + + fn s(v: &str) -> AttributeValue { + AttributeValue::S(v.to_owned()) + } + + /// Parse an update expression into actions plus a maps table whose values + /// are keyed as the parser keys them (the `:name` placeholder without its + /// colon). + fn build(expr: &str, values: &[(&str, AttributeValue)]) -> (Vec, ExpressionMaps) { + let actions = parse_update(&tokenize(expr).unwrap()).unwrap(); + let mut map = HashMap::new(); + for (name, value) in values { + map.insert((*name).to_owned(), value.clone()); + } + (actions, ExpressionMaps::new(HashMap::new(), map)) + } + + fn statement(expr: &str, values: &[(&str, AttributeValue)]) -> usize { + let (actions, maps) = build(expr, values); + update_statement_size(&actions, &maps).unwrap() + } + + #[test] + fn single_set_is_base_clause_path_and_value() { + // 3 base + 18 clause + the written path `p` (name 1, two container + // bytes) + the 4-byte value. + assert_eq!(statement("SET p = :v", &[("v", s("wxyz"))]), 27); + } + + #[test] + fn two_set_clauses_each_cost_their_clause_and_path() { + assert_eq!( + statement("SET a = :v, b = :w", &[("v", s("wxyz")), ("w", s("y"))]), + 48, + ); + } + + #[test] + fn a_nested_path_sizes_each_level_once() { + assert_eq!(statement("SET m.a = :v", &[("v", s("wxyz"))]), 32); + } + + #[test] + fn a_list_index_target_adds_one_byte() { + // `p[1]` writes the attribute `p` (name 1, two container bytes) holding + // a one-element list (three container bytes, one per element) with the + // 4-byte value, plus 18 clause and 1 for writing through a list index: + // 3 + 18 + 1 + 10 = 32. One byte more than the same write to `p[0]`'s + // plain attribute would be without the list, which is the list-index + // target byte. + assert_eq!(statement("SET p[1] = :v", &[("v", s("wxyz"))]), 32); + } + + #[test] + fn remove_costs_its_path_skeleton_only() { + // No clause cost for REMOVE: 3 base + the path `p`. + assert_eq!(statement("REMOVE p", &[]), 5); + } + + #[test] + fn statement_over_the_limit_is_refused_with_the_update_wording() { + // SET p = :v with a ten-byte value is a 33-byte statement; the finished + // item ({p: 10 bytes}) is smaller, so the statement figure binds. + let (actions, maps) = build("SET p = :v", &[("v", s("0123456789"))]); + let mut finished = Item::new(); + finished.insert("p".to_owned(), s("0123456789")); + + assert!(validate_update_item_size(&finished, &actions, &maps, 33).is_ok()); + let err = validate_update_item_size(&finished, &actions, &maps, 32).unwrap_err(); + assert_eq!(err.to_string(), ITEM_SIZE_TO_UPDATE_EXCEEDED); + } + + #[test] + fn an_oversized_finished_item_is_refused_even_when_the_statement_fits() { + // A small statement (REMOVE q, 7 bytes) cannot rescue a finished item + // that is itself over the limit. + let (actions, maps) = build("REMOVE q", &[]); + let mut finished = Item::new(); + finished.insert("p".to_owned(), s("0123456789")); + assert!(validate_update_item_size(&finished, &actions, &maps, 5).is_err()); + } + + #[test] + fn legacy_attribute_updates_size_the_finished_value_not_the_statement() { + // The legacy form charges 3 plus the name and finished value of each + // attribute it names that remains in the item. + let (actions, mut maps) = build("SET p = :v", &[("v", s("wxyz"))]); + maps.legacy_attribute_updates = true; + let mut finished = Item::new(); + finished.insert("p".to_owned(), s("wxyz")); + // 3 base + name `p` (1) + value (4) = 8. + assert_eq!( + legacy_update_statement_size(&finished, &actions, &maps).unwrap(), + 8 + ); + } +} diff --git a/crates/engine/src/update_item.rs b/crates/engine/src/update_item.rs index 7f74c205c..fbd9326ec 100755 --- a/crates/engine/src/update_item.rs +++ b/crates/engine/src/update_item.rs @@ -137,7 +137,7 @@ pub async fn handle_update_item( input.expected.as_ref().map_or(0, HashMap::len), )?; - let (condition, maps) = resolve_condition( + let (condition, mut maps) = resolve_condition( input.condition_expression.as_deref(), effective_expr_names.as_ref(), effective_expr_values.as_ref(), @@ -145,6 +145,9 @@ pub async fn handle_update_item( input.conditional_operator, &ctx.limits, )?; + // The storage layer sizes a desugared legacy update against the item limit + // the way the service sizes `AttributeUpdates`, not an expression. + maps.legacy_attribute_updates = input.attribute_updates.is_some(); // No UpdateExpression and no AttributeUpdates: no-op upsert. // Some("") still errors via parse_update_expr. diff --git a/crates/server/src/serve.rs b/crates/server/src/serve.rs index 9ebbaccd4..a41daea21 100644 --- a/crates/server/src/serve.rs +++ b/crates/server/src/serve.rs @@ -289,6 +289,7 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { extenddb_storage::server_components::ServerComponentsOptions::default(); component_options.bootstrap_if_uninitialized = dev_mode; + component_options.max_item_size_bytes = app_config.limits.max_item_size_bytes; // Wire the node's own bind address as the instance ID so each ExtendDB node // has a distinct node ID (to support logical clocks for sequences, etc.). diff --git a/crates/storage-postgres/src/data/transactions.rs b/crates/storage-postgres/src/data/transactions.rs index e86c00a90..ce0360c25 100644 --- a/crates/storage-postgres/src/data/transactions.rs +++ b/crates/storage-postgres/src/data/transactions.rs @@ -662,10 +662,12 @@ async fn execute_transact_write_op( .map_err(|e| { TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) })?; - // Validate post-update item size - validation::validate_item_size(&item, max_item_size_bytes).map_err(|e| { - TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) - })?; + // A transacted Update is sized on the finished item, with the + // update wording in the cancellation reason. + validation::validate_transact_update_item_size(&item, max_item_size_bytes) + .map_err(|e| { + TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; // Secondary-index key validation on the post-update item: a type // mismatch is a cancellation reason; setting an index key to an // empty value is a top-level ValidationException. diff --git a/crates/storage-postgres/src/data/update_item.rs b/crates/storage-postgres/src/data/update_item.rs index 80ab9a515..56872833f 100755 --- a/crates/storage-postgres/src/data/update_item.rs +++ b/crates/storage-postgres/src/data/update_item.rs @@ -187,8 +187,9 @@ impl PostgresEngine { ) .map_err(|e| StorageError::Validation(e.to_string()))?; - // Validate post-update item size (400 KB limit) - validation::validate_item_size(&item, self.max_item_size_bytes) + // Size the update the way the service does: what the statement + // writes plus a fixed cost per clause, against the item limit. + validation::validate_update_item_size(&item, actions, maps, self.max_item_size_bytes) .map_err(|e| StorageError::Validation(e.to_string()))?; // Secondary-index key validation on the post-update item, as the diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index e9f1ae2e9..acb1aef56 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -745,7 +745,7 @@ fn server_components_factory( // PostgreSQL bootstrap needs operator input (databases, roles, admin // credentials), so `bootstrap_if_uninitialized` is not honored here: // an uninitialized catalog fails with the explicit-`init` guidance. - _options: extenddb_storage::server_components::ServerComponentsOptions, + options: extenddb_storage::server_components::ServerComponentsOptions, ) -> std::pin::Pin< Box> + Send>, > { @@ -758,7 +758,7 @@ fn server_components_factory( let pg_config = PostgresConfig { connection_string: connection_string.clone(), pool_size: max_connections, - max_item_size_bytes: 400_000, + max_item_size_bytes: options.max_item_size_bytes, }; // Create PostgresEngine diff --git a/crates/storage-sqlite/src/data/transactions.rs b/crates/storage-sqlite/src/data/transactions.rs index e9b5f8f5b..8be6a5683 100644 --- a/crates/storage-sqlite/src/data/transactions.rs +++ b/crates/storage-sqlite/src/data/transactions.rs @@ -417,9 +417,11 @@ async fn execute_transact_write_op( &key_info.attribute_definitions, ) .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; - validation::validate_item_size(&item, max_item_size_bytes).map_err(|e| { - TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) - })?; + // A transacted Update is sized on the finished item, with the update + // wording in the cancellation reason. + validation::validate_transact_update_item_size(&item, max_item_size_bytes).map_err( + |e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())), + )?; // Secondary-index key validation on the post-update item: a type // mismatch is a cancellation reason; setting an index key to an // empty value is a top-level ValidationException. diff --git a/crates/storage-sqlite/src/data/update_item.rs b/crates/storage-sqlite/src/data/update_item.rs index ae18ae1e9..58a1db2bb 100644 --- a/crates/storage-sqlite/src/data/update_item.rs +++ b/crates/storage-sqlite/src/data/update_item.rs @@ -72,7 +72,9 @@ impl SqliteEngine { &key_info.attribute_definitions, ) .map_err(|e| StorageError::Validation(e.to_string()))?; - validation::validate_item_size(&item, self.max_item_size_bytes) + // Size the update the way the service does: what the statement writes + // plus a fixed cost per clause, against the item limit. + validation::validate_update_item_size(&item, actions, maps, self.max_item_size_bytes) .map_err(|e| StorageError::Validation(e.to_string()))?; // Secondary-index key validation on the post-update item, matching the // TransactWriteItems update path: a wrong-typed index key attribute or diff --git a/crates/storage-sqlite/src/lib.rs b/crates/storage-sqlite/src/lib.rs index 3953fc350..75c91336f 100644 --- a/crates/storage-sqlite/src/lib.rs +++ b/crates/storage-sqlite/src/lib.rs @@ -193,9 +193,6 @@ use extenddb_storage::server_components::{BackendError, ServerComponents}; use crate::hooks::SqliteRuntimeHooks; -/// Maximum DynamoDB item size in bytes (post-update validation bound). -const MAX_ITEM_SIZE_BYTES: usize = 400_000; - fn sqlite_server_components_factory( config: &(dyn extenddb_storage::config::StorageConfig + 'static), region: &str, @@ -214,12 +211,13 @@ fn sqlite_server_components_factory( tracing::debug!("holding serve lock {}", lock.path().display()); std::sync::Arc::new(lock) }); - let mut engine = SqliteEngine::new(&db_path, pool_size, ®ion, MAX_ITEM_SIZE_BYTES) - .await - .map_err(|e| BackendError::ConnectionFailed { - backend: "sqlite".to_owned(), - details: e.to_string(), - })?; + let mut engine = + SqliteEngine::new(&db_path, pool_size, ®ion, options.max_item_size_bytes) + .await + .map_err(|e| BackendError::ConnectionFailed { + backend: "sqlite".to_owned(), + details: e.to_string(), + })?; // In-memory databases do not persist across the `init` process, so the // catalog must be bootstrapped here, at serve time, on the engine's own diff --git a/crates/storage/src/server_components.rs b/crates/storage/src/server_components.rs index 2ea59c9c9..a2c3cc288 100644 --- a/crates/storage/src/server_components.rs +++ b/crates/storage/src/server_components.rs @@ -86,7 +86,7 @@ impl std::fmt::Display for BackendError { impl std::error::Error for BackendError {} /// Options for server-component creation, beyond the storage config itself. -#[derive(Debug, Clone, Copy, Default)] +#[derive(Debug, Clone, Copy)] #[non_exhaustive] pub struct ServerComponentsOptions { /// Bootstrap the catalog at serve time when it is not initialized, @@ -95,6 +95,19 @@ pub struct ServerComponentsOptions { /// Backends whose bootstrap requires operator input ignore this and fail /// as usual; backends with a self-contained bootstrap (SQLite) honor it. pub bootstrap_if_uninitialized: bool, + /// The item size limit the storage layer applies to the finished item of + /// an update, standalone or transacted. Comes from `limits.max_item_size_bytes` + /// so the storage layer and the engine's put path enforce one figure. + pub max_item_size_bytes: usize, +} + +impl Default for ServerComponentsOptions { + fn default() -> Self { + Self { + bootstrap_if_uninitialized: false, + max_item_size_bytes: extenddb_core::limits::LimitsConfig::default().max_item_size_bytes, + } + } } /// Factory function type for creating server components. diff --git a/devtools/run-tests b/devtools/run-tests index 421a97b3a..ae7984896 100755 --- a/devtools/run-tests +++ b/devtools/run-tests @@ -141,9 +141,10 @@ case "$BACKEND" in exit 1 ;; esac -# Tests that depend on the backend (tests/test_ttl.py) read this rather than -# guess from a config file. Only exported when --backend was given: the -# SQLite CI leg runs without it. +# Tests that depend on the backend (tests/test_ttl.py, and the update-size +# classes in tests/test_item_size_at_limit.py that apply only to the PostgreSQL +# and SQLite storage layers) read this rather than guess from a config file. +# Only exported when --backend was given: the SQLite CI leg runs without it. if [[ -n "${BACKEND_GIVEN:-}" ]]; then export EXTENDDB_TEST_BACKEND="$BACKEND" fi diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index 2d6469b7e..c8d3d97ca 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -139,6 +139,37 @@ The same on all four backends. | `OnDemandThroughputOverride` | Applied on an on-demand restore, as on the service (measured on DynamoDB): equal to the backup's, changed, with one member, or on a `PROVISIONED` backup restored with `BillingModeOverride` `PAY_PER_REQUEST`. It replaces the backup's on-demand throughput as a whole: a member left out means no maximum, as on `CreateTable`, so `{}` restores a table with no maxima. On a `PROVISIONED` restore (from a `PROVISIONED` backup or by override) it is refused with `ValidationException`, as is a member other than `MaxReadRequestUnits` and `MaxWriteRequestUnits`. Measured on SQLite | | `VectorIndexOverride` | Refused with `ValidationException` | +## Item Size + +ExtendDB enforces the 400 KB (409,600-byte) item-size limit. This section is the +single reference for how an item and an update are sized; `docs/dynamodb-limits.md` +records the limit and points here. Every figure was measured against the service +in us-east-1 on 2026-10-19. + +How an item is sized: its attribute names plus its values. A string or binary +value costs its byte length. A number costs `1 + ceil(integer significant digits +/ 2) + ceil(fraction significant digits / 2)`, plus one for a sign, the two +halves rounded up separately, so `1.5` is 3 bytes and `3.14159` is 5. A list +costs 3 bytes plus one per element; a map costs 3 bytes plus, per entry, the +entry name, its value, and one more. + +How a standalone `UpdateItem` is sized: by the statement, not the finished item. +A base of 3; 18 per `SET` or `ADD` clause; a skeleton of the written paths (each +map level 3 bytes plus one per named entry, each list level 3 bytes plus one per +element, untouched siblings left out) whose `SET` or `ADD` leaves carry their +value-placeholder bytes; one more per clause written through a list index; 26 +plus the name length per function call; 34 per `+` and 37 per `-`; and 15 plus +the name per document path operand (14 for a nested attribute, 15 for an index). +`REMOVE` and `DELETE` cost their path skeleton only. The legacy `AttributeUpdates` +form costs 3 plus the name and finished value of each attribute it names that +remains. The key and untouched attributes are left out. Both the statement figure +and the finished item are checked against the limit, and a breach refuses with +`Item size to update has exceeded the maximum allowed size`. + +| Area | DynamoDB | ExtendDB | +|------|----------|------| +| `UpdateItem` item-size accounting | In 31 of 33 commercial regions a standalone `UpdateItem` is sized by the statement (the rule above). At a one-byte key the largest finished item it can produce is 409,581 bytes. Two regions (`eu-south-2`, `me-central-1`) size an `UpdateItem` on the finished item, like a `PutItem`. | PostgreSQL and SQLite follow the 31-region statement rule. A transacted `Update` is sized on the finished item on both, as on the service. Cassandra sizes every update on the finished item against a fixed 409,600 bytes; MongoDB applies no item-size limit on updates. | + ## Capacity and Throttling | Area | DynamoDB | ExtendDB | diff --git a/docs/dynamodb-limits.md b/docs/dynamodb-limits.md index 503836958..55d5de06b 100755 --- a/docs/dynamodb-limits.md +++ b/docs/dynamodb-limits.md @@ -1,6 +1,6 @@ # DynamoDB Limits Enforcement Status -Last updated: 2026-04-23 (P42) +Last updated: 2026-10-19 Source: [AWS DynamoDB Service Quotas](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ServiceQuotas.html) @@ -38,7 +38,7 @@ Source: [AWS DynamoDB Service Quotas](https://docs.aws.amazon.com/amazondynamodb | Limit | DynamoDB Value | Status | Notes | |-------|---------------|--------|-------------------------------------------------------------------------------| -| Maximum item size | 400 KB (409,600 bytes) | Enforced | `LimitsConfig::max_item_size_bytes`, validated on PutItem and post-UpdateItem | +| Maximum item size | 400 KB (409,600 bytes) | Enforced | `LimitsConfig::max_item_size_bytes`. The engine enforces it on PutItem, BatchWriteItem, the transaction Put path and ImportTable; the PostgreSQL and SQLite storage layers enforce it on UpdateItem and the transaction Update path. Cassandra applies a fixed 409,600 to the finished item of an update with the put wording; MongoDB applies no limit on updates. The sizing calculation (item bytes, number cost, and the statement-based UpdateItem rule) lives in [differences-from-dynamodb.md](differences-from-dynamodb.md#item-size). | | Partition key size | 1–2,048 bytes | Enforced | `validate_key_sizes`, `LimitsConfig::max_partition_key_size_bytes` | | Sort key size | 1–1,024 bytes | Enforced | `validate_key_sizes`, `LimitsConfig::max_sort_key_size_bytes` | | Attribute name size | 1–64 KB (65,535 bytes) | Partial | `validate_attribute_name_sizes`, `LimitsConfig::max_attribute_name_bytes`; top-level names and nested map keys at every depth, on PutItem, BatchWriteItem.PutRequest, TransactWriteItems.Put, and ImportTable. Not enforced on UpdateItem or TransactWriteItems.Update at any level. | diff --git a/tests/test_item_size_at_limit.py b/tests/test_item_size_at_limit.py new file mode 100644 index 000000000..ec71c2838 --- /dev/null +++ b/tests/test_item_size_at_limit.py @@ -0,0 +1,431 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Item size at the 409,600-byte limit: number byte costs, the UpdateItem rule, +the transacted Update rule, and the empty-binary index key refusal on UpdateItem. + +Every figure here was measured against Amazon DynamoDB (us-east-1, 2026-10-19) +by an accept/refuse pair either side of the limit: an item of exactly N bytes is +accepted and one of N+1 is refused, so the pair measures a single attribute's +cost to the byte with two requests. + +Numbers: ``1 + ceil(integer significant digits / 2) + ceil(fraction significant +digits / 2)``, plus one for a sign. The two halves round up separately, so +``1.5`` costs 3 where ``15`` costs 2 and ``3.14159`` costs 5 where ``123456`` +costs 4. + +UpdateItem is sized by the statement, not the finished item: a base of 3; 18 +per SET or ADD clause and nothing per REMOVE or DELETE; a skeleton item holding +only the written paths (each map entry its name plus one, each map or list 3, +each list element one more, untouched siblings left out), where a SET or ADD +leaf costs the clause's value placeholders and a REMOVE or DELETE leaf costs +nothing; one more for a clause through a list index; and inside a SET operand +26 plus the name length per function call, 34 per "+", 37 per "-", and for a +document path operand 15 plus the name for its first element, 14 plus the name +for a later attribute, 15 for an index, with the attribute's stored value not +charged. The legacy AttributeUpdates form costs 3 plus the name and finished +value of each attribute it names that is present afterwards. The key and +untouched attributes are left out, so at a one-byte key the largest finished +item a plain SET can produce is 409,581 bytes, and a longer key or an +untouched attribute buys the headroom back, capped at 409,600, which the +finished item never exceeds. The service applies this rule in 31 of 33 +commercial regions, us-east-1 among them. A transacted Update is sized on the +finished item alone: exactly 409,600 is stored, one byte over is a +cancellation. Both update paths word the refusal "Item size to update has +exceeded the maximum allowed size". + +Dual-target against Amazon DynamoDB and extenddb. +""" + +from __future__ import annotations + +import os + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +# The statement-sized UpdateItem rule and the empty-binary index-key refusal are +# enforced by the PostgreSQL and SQLite storage layers. The Cassandra layer +# sizes every update on the finished item against a fixed 409,600 bytes and the +# MongoDB layer applies no item-size limit on updates, both documented in +# docs/differences-from-dynamodb.md, so the update-specific classes below are +# skipped on those two backends. Number sizing lives in shared core and is +# asserted on every backend. When the backend is unset (a run against live +# DynamoDB through conf-run/live), every class runs. +on_default_backends_only = pytest.mark.skipif( + os.environ.get("EXTENDDB_TEST_BACKEND", "") in {"mongodb", "cassandra"}, + reason="UpdateItem is sized on the finished item (Cassandra) or unbounded (MongoDB); " + "see docs/differences-from-dynamodb.md", +) + +MAX = 409_600 +# A one-byte partition key "k": pk(2) + value(1) bytes the statement never writes. +KEY_K_BYTES = 3 +PUT_WORDING = "Item size has exceeded the maximum allowed size" +UPDATE_WORDING = "Item size to update has exceeded the maximum allowed size" +UPDATE_BASE = 3 +SET_COST = 19 # 18 for the clause plus the written attribute's map entry byte +REMOVE_COST = 2 # nothing for the clause plus the attribute's name and entry byte + + +@pytest.fixture(scope="class") +def table(dynamodb_client): + with scoped_table(dynamodb_client) as name: + yield name + + +@pytest.fixture(scope="class") +def gsi_table(dynamodb_client): + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "bidx", "AttributeType": "B"}, + ], + GlobalSecondaryIndexes=[ + { + "IndexName": "by-bidx", + "KeySchema": [{"AttributeName": "bidx", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "KEYS_ONLY"}, + } + ], + ) as name: + yield name + + +def _err(exc_info) -> dict: + return exc_info.value.response["Error"] + + +class TestNumberByteCost: + """A number's byte cost, measured by the accept/refuse pair at the limit.""" + + # (literal, bytes) measured on the service. + CASES = [ + ("0", 1), ("1", 2), ("15", 2), ("100", 2), ("1E125", 2), ("0.15", 2), + ("1.5", 3), ("1.2", 3), ("1.200", 3), ("1010", 3), ("-42", 3), + ("1.234", 4), ("100.5", 4), ("123456", 4), ("3.14159", 5), + ] + + @pytest.mark.parametrize(("literal", "cost"), CASES) + def test_number_costs_measured_bytes(self, dynamodb_client, table, literal, cost): + pk = "n-" + literal.replace(".", "_").replace("-", "m") + # pk(2) + pk value + n(1) + cost + p(1) + pad == MAX + pad = MAX - (2 + len(pk) + 1 + cost + 1) + dynamodb_client.put_item( + TableName=table, + Item={"pk": {"S": pk}, "n": {"N": literal}, "p": {"S": "x" * pad}}, + ) + with pytest.raises(ClientError) as exc_info: + dynamodb_client.put_item( + TableName=table, + Item={"pk": {"S": pk}, "n": {"N": literal}, "p": {"S": "x" * (pad + 1)}}, + ) + assert _err(exc_info)["Code"] == "ValidationException" + assert _err(exc_info)["Message"] == PUT_WORDING + + +@on_default_backends_only +class TestUpdateItemSizeRule: + """UpdateItem is sized by the statement, not the finished item.""" + + def _set(self, client, table, pk, finished_bytes, extra_expr="", extra_values=None): + pad = finished_bytes - (2 + len(pk) + 1) # pk name, pk value, "p" name + values = {":v": {"S": "x" * pad}} + values.update(extra_values or {}) + client.update_item( + TableName=table, + Key={"pk": {"S": pk}}, + UpdateExpression="SET p = :v" + extra_expr, + ExpressionAttributeValues=values, + ) + + def test_one_set_at_a_one_byte_key_caps_below_the_limit(self, dynamodb_client, table): + ceiling = MAX + KEY_K_BYTES - UPDATE_BASE - SET_COST # 409,581 + self._set(dynamodb_client, table, "k", ceiling) + dynamodb_client.delete_item(TableName=table, Key={"pk": {"S": "k"}}) + with pytest.raises(ClientError) as exc_info: + self._set(dynamodb_client, table, "k", ceiling + 1) + assert _err(exc_info)["Code"] == "ValidationException" + assert _err(exc_info)["Message"] == UPDATE_WORDING + + def test_a_second_set_costs_nineteen_bytes(self, dynamodb_client, table): + ceiling = MAX + KEY_K_BYTES - UPDATE_BASE - 2 * SET_COST # 409,562 + extra = {":w": {"S": "y"}} + # finished = pk(2)+1 + p(1)+pad + q(1)+1; the SET q costs 19, writes 2. + pad = ceiling - (2 + 1 + 1 + 2) + dynamodb_client.update_item( + TableName=table, Key={"pk": {"S": "k"}}, + UpdateExpression="SET p = :v, q = :w", + ExpressionAttributeValues={":v": {"S": "x" * pad}, **extra}, + ) + dynamodb_client.delete_item(TableName=table, Key={"pk": {"S": "k"}}) + with pytest.raises(ClientError) as exc_info: + dynamodb_client.update_item( + TableName=table, Key={"pk": {"S": "k"}}, + UpdateExpression="SET p = :v, q = :w", + ExpressionAttributeValues={":v": {"S": "x" * (pad + 1)}, **extra}, + ) + assert _err(exc_info)["Message"] == UPDATE_WORDING + + def test_a_remove_costs_two_bytes(self, dynamodb_client, table): + ceiling = MAX + KEY_K_BYTES - UPDATE_BASE - SET_COST - REMOVE_COST # 409,579 + pad = ceiling - (2 + 1 + 1) + for size, ok in ((pad, True), (pad + 1, False)): + dynamodb_client.put_item(TableName=table, Item={"pk": {"S": "k"}, "r": {"S": "z"}}) + call = lambda: dynamodb_client.update_item( # noqa: E731 + TableName=table, Key={"pk": {"S": "k"}}, + UpdateExpression="SET p = :v REMOVE r", + ExpressionAttributeValues={":v": {"S": "x" * size}}, + ) + if ok: + call() + else: + with pytest.raises(ClientError) as exc_info: + call() + assert _err(exc_info)["Message"] == UPDATE_WORDING + + def test_an_untouched_attribute_buys_headroom_back_to_the_limit(self, dynamodb_client, table): + # 100-byte attribute u is left alone; finished item may reach 409,600 exactly. + for finished, ok in ((MAX, True), (MAX + 1, False)): + dynamodb_client.put_item(TableName=table, Item={"pk": {"S": "k"}, "u": {"S": "u" * 100}}) + pad = finished - (2 + 1 + 1 + 101) + call = lambda: dynamodb_client.update_item( # noqa: E731 + TableName=table, Key={"pk": {"S": "k"}}, + UpdateExpression="SET p = :v", + ExpressionAttributeValues={":v": {"S": "x" * pad}}, + ) + if ok: + call() + else: + with pytest.raises(ClientError) as exc_info: + call() + assert _err(exc_info)["Message"] == UPDATE_WORDING + + def test_a_long_key_buys_headroom_back_to_the_limit(self, dynamodb_client, table): + pk = "K" * 100 + self._set(dynamodb_client, table, pk, MAX) + dynamodb_client.delete_item(TableName=table, Key={"pk": {"S": pk}}) + with pytest.raises(ClientError) as exc_info: + self._set(dynamodb_client, table, pk, MAX + 1) + assert _err(exc_info)["Message"] == UPDATE_WORDING + + def test_an_alias_is_charged_as_the_attribute_it_names(self, dynamodb_client, table): + # SET #twelvechars = :v writes attribute "p"; the alias costs nothing extra. + ceiling = MAX + KEY_K_BYTES - UPDATE_BASE - SET_COST # 409,581 + for finished, ok in ((ceiling, True), (ceiling + 1, False)): + dynamodb_client.delete_item(TableName=table, Key={"pk": {"S": "k"}}) + pad = finished - (2 + 1 + 1) + call = lambda: dynamodb_client.update_item( # noqa: E731 + TableName=table, Key={"pk": {"S": "k"}}, + UpdateExpression="SET #twelvechars = :v", + ExpressionAttributeNames={"#twelvechars": "p"}, + ExpressionAttributeValues={":v": {"S": "x" * pad}}, + ) + if ok: + call() + else: + with pytest.raises(ClientError) as exc_info: + call() + assert _err(exc_info)["Message"] == UPDATE_WORDING + + def test_a_set_through_a_list_index_costs_one_byte_more(self, dynamodb_client, table): + # Measured: SET d[0] = :v is charged 20 where SET d = :whole is charged 19. + # finished = pk(2)+1 + d(1) + 3 + (pad + 1) + ceiling = MAX + KEY_K_BYTES - UPDATE_BASE - SET_COST - 1 # 409,580 + for finished, ok in ((ceiling, True), (ceiling + 1, False)): + dynamodb_client.put_item(TableName=table, Item={"pk": {"S": "k"}, "d": {"L": [{"S": ""}]}}) + pad = finished - (2 + 1 + 1 + 3 + 1) + call = lambda: dynamodb_client.update_item( # noqa: E731 + TableName=table, Key={"pk": {"S": "k"}}, + UpdateExpression="SET d[0] = :v", + ExpressionAttributeValues={":v": {"S": "x" * pad}}, + ) + if ok: + call() + else: + with pytest.raises(ClientError) as exc_info: + call() + assert _err(exc_info)["Message"] == UPDATE_WORDING + + def test_a_nested_path_set_is_charged_like_a_plain_set(self, dynamodb_client, table): + ceiling = MAX + KEY_K_BYTES - UPDATE_BASE - SET_COST # 409,581 + for finished, ok in ((ceiling, True), (ceiling + 1, False)): + dynamodb_client.put_item(TableName=table, Item={"pk": {"S": "k"}, "m": {"M": {"a": {"S": ""}}}}) + # finished = pk(2)+1 + m(1) + 3 + (a(1) + pad + 1) + pad = finished - (2 + 1 + 1 + 3 + 1 + 1) + call = lambda: dynamodb_client.update_item( # noqa: E731 + TableName=table, Key={"pk": {"S": "k"}}, + UpdateExpression="SET m.a = :v", + ExpressionAttributeValues={":v": {"S": "x" * pad}}, + ) + if ok: + call() + else: + with pytest.raises(ClientError) as exc_info: + call() + assert _err(exc_info)["Message"] == UPDATE_WORDING + + +def _s(n: int) -> dict: + return {"S": "x" * n} + + +@on_default_backends_only +class TestUpdateStatementShapes: + """The largest accepted padding at a one-byte key, per statement shape. + + Each row is (name, seed item or None, builder) where the builder returns the + UpdateItem arguments for a given padding, and the figure is the largest + padding the service accepted on 2026-10-19; the padding one larger is refused + with the update wording. The seed is written with PutItem before each + attempt. Shapes whose figure is the finished-item cap rather than the + statement figure are marked so in the name. + """ + + SHAPES = [ + ("plain_set", None, 409_577, + lambda n: {"UpdateExpression": "SET p = :v", "ExpressionAttributeValues": {":v": _s(n)}}), + ("two_sets_different_attributes", None, 409_556, + lambda n: {"UpdateExpression": "SET p = :v, q = :w", "ExpressionAttributeValues": {":v": _s(n), ":w": {"S": "y"}}}), + ("two_clauses_same_map_share_the_container", {"m": {"M": {}}}, 409_551, + lambda n: {"UpdateExpression": "SET m.a = :v, m.b = :w", "ExpressionAttributeValues": {":v": _s(n), ":w": {"S": "y"}}}), + ("three_clauses_same_map", {"m": {"M": {}}}, 409_530, + lambda n: {"UpdateExpression": "SET m.a = :v, m.b = :w, m.c = :x", "ExpressionAttributeValues": {":v": _s(n), ":w": {"S": "y"}, ":x": {"S": "y"}}}), + ("nested_two_deep", {"d": {"M": {"a": {"M": {}}}}}, 409_567, + lambda n: {"UpdateExpression": "SET d.a.b = :v", "ExpressionAttributeValues": {":v": _s(n)}}), + ("map_operand", None, 409_572, + lambda n: {"UpdateExpression": "SET p = :v", "ExpressionAttributeValues": {":v": {"M": {"a": _s(n)}}}}), + ("list_index_siblings_not_charged_cap_binds", {"p": {"L": [{"S": "s" * 50}, {"S": "seed"}]}}, 409_541, + lambda n: {"UpdateExpression": "SET p[1] = :v", "ExpressionAttributeValues": {":v": _s(n)}}), + ("if_not_exists_charges_the_operand", None, 409_522, + lambda n: {"UpdateExpression": "SET p = if_not_exists(p, :v)", "ExpressionAttributeValues": {":v": _s(n)}}), + ("if_not_exists_charges_the_operand_when_present", {"p": {"S": "q"}}, 409_522, + lambda n: {"UpdateExpression": "SET p = if_not_exists(p, :v)", "ExpressionAttributeValues": {":v": _s(n)}}), + ("if_not_exists_ten_byte_names", None, 409_504, + lambda n: {"UpdateExpression": "SET pppppppppp = if_not_exists(pppppppppp, :v)", "ExpressionAttributeValues": {":v": _s(n)}}), + ("if_not_exists_other_inner_path", None, 409_513, + lambda n: {"UpdateExpression": "SET p = if_not_exists(qqqqqqqqqq, :v)", "ExpressionAttributeValues": {":v": _s(n)}}), + ("if_not_exists_nested_inner_path", {"d": {"M": {}}}, 409_504, + lambda n: {"UpdateExpression": "SET p = if_not_exists(d.leaf, :v)", "ExpressionAttributeValues": {":v": _s(n)}}), + ("if_not_exists_nested_target", {"d": {"M": {}}}, 409_496, + lambda n: {"UpdateExpression": "SET d.leaf = if_not_exists(d.leaf, :v)", "ExpressionAttributeValues": {":v": _s(n)}}), + ("if_not_exists_indexed_inner_path", {"d": {"L": [{"S": "q"}]}}, 409_507, + lambda n: {"UpdateExpression": "SET p = if_not_exists(d[0], :v)", "ExpressionAttributeValues": {":v": _s(n)}}), + ("list_append_two_values", None, 409_531, + lambda n: {"UpdateExpression": "SET p = list_append(:v, :w)", "ExpressionAttributeValues": {":v": {"L": [_s(n)]}, ":w": {"L": [{"S": "y"}]}}}), + ("list_append_path_value_not_charged", {"p": {"L": [{"S": "q"}]}}, 409_520, + lambda n: {"UpdateExpression": "SET p = list_append(p, :v)", "ExpressionAttributeValues": {":v": {"L": [_s(n)]}}}), + ("list_append_to_big_list_cap_binds", "big_list", 409_590, + lambda n: {"UpdateExpression": "SET p = list_append(p, :v)", "ExpressionAttributeValues": {":v": {"L": [{"S": "y"}]}}}), + ("nested_functions", None, 409_478, + lambda n: {"UpdateExpression": "SET p = list_append(if_not_exists(p, :e), :v)", "ExpressionAttributeValues": {":e": {"L": []}, ":v": {"L": [_s(n)]}}}), + ("addition_with_path_operand", {"n": {"N": "1"}}, 409_505, + lambda n: {"UpdateExpression": "SET p = :v, n = n + :w", "ExpressionAttributeValues": {":v": _s(n), ":w": {"N": "1"}}}), + ("addition_with_two_values", None, 409_519, + lambda n: {"UpdateExpression": "SET p = :v, n = :w + :x", "ExpressionAttributeValues": {":v": _s(n), ":w": {"N": "1"}, ":x": {"N": "1"}}}), + ("subtraction_with_two_values", None, 409_516, + lambda n: {"UpdateExpression": "SET p = :v, n = :w - :x", "ExpressionAttributeValues": {":v": _s(n), ":w": {"N": "1"}, ":x": {"N": "1"}}}), + ("add_number", None, 409_555, + lambda n: {"UpdateExpression": "SET p = :v ADD n :w", "ExpressionAttributeValues": {":v": _s(n), ":w": {"N": "1"}}}), + ("add_to_big_set_cap_binds", "big_set", 409_595, + lambda n: {"UpdateExpression": "ADD s :v", "ExpressionAttributeValues": {":v": {"SS": ["y"]}}}), + ("remove_nested_path", {"d": {"M": {"leaf": {"S": "y"}}}}, 409_567, + lambda n: {"UpdateExpression": "SET p = :v REMOVE d.leaf", "ExpressionAttributeValues": {":v": _s(n)}}), + ("remove_through_list_index", {"d": {"L": [{"S": "y"}]}}, 409_570, + lambda n: {"UpdateExpression": "SET p = :v REMOVE d[0]", "ExpressionAttributeValues": {":v": _s(n)}}), + ("delete_operand_not_charged", {"s": {"SS": ["a", "b"]}}, 409_575, + lambda n: {"UpdateExpression": "SET p = :v DELETE s :w", "ExpressionAttributeValues": {":v": _s(n), ":w": {"SS": ["a"]}}}), + ("legacy_put", None, 409_596, + lambda n: {"AttributeUpdates": {"p": {"Value": _s(n), "Action": "PUT"}}}), + ("legacy_two_puts", None, 409_594, + lambda n: {"AttributeUpdates": {"p": {"Value": _s(n), "Action": "PUT"}, "q": {"Value": {"S": "y"}, "Action": "PUT"}}}), + ("legacy_put_ten_byte_name", None, 409_587, + lambda n: {"AttributeUpdates": {"pppppppppp": {"Value": _s(n), "Action": "PUT"}}}), + ("legacy_put_map", None, 409_591, + lambda n: {"AttributeUpdates": {"p": {"Value": {"M": {"a": _s(n)}}, "Action": "PUT"}}}), + ("legacy_put_and_delete_charges_the_finished_set", {"s": {"SS": ["a", "b"]}}, 409_594, + lambda n: {"AttributeUpdates": {"p": {"Value": _s(n), "Action": "PUT"}, "s": {"Value": {"SS": ["a"]}, "Action": "DELETE"}}}), + ("legacy_put_and_remove_charges_nothing_for_the_removed", {"s": {"SS": ["a", "b"]}}, 409_596, + lambda n: {"AttributeUpdates": {"p": {"Value": _s(n), "Action": "PUT"}, "s": {"Action": "DELETE"}}}), + ] + + @staticmethod + def _seed(seed, n): + if seed == "big_list": + return {"p": {"L": [_s(n)]}} + if seed == "big_set": + return {"s": {"SS": ["x" * n]}} + return seed + + @pytest.mark.parametrize("name,seed,ceiling,build", SHAPES, ids=[r[0] for r in SHAPES]) + def test_largest_accepted_padding(self, dynamodb_client, table, name, seed, ceiling, build): + key = {"pk": {"S": "k"}} + for n, ok in ((ceiling, True), (ceiling + 1, False)): + dynamodb_client.delete_item(TableName=table, Key=key) + seeded = self._seed(seed, n) + try: + if seeded: + dynamodb_client.put_item(TableName=table, Item={"pk": {"S": "k"}, **seeded}) + dynamodb_client.update_item(TableName=table, Key=key, **build(n)) + accepted = True + except ClientError as exc: + assert exc.response["Error"]["Code"] == "ValidationException" + assert "size" in exc.response["Error"]["Message"], exc.response["Error"]["Message"] + accepted = False + assert accepted is ok, f"{name}: padding {n} {'accepted' if accepted else 'refused'}" + + +@on_default_backends_only +class TestTransactedUpdateSizeRule: + """A transacted Update is sized on the finished item and cancels with the update wording.""" + + def _txn(self, client, table, finished): + pad = finished - (2 + 1 + 1) + client.transact_write_items(TransactItems=[{ + "Update": { + "TableName": table, "Key": {"pk": {"S": "t"}}, + "UpdateExpression": "SET p = :v", + "ExpressionAttributeValues": {":v": {"S": "x" * pad}}, + } + }]) + + def test_exactly_the_limit_is_stored(self, dynamodb_client, table): + dynamodb_client.delete_item(TableName=table, Key={"pk": {"S": "t"}}) + self._txn(dynamodb_client, table, MAX) + got = dynamodb_client.get_item(TableName=table, Key={"pk": {"S": "t"}}, ConsistentRead=True) + assert len(got["Item"]["p"]["S"]) == MAX - 4 + + def test_one_byte_over_cancels_with_the_update_wording(self, dynamodb_client, table): + dynamodb_client.delete_item(TableName=table, Key={"pk": {"S": "t"}}) + with pytest.raises(ClientError) as exc_info: + self._txn(dynamodb_client, table, MAX + 1) + resp = exc_info.value.response + assert resp["Error"]["Code"] == "TransactionCanceledException" + reasons = resp["CancellationReasons"] + assert reasons[0]["Code"] == "ValidationError" + assert reasons[0]["Message"] == UPDATE_WORDING + + +@on_default_backends_only +class TestEmptyBinaryIndexKeyOnUpdate: + """UpdateItem setting a GSI key to an empty binary is refused with the update-form message.""" + + def test_update_refuses_empty_binary_index_key(self, dynamodb_client, gsi_table): + dynamodb_client.put_item(TableName=gsi_table, Item={"pk": {"S": "eb"}, "v": {"S": "1"}}) + with pytest.raises(ClientError) as exc_info: + dynamodb_client.update_item( + TableName=gsi_table, Key={"pk": {"S": "eb"}}, + UpdateExpression="SET bidx = :b", + ExpressionAttributeValues={":b": {"B": b""}}, + ) + assert _err(exc_info)["Code"] == "ValidationException" + assert _err(exc_info)["Message"] == ( + "One or more parameter values are not valid. The update expression attempted to " + "update a secondary index key to a value that is not supported. The AttributeValue " + "for a key attribute cannot contain an empty binary value." + ) + got = dynamodb_client.get_item(TableName=gsi_table, Key={"pk": {"S": "eb"}}, ConsistentRead=True) + assert "bidx" not in got["Item"]