diff --git a/crates/core/src/expression/resolver.rs b/crates/core/src/expression/resolver.rs index fa14081ec..699cd4fdf 100755 --- a/crates/core/src/expression/resolver.rs +++ b/crates/core/src/expression/resolver.rs @@ -26,6 +26,10 @@ use super::ast::{CompareOp, Expr, PathElement}; pub struct ExpressionMaps { pub names: HashMap, pub values: HashMap, + /// Set by the engine when the update arrived as legacy `AttributeUpdates` + /// and was desugared into an update expression. The service sizes that + /// form against the item limit differently from an expression. + pub legacy_attribute_updates: bool, /// Pre-parsed `BigDecimal` values for numeric placeholders. /// Populated eagerly by `new()` and refreshed by `pre_parse_numerics()`. parsed_numerics: HashMap, @@ -40,6 +44,7 @@ impl ExpressionMaps { let mut maps = Self { names, values, + legacy_attribute_updates: false, parsed_numerics: HashMap::new(), }; maps.pre_parse_numerics(); diff --git a/crates/core/src/types/item.rs b/crates/core/src/types/item.rs index 0d31edb72..e363cca46 100755 --- a/crates/core/src/types/item.rs +++ b/crates/core/src/types/item.rs @@ -388,35 +388,75 @@ pub fn attribute_value_size(value: &AttributeValue) -> usize { /// Calculate the `DynamoDB` size of a number in bytes. /// -/// `DynamoDB` number sizing: approximately 1 byte per 2 significant digits + 1 byte. -/// Zero is 1 byte. Negative numbers add 1 byte. Max 21 bytes. +/// Measured against the service (us-east-1, 2026-10-19) by accept/refuse pairs +/// either side of the 409,600-byte item limit: +/// +/// ```text +/// 1 + ceil(integer significant digits / 2) + ceil(fraction significant digits / 2) +/// + 1 if negative +/// ``` +/// +/// The two halves are rounded up separately, so digits that straddle the +/// decimal point cost one byte more than the same digits without a point: +/// `1.5` is 3 bytes where `15` is 2, and `3.14159` is 5 where `123456` is 4. +/// Leading zeros of the integer half and trailing zeros of the fraction half +/// are not significant; trailing zeros of the integer half are significant +/// when a fraction follows them (`100` is 2, `100.5` is 4) and a zero between +/// two significant digits always is (`1010` is 3). The exponent is not +/// counted (`1E125` is 2). Zero is 1 byte however it is spelled, and is never +/// negative (`-0` is 1). fn dynamodb_number_size(n: &str) -> usize { - let s = n.trim_start_matches('-'); - let is_zero = s.chars().all(|c| c == '0' || c == '.'); - if is_zero { + let (negative, body) = match n.strip_prefix('-') { + Some(rest) => (true, rest), + None => (false, n.strip_prefix('+').unwrap_or(n)), + }; + let (mantissa, exponent) = match body.find(['e', 'E']) { + Some(pos) => (&body[..pos], body[pos + 1..].parse::().unwrap_or(0)), + None => (body, 0), + }; + let (int_text, frac_text) = match mantissa.find('.') { + Some(pos) => (&mantissa[..pos], &mantissa[pos + 1..]), + None => (mantissa, ""), + }; + let digits: String = int_text.chars().chain(frac_text.chars()).collect(); + if !digits.bytes().any(|b| (b'1'..=b'9').contains(&b)) { return 1; } - - let significant = if let Some(dot_pos) = s.find('.') { - let (int_part, frac_part) = s.split_at(dot_pos); - let frac = &frac_part[1..]; - let int_trimmed = int_part.trim_start_matches('0'); - let frac_trimmed = frac.trim_end_matches('0'); - if int_trimmed.is_empty() { - let frac_sig = frac.trim_start_matches('0'); - frac_sig.trim_end_matches('0').len() - } else { - format!("{int_trimmed}{frac_trimmed}").len() - } + // Where the decimal point sits in `digits` once the exponent is applied. + let point = i64::try_from(int_text.len()) + .unwrap_or(i64::MAX) + .saturating_add(exponent); + let len = i64::try_from(digits.len()).unwrap_or(i64::MAX); + let (int_part, frac_part): (String, String) = if point <= 0 { + ( + String::new(), + "0".repeat(usize::try_from(-point).unwrap_or(0)) + &digits, + ) + } else if point >= len { + ( + digits.clone() + &"0".repeat(usize::try_from(point - len).unwrap_or(0)), + String::new(), + ) } else { - let trimmed = s.trim_start_matches('0').trim_end_matches('0'); - if trimmed.is_empty() { 1 } else { trimmed.len() } + let split = usize::try_from(point).unwrap_or(0); + (digits[..split].to_owned(), digits[split..].to_owned()) }; - - let mut size = significant.div_ceil(2) + 1; - if n.starts_with('-') { - size += 1; - } + let int_part = int_part.trim_start_matches('0'); + let frac_part = frac_part.trim_end_matches('0'); + // Trailing zeros of the integer half fall away only when no fraction follows. + let int_significant = if frac_part.is_empty() { + int_part.trim_end_matches('0').len() + } else { + int_part.len() + }; + // Leading zeros of the fraction half fall away only when no integer precedes. + let frac_significant = if int_part.is_empty() { + frac_part.trim_start_matches('0').len() + } else { + frac_part.len() + }; + let size = + 1 + int_significant.div_ceil(2) + frac_significant.div_ceil(2) + usize::from(negative); size.min(21) } @@ -428,6 +468,32 @@ mod tests { AttributeValue::S(v.to_owned()) } + #[test] + fn number_size_matches_the_service_for_measured_literals() { + // Each figure measured against the service (us-east-1, 2026-10-19) by an + // accept/refuse pair either side of the 409,600-byte item limit. + for (literal, bytes) in [ + ("0", 1), + ("-0", 1), + ("1", 2), + ("15", 2), + ("100", 2), + ("1E125", 2), + ("0.15", 2), + ("1.5", 3), + ("1.2", 3), + ("1.200", 3), + ("1010", 3), + ("-42", 3), + ("1.234", 4), + ("100.5", 4), + ("123456", 4), + ("3.14159", 5), + ] { + assert_eq!(dynamodb_number_size(literal), bytes, "literal {literal}"); + } + } + #[test] fn scalar_sizes_unchanged() { assert_eq!(attribute_value_size(&s("hello")), 5); diff --git a/crates/core/src/validation/mod.rs b/crates/core/src/validation/mod.rs index 3decb4666..b34d8248d 100755 --- a/crates/core/src/validation/mod.rs +++ b/crates/core/src/validation/mod.rs @@ -1876,6 +1876,257 @@ pub fn validate_item_size(item: &Item, max_bytes: usize) -> Result<(), DynamoDbE Ok(()) } +/// The wording the service uses when an update, standalone or transacted, would +/// exceed the item size limit. +pub const ITEM_SIZE_TO_UPDATE_EXCEEDED: &str = + "Item size to update has exceeded the maximum allowed size"; + +/// Fixed cost the service charges an `UpdateItem` statement, before its clauses. +pub const UPDATE_STATEMENT_BASE_COST: usize = 3; +/// Fixed cost per `SET` or `ADD` clause, on top of the written path and operands. +pub const UPDATE_SET_CLAUSE_COST: usize = 18; +/// Fixed cost of a function call in a `SET` operand, plus the function name's length. +pub const UPDATE_FUNCTION_COST: usize = 26; +/// Fixed cost of a `+` in a `SET` operand (26 plus the length of "addition"). +pub const UPDATE_ADDITION_COST: usize = 34; +/// Fixed cost of a `-` in a `SET` operand (26 plus the length of "subtraction"). +pub const UPDATE_SUBTRACTION_COST: usize = 37; +/// Cost of the first element of a document path used as a function or arithmetic +/// operand, plus the attribute name's length. The attribute's stored value is not +/// charged. +pub const UPDATE_PATH_OPERAND_FIRST_COST: usize = 15; +/// Cost of a later attribute element of such a path, plus the name's length. +pub const UPDATE_PATH_OPERAND_NESTED_COST: usize = 14; +/// Cost of a list index element of such a path. +pub const UPDATE_PATH_OPERAND_INDEX_COST: usize = 15; +/// Extra cost of a clause whose target path ends in a list index. +pub const UPDATE_LIST_INDEX_TARGET_COST: usize = 1; + +/// Size a standalone `UpdateItem` the way the service does. +/// +/// Measured against the service (us-east-1, 2026-10-19) by accept/refuse pairs +/// either side of the item limit, for whole-attribute and document-path +/// targets, aliases, one to three clauses on the same or different attributes, +/// `if_not_exists`, `list_append`, nested functions, arithmetic, `ADD`, +/// `REMOVE`, `DELETE`, and the legacy `AttributeUpdates` form. The figure the +/// limit applies to is: +/// +/// * a base of 3; +/// * 18 per `SET` or `ADD` clause, nothing per `REMOVE` or `DELETE`; +/// * the size of a skeleton item holding only the written paths: each map +/// entry its name plus one, each map or list 3, each list element one more, +/// siblings the statement does not touch left out. A `SET` or `ADD` leaf is +/// charged the sizes of the clause's value placeholders; a `REMOVE` or +/// `DELETE` leaf is charged nothing, and the `DELETE` operand is not charged; +/// * one more for a clause whose target path ends in a list index; +/// * for operands inside a `SET` value: 26 plus the name length per function +/// call, 34 per `+` and 37 per `-`, and for a document path 15 plus the +/// name for its first element, 14 plus the name for a later attribute, 15 +/// for an index, with the attribute's stored value not charged. +/// +/// The legacy `AttributeUpdates` form is charged 3 plus, for each attribute +/// it names that is present afterwards, the name and the finished value's +/// bytes; an attribute it removes costs nothing. +/// +/// The key and untouched attributes are left out, so at a short key the largest +/// finished item an update can produce sits below the limit, and a long key or +/// an untouched attribute buys the headroom back up to the limit itself, which +/// the finished item never exceeds. +/// +/// `finished` is the whole finished item; `actions` are the statement's clauses +/// with `#name` placeholders still in place, resolved here through `maps`. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` with the update wording when +/// either the statement figure or the finished item exceeds `max_bytes`. +pub fn validate_update_item_size( + finished: &Item, + actions: &[crate::expression::UpdateAction], + maps: &crate::expression::ExpressionMaps, + max_bytes: usize, +) -> Result<(), DynamoDbError> { + let statement = if maps.legacy_attribute_updates { + legacy_update_statement_size(finished, actions, maps)? + } else { + update_statement_size(actions, maps)? + }; + if statement > max_bytes || item_size_bytes(finished) > max_bytes { + return Err(DynamoDbError::ValidationException( + ITEM_SIZE_TO_UPDATE_EXCEEDED.to_owned(), + )); + } + Ok(()) +} + +/// The bytes the service charges the operand expression of a `SET` clause: +/// value placeholders at their size, function calls, arithmetic operators and +/// document paths at their fixed costs. +fn update_operand_size( + expr: &crate::expression::Expr, + maps: &crate::expression::ExpressionMaps, +) -> Result { + use crate::expression::{Expr, PathElement, resolve_element_name}; + Ok(match expr { + Expr::Placeholder(name) => maps + .values + .get(name) + .map_or(0, crate::types::attribute_value_size), + Expr::Path(elements) => { + let mut bytes = 0; + for (i, element) in elements.iter().enumerate() { + bytes += match element { + PathElement::Index(_) => UPDATE_PATH_OPERAND_INDEX_COST, + PathElement::Attribute(_) => { + let name = resolve_element_name(element, maps)?; + name.len() + + if i == 0 { + UPDATE_PATH_OPERAND_FIRST_COST + } else { + UPDATE_PATH_OPERAND_NESTED_COST + } + } + }; + } + bytes + } + Expr::Function { name, args } => { + let mut bytes = UPDATE_FUNCTION_COST + name.len(); + for arg in args { + bytes += update_operand_size(arg, maps)?; + } + bytes + } + Expr::Arithmetic { left, op, right } => { + let operator = match op { + crate::expression::ArithOp::Add => UPDATE_ADDITION_COST, + crate::expression::ArithOp::Sub => UPDATE_SUBTRACTION_COST, + }; + operator + update_operand_size(left, maps)? + update_operand_size(right, maps)? + } + _ => 0, + }) +} + +/// A written path as a tree: map entries by name, list elements by index. +#[derive(Default)] +struct WrittenNode { + leaf_bytes: usize, + entries: std::collections::BTreeMap, + elements: std::collections::BTreeMap, +} + +fn written_node_size(node: &WrittenNode) -> usize { + let maps: usize = if node.entries.is_empty() { + 0 + } else { + 3 + node + .entries + .iter() + .map(|(name, child)| name.len() + written_node_size(child) + 1) + .sum::() + }; + let lists: usize = if node.elements.is_empty() { + 0 + } else { + 3 + node + .elements + .values() + .map(|child| written_node_size(child) + 1) + .sum::() + }; + node.leaf_bytes + maps + lists +} + +/// Build the skeleton the statement writes and size it with the clause costs. +fn update_statement_size( + actions: &[crate::expression::UpdateAction], + maps: &crate::expression::ExpressionMaps, +) -> Result { + use crate::expression::{PathElement, UpdateAction, resolve_element_name}; + let mut root = WrittenNode::default(); + let mut bytes = UPDATE_STATEMENT_BASE_COST; + for action in actions { + let (path, leaf) = match action { + UpdateAction::Set { path, value } | UpdateAction::Add { path, value } => { + bytes += UPDATE_SET_CLAUSE_COST; + (path, update_operand_size(value, maps)?) + } + UpdateAction::Remove { path } | UpdateAction::Delete { path, .. } => (path, 0), + }; + if matches!(path.last(), Some(PathElement::Index(_))) { + bytes += UPDATE_LIST_INDEX_TARGET_COST; + } + let mut node = &mut root; + for element in path { + node = match element { + PathElement::Index(i) => node.elements.entry(*i).or_default(), + PathElement::Attribute(_) => { + let name = resolve_element_name(element, maps)?; + node.entries.entry(name.into_owned()).or_default() + } + }; + } + node.leaf_bytes += leaf; + } + // The item itself is a map whose 3 bytes the base already covers. + Ok(bytes + written_node_size(&root).saturating_sub(3)) +} + +/// The legacy `AttributeUpdates` form: 3 plus, for each named attribute still +/// present in the finished item, its name and finished value bytes. Nothing per +/// action, no entry byte, nothing for an attribute the update removed. +fn legacy_update_statement_size( + finished: &Item, + actions: &[crate::expression::UpdateAction], + maps: &crate::expression::ExpressionMaps, +) -> Result { + use crate::expression::{UpdateAction, resolve_element_name}; + let mut names: Vec = Vec::new(); + for action in actions { + let path = match action { + UpdateAction::Set { path, .. } + | UpdateAction::Add { path, .. } + | UpdateAction::Remove { path } + | UpdateAction::Delete { path, .. } => path, + }; + if let Some(first) = path.first() { + let name = resolve_element_name(first, maps)?; + if !names.iter().any(|n| n == name.as_ref()) { + names.push(name.into_owned()); + } + } + } + Ok(UPDATE_STATEMENT_BASE_COST + + names + .iter() + .filter_map(|name| finished.get(name).map(|value| (name, value))) + .map(|(name, value)| name.len() + crate::types::attribute_value_size(value)) + .sum::()) +} + +/// Size a transacted `Update` the way the service does: the finished item +/// against the limit, with the update wording. Measured against the service +/// (us-east-1, 2026-10-19): a transacted Update accepts a finished item of +/// exactly the limit and cancels one byte over with this wording, so the +/// statement-sized rule of a standalone `UpdateItem` does not apply. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` with the update wording when the +/// finished item exceeds `max_bytes`. +pub fn validate_transact_update_item_size( + finished: &Item, + max_bytes: usize, +) -> Result<(), DynamoDbError> { + if item_size_bytes(finished) > max_bytes { + return Err(DynamoDbError::ValidationException( + ITEM_SIZE_TO_UPDATE_EXCEEDED.to_owned(), + )); + } + Ok(()) +} + /// Validate all number values in an item are within `DynamoDB` limits. pub fn validate_item_numbers(item: &Item) -> Result<(), DynamoDbError> { for value in item.values() { @@ -3801,3 +4052,117 @@ mod tests { assert!(validate_conditional_operator_usage(false, 1).is_ok()); } } + +#[cfg(test)] +mod update_size_tests { + //! Direct unit tests for the UpdateItem statement sizer. The byte figures + //! are the ones measured against the service in us-east-1 on 2026-10-19 and + //! exercised end to end by tests/test_item_size_at_limit.py; these assert + //! the same numbers at the function boundary so a developer can run them + //! with `cargo test` and no server. + + use std::collections::HashMap; + + use super::{ + ITEM_SIZE_TO_UPDATE_EXCEEDED, legacy_update_statement_size, update_statement_size, + validate_update_item_size, + }; + use crate::expression::{ExpressionMaps, UpdateAction, parse_update, tokenize}; + use crate::types::{AttributeValue, Item}; + + fn s(v: &str) -> AttributeValue { + AttributeValue::S(v.to_owned()) + } + + /// Parse an update expression into actions plus a maps table whose values + /// are keyed as the parser keys them (the `:name` placeholder without its + /// colon). + fn build(expr: &str, values: &[(&str, AttributeValue)]) -> (Vec, ExpressionMaps) { + let actions = parse_update(&tokenize(expr).unwrap()).unwrap(); + let mut map = HashMap::new(); + for (name, value) in values { + map.insert((*name).to_owned(), value.clone()); + } + (actions, ExpressionMaps::new(HashMap::new(), map)) + } + + fn statement(expr: &str, values: &[(&str, AttributeValue)]) -> usize { + let (actions, maps) = build(expr, values); + update_statement_size(&actions, &maps).unwrap() + } + + #[test] + fn single_set_is_base_clause_path_and_value() { + // 3 base + 18 clause + the written path `p` (name 1, two container + // bytes) + the 4-byte value. + assert_eq!(statement("SET p = :v", &[("v", s("wxyz"))]), 27); + } + + #[test] + fn two_set_clauses_each_cost_their_clause_and_path() { + assert_eq!( + statement("SET a = :v, b = :w", &[("v", s("wxyz")), ("w", s("y"))]), + 48, + ); + } + + #[test] + fn a_nested_path_sizes_each_level_once() { + assert_eq!(statement("SET m.a = :v", &[("v", s("wxyz"))]), 32); + } + + #[test] + fn a_list_index_target_adds_one_byte() { + // `p[1]` writes the attribute `p` (name 1, two container bytes) holding + // a one-element list (three container bytes, one per element) with the + // 4-byte value, plus 18 clause and 1 for writing through a list index: + // 3 + 18 + 1 + 10 = 32. One byte more than the same write to `p[0]`'s + // plain attribute would be without the list, which is the list-index + // target byte. + assert_eq!(statement("SET p[1] = :v", &[("v", s("wxyz"))]), 32); + } + + #[test] + fn remove_costs_its_path_skeleton_only() { + // No clause cost for REMOVE: 3 base + the path `p`. + assert_eq!(statement("REMOVE p", &[]), 5); + } + + #[test] + fn statement_over_the_limit_is_refused_with_the_update_wording() { + // SET p = :v with a ten-byte value is a 33-byte statement; the finished + // item ({p: 10 bytes}) is smaller, so the statement figure binds. + let (actions, maps) = build("SET p = :v", &[("v", s("0123456789"))]); + let mut finished = Item::new(); + finished.insert("p".to_owned(), s("0123456789")); + + assert!(validate_update_item_size(&finished, &actions, &maps, 33).is_ok()); + let err = validate_update_item_size(&finished, &actions, &maps, 32).unwrap_err(); + assert_eq!(err.to_string(), ITEM_SIZE_TO_UPDATE_EXCEEDED); + } + + #[test] + fn an_oversized_finished_item_is_refused_even_when_the_statement_fits() { + // A small statement (REMOVE q, 7 bytes) cannot rescue a finished item + // that is itself over the limit. + let (actions, maps) = build("REMOVE q", &[]); + let mut finished = Item::new(); + finished.insert("p".to_owned(), s("0123456789")); + assert!(validate_update_item_size(&finished, &actions, &maps, 5).is_err()); + } + + #[test] + fn legacy_attribute_updates_size_the_finished_value_not_the_statement() { + // The legacy form charges 3 plus the name and finished value of each + // attribute it names that remains in the item. + let (actions, mut maps) = build("SET p = :v", &[("v", s("wxyz"))]); + maps.legacy_attribute_updates = true; + let mut finished = Item::new(); + finished.insert("p".to_owned(), s("wxyz")); + // 3 base + name `p` (1) + value (4) = 8. + assert_eq!( + legacy_update_statement_size(&finished, &actions, &maps).unwrap(), + 8 + ); + } +} diff --git a/crates/engine/src/update_item.rs b/crates/engine/src/update_item.rs index 7f74c205c..fbd9326ec 100755 --- a/crates/engine/src/update_item.rs +++ b/crates/engine/src/update_item.rs @@ -137,7 +137,7 @@ pub async fn handle_update_item( input.expected.as_ref().map_or(0, HashMap::len), )?; - let (condition, maps) = resolve_condition( + let (condition, mut maps) = resolve_condition( input.condition_expression.as_deref(), effective_expr_names.as_ref(), effective_expr_values.as_ref(), @@ -145,6 +145,9 @@ pub async fn handle_update_item( input.conditional_operator, &ctx.limits, )?; + // The storage layer sizes a desugared legacy update against the item limit + // the way the service sizes `AttributeUpdates`, not an expression. + maps.legacy_attribute_updates = input.attribute_updates.is_some(); // No UpdateExpression and no AttributeUpdates: no-op upsert. // Some("") still errors via parse_update_expr. diff --git a/crates/server/src/serve.rs b/crates/server/src/serve.rs index 9ebbaccd4..a41daea21 100644 --- a/crates/server/src/serve.rs +++ b/crates/server/src/serve.rs @@ -289,6 +289,7 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { extenddb_storage::server_components::ServerComponentsOptions::default(); component_options.bootstrap_if_uninitialized = dev_mode; + component_options.max_item_size_bytes = app_config.limits.max_item_size_bytes; // Wire the node's own bind address as the instance ID so each ExtendDB node // has a distinct node ID (to support logical clocks for sequences, etc.). diff --git a/crates/storage-postgres/src/data/transactions.rs b/crates/storage-postgres/src/data/transactions.rs index e86c00a90..ce0360c25 100644 --- a/crates/storage-postgres/src/data/transactions.rs +++ b/crates/storage-postgres/src/data/transactions.rs @@ -662,10 +662,12 @@ async fn execute_transact_write_op( .map_err(|e| { TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) })?; - // Validate post-update item size - validation::validate_item_size(&item, max_item_size_bytes).map_err(|e| { - TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) - })?; + // A transacted Update is sized on the finished item, with the + // update wording in the cancellation reason. + validation::validate_transact_update_item_size(&item, max_item_size_bytes) + .map_err(|e| { + TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) + })?; // Secondary-index key validation on the post-update item: a type // mismatch is a cancellation reason; setting an index key to an // empty value is a top-level ValidationException. diff --git a/crates/storage-postgres/src/data/update_item.rs b/crates/storage-postgres/src/data/update_item.rs index 80ab9a515..56872833f 100755 --- a/crates/storage-postgres/src/data/update_item.rs +++ b/crates/storage-postgres/src/data/update_item.rs @@ -187,8 +187,9 @@ impl PostgresEngine { ) .map_err(|e| StorageError::Validation(e.to_string()))?; - // Validate post-update item size (400 KB limit) - validation::validate_item_size(&item, self.max_item_size_bytes) + // Size the update the way the service does: what the statement + // writes plus a fixed cost per clause, against the item limit. + validation::validate_update_item_size(&item, actions, maps, self.max_item_size_bytes) .map_err(|e| StorageError::Validation(e.to_string()))?; // Secondary-index key validation on the post-update item, as the diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index e9f1ae2e9..acb1aef56 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -745,7 +745,7 @@ fn server_components_factory( // PostgreSQL bootstrap needs operator input (databases, roles, admin // credentials), so `bootstrap_if_uninitialized` is not honored here: // an uninitialized catalog fails with the explicit-`init` guidance. - _options: extenddb_storage::server_components::ServerComponentsOptions, + options: extenddb_storage::server_components::ServerComponentsOptions, ) -> std::pin::Pin< Box> + Send>, > { @@ -758,7 +758,7 @@ fn server_components_factory( let pg_config = PostgresConfig { connection_string: connection_string.clone(), pool_size: max_connections, - max_item_size_bytes: 400_000, + max_item_size_bytes: options.max_item_size_bytes, }; // Create PostgresEngine diff --git a/crates/storage-sqlite/src/data/transactions.rs b/crates/storage-sqlite/src/data/transactions.rs index e9b5f8f5b..8be6a5683 100644 --- a/crates/storage-sqlite/src/data/transactions.rs +++ b/crates/storage-sqlite/src/data/transactions.rs @@ -417,9 +417,11 @@ async fn execute_transact_write_op( &key_info.attribute_definitions, ) .map_err(|e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())))?; - validation::validate_item_size(&item, max_item_size_bytes).map_err(|e| { - TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())) - })?; + // A transacted Update is sized on the finished item, with the update + // wording in the cancellation reason. + validation::validate_transact_update_item_size(&item, max_item_size_bytes).map_err( + |e| TxnOpError::Cancel(CancellationReason::validation_error(e.to_string())), + )?; // Secondary-index key validation on the post-update item: a type // mismatch is a cancellation reason; setting an index key to an // empty value is a top-level ValidationException. diff --git a/crates/storage-sqlite/src/data/update_item.rs b/crates/storage-sqlite/src/data/update_item.rs index ae18ae1e9..58a1db2bb 100644 --- a/crates/storage-sqlite/src/data/update_item.rs +++ b/crates/storage-sqlite/src/data/update_item.rs @@ -72,7 +72,9 @@ impl SqliteEngine { &key_info.attribute_definitions, ) .map_err(|e| StorageError::Validation(e.to_string()))?; - validation::validate_item_size(&item, self.max_item_size_bytes) + // Size the update the way the service does: what the statement writes + // plus a fixed cost per clause, against the item limit. + validation::validate_update_item_size(&item, actions, maps, self.max_item_size_bytes) .map_err(|e| StorageError::Validation(e.to_string()))?; // Secondary-index key validation on the post-update item, matching the // TransactWriteItems update path: a wrong-typed index key attribute or diff --git a/crates/storage-sqlite/src/lib.rs b/crates/storage-sqlite/src/lib.rs index 3953fc350..75c91336f 100644 --- a/crates/storage-sqlite/src/lib.rs +++ b/crates/storage-sqlite/src/lib.rs @@ -193,9 +193,6 @@ use extenddb_storage::server_components::{BackendError, ServerComponents}; use crate::hooks::SqliteRuntimeHooks; -/// Maximum DynamoDB item size in bytes (post-update validation bound). -const MAX_ITEM_SIZE_BYTES: usize = 400_000; - fn sqlite_server_components_factory( config: &(dyn extenddb_storage::config::StorageConfig + 'static), region: &str, @@ -214,12 +211,13 @@ fn sqlite_server_components_factory( tracing::debug!("holding serve lock {}", lock.path().display()); std::sync::Arc::new(lock) }); - let mut engine = SqliteEngine::new(&db_path, pool_size, ®ion, MAX_ITEM_SIZE_BYTES) - .await - .map_err(|e| BackendError::ConnectionFailed { - backend: "sqlite".to_owned(), - details: e.to_string(), - })?; + let mut engine = + SqliteEngine::new(&db_path, pool_size, ®ion, options.max_item_size_bytes) + .await + .map_err(|e| BackendError::ConnectionFailed { + backend: "sqlite".to_owned(), + details: e.to_string(), + })?; // In-memory databases do not persist across the `init` process, so the // catalog must be bootstrapped here, at serve time, on the engine's own diff --git a/crates/storage/src/server_components.rs b/crates/storage/src/server_components.rs index 2ea59c9c9..a2c3cc288 100644 --- a/crates/storage/src/server_components.rs +++ b/crates/storage/src/server_components.rs @@ -86,7 +86,7 @@ impl std::fmt::Display for BackendError { impl std::error::Error for BackendError {} /// Options for server-component creation, beyond the storage config itself. -#[derive(Debug, Clone, Copy, Default)] +#[derive(Debug, Clone, Copy)] #[non_exhaustive] pub struct ServerComponentsOptions { /// Bootstrap the catalog at serve time when it is not initialized, @@ -95,6 +95,19 @@ pub struct ServerComponentsOptions { /// Backends whose bootstrap requires operator input ignore this and fail /// as usual; backends with a self-contained bootstrap (SQLite) honor it. pub bootstrap_if_uninitialized: bool, + /// The item size limit the storage layer applies to the finished item of + /// an update, standalone or transacted. Comes from `limits.max_item_size_bytes` + /// so the storage layer and the engine's put path enforce one figure. + pub max_item_size_bytes: usize, +} + +impl Default for ServerComponentsOptions { + fn default() -> Self { + Self { + bootstrap_if_uninitialized: false, + max_item_size_bytes: extenddb_core::limits::LimitsConfig::default().max_item_size_bytes, + } + } } /// Factory function type for creating server components. diff --git a/devtools/run-tests b/devtools/run-tests index 421a97b3a..ae7984896 100755 --- a/devtools/run-tests +++ b/devtools/run-tests @@ -141,9 +141,10 @@ case "$BACKEND" in exit 1 ;; esac -# Tests that depend on the backend (tests/test_ttl.py) read this rather than -# guess from a config file. Only exported when --backend was given: the -# SQLite CI leg runs without it. +# Tests that depend on the backend (tests/test_ttl.py, and the update-size +# classes in tests/test_item_size_at_limit.py that apply only to the PostgreSQL +# and SQLite storage layers) read this rather than guess from a config file. +# Only exported when --backend was given: the SQLite CI leg runs without it. if [[ -n "${BACKEND_GIVEN:-}" ]]; then export EXTENDDB_TEST_BACKEND="$BACKEND" fi diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index 2d6469b7e..c8d3d97ca 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -139,6 +139,37 @@ The same on all four backends. | `OnDemandThroughputOverride` | Applied on an on-demand restore, as on the service (measured on DynamoDB): equal to the backup's, changed, with one member, or on a `PROVISIONED` backup restored with `BillingModeOverride` `PAY_PER_REQUEST`. It replaces the backup's on-demand throughput as a whole: a member left out means no maximum, as on `CreateTable`, so `{}` restores a table with no maxima. On a `PROVISIONED` restore (from a `PROVISIONED` backup or by override) it is refused with `ValidationException`, as is a member other than `MaxReadRequestUnits` and `MaxWriteRequestUnits`. Measured on SQLite | | `VectorIndexOverride` | Refused with `ValidationException` | +## Item Size + +ExtendDB enforces the 400 KB (409,600-byte) item-size limit. This section is the +single reference for how an item and an update are sized; `docs/dynamodb-limits.md` +records the limit and points here. Every figure was measured against the service +in us-east-1 on 2026-10-19. + +How an item is sized: its attribute names plus its values. A string or binary +value costs its byte length. A number costs `1 + ceil(integer significant digits +/ 2) + ceil(fraction significant digits / 2)`, plus one for a sign, the two +halves rounded up separately, so `1.5` is 3 bytes and `3.14159` is 5. A list +costs 3 bytes plus one per element; a map costs 3 bytes plus, per entry, the +entry name, its value, and one more. + +How a standalone `UpdateItem` is sized: by the statement, not the finished item. +A base of 3; 18 per `SET` or `ADD` clause; a skeleton of the written paths (each +map level 3 bytes plus one per named entry, each list level 3 bytes plus one per +element, untouched siblings left out) whose `SET` or `ADD` leaves carry their +value-placeholder bytes; one more per clause written through a list index; 26 +plus the name length per function call; 34 per `+` and 37 per `-`; and 15 plus +the name per document path operand (14 for a nested attribute, 15 for an index). +`REMOVE` and `DELETE` cost their path skeleton only. The legacy `AttributeUpdates` +form costs 3 plus the name and finished value of each attribute it names that +remains. The key and untouched attributes are left out. Both the statement figure +and the finished item are checked against the limit, and a breach refuses with +`Item size to update has exceeded the maximum allowed size`. + +| Area | DynamoDB | ExtendDB | +|------|----------|------| +| `UpdateItem` item-size accounting | In 31 of 33 commercial regions a standalone `UpdateItem` is sized by the statement (the rule above). At a one-byte key the largest finished item it can produce is 409,581 bytes. Two regions (`eu-south-2`, `me-central-1`) size an `UpdateItem` on the finished item, like a `PutItem`. | PostgreSQL and SQLite follow the 31-region statement rule. A transacted `Update` is sized on the finished item on both, as on the service. Cassandra sizes every update on the finished item against a fixed 409,600 bytes; MongoDB applies no item-size limit on updates. | + ## Capacity and Throttling | Area | DynamoDB | ExtendDB | diff --git a/docs/dynamodb-limits.md b/docs/dynamodb-limits.md index 503836958..55d5de06b 100755 --- a/docs/dynamodb-limits.md +++ b/docs/dynamodb-limits.md @@ -1,6 +1,6 @@ # DynamoDB Limits Enforcement Status -Last updated: 2026-04-23 (P42) +Last updated: 2026-10-19 Source: [AWS DynamoDB Service Quotas](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ServiceQuotas.html) @@ -38,7 +38,7 @@ Source: [AWS DynamoDB Service Quotas](https://docs.aws.amazon.com/amazondynamodb | Limit | DynamoDB Value | Status | Notes | |-------|---------------|--------|-------------------------------------------------------------------------------| -| Maximum item size | 400 KB (409,600 bytes) | Enforced | `LimitsConfig::max_item_size_bytes`, validated on PutItem and post-UpdateItem | +| Maximum item size | 400 KB (409,600 bytes) | Enforced | `LimitsConfig::max_item_size_bytes`. The engine enforces it on PutItem, BatchWriteItem, the transaction Put path and ImportTable; the PostgreSQL and SQLite storage layers enforce it on UpdateItem and the transaction Update path. Cassandra applies a fixed 409,600 to the finished item of an update with the put wording; MongoDB applies no limit on updates. The sizing calculation (item bytes, number cost, and the statement-based UpdateItem rule) lives in [differences-from-dynamodb.md](differences-from-dynamodb.md#item-size). | | Partition key size | 1–2,048 bytes | Enforced | `validate_key_sizes`, `LimitsConfig::max_partition_key_size_bytes` | | Sort key size | 1–1,024 bytes | Enforced | `validate_key_sizes`, `LimitsConfig::max_sort_key_size_bytes` | | Attribute name size | 1–64 KB (65,535 bytes) | Partial | `validate_attribute_name_sizes`, `LimitsConfig::max_attribute_name_bytes`; top-level names and nested map keys at every depth, on PutItem, BatchWriteItem.PutRequest, TransactWriteItems.Put, and ImportTable. Not enforced on UpdateItem or TransactWriteItems.Update at any level. | diff --git a/tests/test_item_size_at_limit.py b/tests/test_item_size_at_limit.py new file mode 100644 index 000000000..ec71c2838 --- /dev/null +++ b/tests/test_item_size_at_limit.py @@ -0,0 +1,431 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Item size at the 409,600-byte limit: number byte costs, the UpdateItem rule, +the transacted Update rule, and the empty-binary index key refusal on UpdateItem. + +Every figure here was measured against Amazon DynamoDB (us-east-1, 2026-10-19) +by an accept/refuse pair either side of the limit: an item of exactly N bytes is +accepted and one of N+1 is refused, so the pair measures a single attribute's +cost to the byte with two requests. + +Numbers: ``1 + ceil(integer significant digits / 2) + ceil(fraction significant +digits / 2)``, plus one for a sign. The two halves round up separately, so +``1.5`` costs 3 where ``15`` costs 2 and ``3.14159`` costs 5 where ``123456`` +costs 4. + +UpdateItem is sized by the statement, not the finished item: a base of 3; 18 +per SET or ADD clause and nothing per REMOVE or DELETE; a skeleton item holding +only the written paths (each map entry its name plus one, each map or list 3, +each list element one more, untouched siblings left out), where a SET or ADD +leaf costs the clause's value placeholders and a REMOVE or DELETE leaf costs +nothing; one more for a clause through a list index; and inside a SET operand +26 plus the name length per function call, 34 per "+", 37 per "-", and for a +document path operand 15 plus the name for its first element, 14 plus the name +for a later attribute, 15 for an index, with the attribute's stored value not +charged. The legacy AttributeUpdates form costs 3 plus the name and finished +value of each attribute it names that is present afterwards. The key and +untouched attributes are left out, so at a one-byte key the largest finished +item a plain SET can produce is 409,581 bytes, and a longer key or an +untouched attribute buys the headroom back, capped at 409,600, which the +finished item never exceeds. The service applies this rule in 31 of 33 +commercial regions, us-east-1 among them. A transacted Update is sized on the +finished item alone: exactly 409,600 is stored, one byte over is a +cancellation. Both update paths word the refusal "Item size to update has +exceeded the maximum allowed size". + +Dual-target against Amazon DynamoDB and extenddb. +""" + +from __future__ import annotations + +import os + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +# The statement-sized UpdateItem rule and the empty-binary index-key refusal are +# enforced by the PostgreSQL and SQLite storage layers. The Cassandra layer +# sizes every update on the finished item against a fixed 409,600 bytes and the +# MongoDB layer applies no item-size limit on updates, both documented in +# docs/differences-from-dynamodb.md, so the update-specific classes below are +# skipped on those two backends. Number sizing lives in shared core and is +# asserted on every backend. When the backend is unset (a run against live +# DynamoDB through conf-run/live), every class runs. +on_default_backends_only = pytest.mark.skipif( + os.environ.get("EXTENDDB_TEST_BACKEND", "") in {"mongodb", "cassandra"}, + reason="UpdateItem is sized on the finished item (Cassandra) or unbounded (MongoDB); " + "see docs/differences-from-dynamodb.md", +) + +MAX = 409_600 +# A one-byte partition key "k": pk(2) + value(1) bytes the statement never writes. +KEY_K_BYTES = 3 +PUT_WORDING = "Item size has exceeded the maximum allowed size" +UPDATE_WORDING = "Item size to update has exceeded the maximum allowed size" +UPDATE_BASE = 3 +SET_COST = 19 # 18 for the clause plus the written attribute's map entry byte +REMOVE_COST = 2 # nothing for the clause plus the attribute's name and entry byte + + +@pytest.fixture(scope="class") +def table(dynamodb_client): + with scoped_table(dynamodb_client) as name: + yield name + + +@pytest.fixture(scope="class") +def gsi_table(dynamodb_client): + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "bidx", "AttributeType": "B"}, + ], + GlobalSecondaryIndexes=[ + { + "IndexName": "by-bidx", + "KeySchema": [{"AttributeName": "bidx", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "KEYS_ONLY"}, + } + ], + ) as name: + yield name + + +def _err(exc_info) -> dict: + return exc_info.value.response["Error"] + + +class TestNumberByteCost: + """A number's byte cost, measured by the accept/refuse pair at the limit.""" + + # (literal, bytes) measured on the service. + CASES = [ + ("0", 1), ("1", 2), ("15", 2), ("100", 2), ("1E125", 2), ("0.15", 2), + ("1.5", 3), ("1.2", 3), ("1.200", 3), ("1010", 3), ("-42", 3), + ("1.234", 4), ("100.5", 4), ("123456", 4), ("3.14159", 5), + ] + + @pytest.mark.parametrize(("literal", "cost"), CASES) + def test_number_costs_measured_bytes(self, dynamodb_client, table, literal, cost): + pk = "n-" + literal.replace(".", "_").replace("-", "m") + # pk(2) + pk value + n(1) + cost + p(1) + pad == MAX + pad = MAX - (2 + len(pk) + 1 + cost + 1) + dynamodb_client.put_item( + TableName=table, + Item={"pk": {"S": pk}, "n": {"N": literal}, "p": {"S": "x" * pad}}, + ) + with pytest.raises(ClientError) as exc_info: + dynamodb_client.put_item( + TableName=table, + Item={"pk": {"S": pk}, "n": {"N": literal}, "p": {"S": "x" * (pad + 1)}}, + ) + assert _err(exc_info)["Code"] == "ValidationException" + assert _err(exc_info)["Message"] == PUT_WORDING + + +@on_default_backends_only +class TestUpdateItemSizeRule: + """UpdateItem is sized by the statement, not the finished item.""" + + def _set(self, client, table, pk, finished_bytes, extra_expr="", extra_values=None): + pad = finished_bytes - (2 + len(pk) + 1) # pk name, pk value, "p" name + values = {":v": {"S": "x" * pad}} + values.update(extra_values or {}) + client.update_item( + TableName=table, + Key={"pk": {"S": pk}}, + UpdateExpression="SET p = :v" + extra_expr, + ExpressionAttributeValues=values, + ) + + def test_one_set_at_a_one_byte_key_caps_below_the_limit(self, dynamodb_client, table): + ceiling = MAX + KEY_K_BYTES - UPDATE_BASE - SET_COST # 409,581 + self._set(dynamodb_client, table, "k", ceiling) + dynamodb_client.delete_item(TableName=table, Key={"pk": {"S": "k"}}) + with pytest.raises(ClientError) as exc_info: + self._set(dynamodb_client, table, "k", ceiling + 1) + assert _err(exc_info)["Code"] == "ValidationException" + assert _err(exc_info)["Message"] == UPDATE_WORDING + + def test_a_second_set_costs_nineteen_bytes(self, dynamodb_client, table): + ceiling = MAX + KEY_K_BYTES - UPDATE_BASE - 2 * SET_COST # 409,562 + extra = {":w": {"S": "y"}} + # finished = pk(2)+1 + p(1)+pad + q(1)+1; the SET q costs 19, writes 2. + pad = ceiling - (2 + 1 + 1 + 2) + dynamodb_client.update_item( + TableName=table, Key={"pk": {"S": "k"}}, + UpdateExpression="SET p = :v, q = :w", + ExpressionAttributeValues={":v": {"S": "x" * pad}, **extra}, + ) + dynamodb_client.delete_item(TableName=table, Key={"pk": {"S": "k"}}) + with pytest.raises(ClientError) as exc_info: + dynamodb_client.update_item( + TableName=table, Key={"pk": {"S": "k"}}, + UpdateExpression="SET p = :v, q = :w", + ExpressionAttributeValues={":v": {"S": "x" * (pad + 1)}, **extra}, + ) + assert _err(exc_info)["Message"] == UPDATE_WORDING + + def test_a_remove_costs_two_bytes(self, dynamodb_client, table): + ceiling = MAX + KEY_K_BYTES - UPDATE_BASE - SET_COST - REMOVE_COST # 409,579 + pad = ceiling - (2 + 1 + 1) + for size, ok in ((pad, True), (pad + 1, False)): + dynamodb_client.put_item(TableName=table, Item={"pk": {"S": "k"}, "r": {"S": "z"}}) + call = lambda: dynamodb_client.update_item( # noqa: E731 + TableName=table, Key={"pk": {"S": "k"}}, + UpdateExpression="SET p = :v REMOVE r", + ExpressionAttributeValues={":v": {"S": "x" * size}}, + ) + if ok: + call() + else: + with pytest.raises(ClientError) as exc_info: + call() + assert _err(exc_info)["Message"] == UPDATE_WORDING + + def test_an_untouched_attribute_buys_headroom_back_to_the_limit(self, dynamodb_client, table): + # 100-byte attribute u is left alone; finished item may reach 409,600 exactly. + for finished, ok in ((MAX, True), (MAX + 1, False)): + dynamodb_client.put_item(TableName=table, Item={"pk": {"S": "k"}, "u": {"S": "u" * 100}}) + pad = finished - (2 + 1 + 1 + 101) + call = lambda: dynamodb_client.update_item( # noqa: E731 + TableName=table, Key={"pk": {"S": "k"}}, + UpdateExpression="SET p = :v", + ExpressionAttributeValues={":v": {"S": "x" * pad}}, + ) + if ok: + call() + else: + with pytest.raises(ClientError) as exc_info: + call() + assert _err(exc_info)["Message"] == UPDATE_WORDING + + def test_a_long_key_buys_headroom_back_to_the_limit(self, dynamodb_client, table): + pk = "K" * 100 + self._set(dynamodb_client, table, pk, MAX) + dynamodb_client.delete_item(TableName=table, Key={"pk": {"S": pk}}) + with pytest.raises(ClientError) as exc_info: + self._set(dynamodb_client, table, pk, MAX + 1) + assert _err(exc_info)["Message"] == UPDATE_WORDING + + def test_an_alias_is_charged_as_the_attribute_it_names(self, dynamodb_client, table): + # SET #twelvechars = :v writes attribute "p"; the alias costs nothing extra. + ceiling = MAX + KEY_K_BYTES - UPDATE_BASE - SET_COST # 409,581 + for finished, ok in ((ceiling, True), (ceiling + 1, False)): + dynamodb_client.delete_item(TableName=table, Key={"pk": {"S": "k"}}) + pad = finished - (2 + 1 + 1) + call = lambda: dynamodb_client.update_item( # noqa: E731 + TableName=table, Key={"pk": {"S": "k"}}, + UpdateExpression="SET #twelvechars = :v", + ExpressionAttributeNames={"#twelvechars": "p"}, + ExpressionAttributeValues={":v": {"S": "x" * pad}}, + ) + if ok: + call() + else: + with pytest.raises(ClientError) as exc_info: + call() + assert _err(exc_info)["Message"] == UPDATE_WORDING + + def test_a_set_through_a_list_index_costs_one_byte_more(self, dynamodb_client, table): + # Measured: SET d[0] = :v is charged 20 where SET d = :whole is charged 19. + # finished = pk(2)+1 + d(1) + 3 + (pad + 1) + ceiling = MAX + KEY_K_BYTES - UPDATE_BASE - SET_COST - 1 # 409,580 + for finished, ok in ((ceiling, True), (ceiling + 1, False)): + dynamodb_client.put_item(TableName=table, Item={"pk": {"S": "k"}, "d": {"L": [{"S": ""}]}}) + pad = finished - (2 + 1 + 1 + 3 + 1) + call = lambda: dynamodb_client.update_item( # noqa: E731 + TableName=table, Key={"pk": {"S": "k"}}, + UpdateExpression="SET d[0] = :v", + ExpressionAttributeValues={":v": {"S": "x" * pad}}, + ) + if ok: + call() + else: + with pytest.raises(ClientError) as exc_info: + call() + assert _err(exc_info)["Message"] == UPDATE_WORDING + + def test_a_nested_path_set_is_charged_like_a_plain_set(self, dynamodb_client, table): + ceiling = MAX + KEY_K_BYTES - UPDATE_BASE - SET_COST # 409,581 + for finished, ok in ((ceiling, True), (ceiling + 1, False)): + dynamodb_client.put_item(TableName=table, Item={"pk": {"S": "k"}, "m": {"M": {"a": {"S": ""}}}}) + # finished = pk(2)+1 + m(1) + 3 + (a(1) + pad + 1) + pad = finished - (2 + 1 + 1 + 3 + 1 + 1) + call = lambda: dynamodb_client.update_item( # noqa: E731 + TableName=table, Key={"pk": {"S": "k"}}, + UpdateExpression="SET m.a = :v", + ExpressionAttributeValues={":v": {"S": "x" * pad}}, + ) + if ok: + call() + else: + with pytest.raises(ClientError) as exc_info: + call() + assert _err(exc_info)["Message"] == UPDATE_WORDING + + +def _s(n: int) -> dict: + return {"S": "x" * n} + + +@on_default_backends_only +class TestUpdateStatementShapes: + """The largest accepted padding at a one-byte key, per statement shape. + + Each row is (name, seed item or None, builder) where the builder returns the + UpdateItem arguments for a given padding, and the figure is the largest + padding the service accepted on 2026-10-19; the padding one larger is refused + with the update wording. The seed is written with PutItem before each + attempt. Shapes whose figure is the finished-item cap rather than the + statement figure are marked so in the name. + """ + + SHAPES = [ + ("plain_set", None, 409_577, + lambda n: {"UpdateExpression": "SET p = :v", "ExpressionAttributeValues": {":v": _s(n)}}), + ("two_sets_different_attributes", None, 409_556, + lambda n: {"UpdateExpression": "SET p = :v, q = :w", "ExpressionAttributeValues": {":v": _s(n), ":w": {"S": "y"}}}), + ("two_clauses_same_map_share_the_container", {"m": {"M": {}}}, 409_551, + lambda n: {"UpdateExpression": "SET m.a = :v, m.b = :w", "ExpressionAttributeValues": {":v": _s(n), ":w": {"S": "y"}}}), + ("three_clauses_same_map", {"m": {"M": {}}}, 409_530, + lambda n: {"UpdateExpression": "SET m.a = :v, m.b = :w, m.c = :x", "ExpressionAttributeValues": {":v": _s(n), ":w": {"S": "y"}, ":x": {"S": "y"}}}), + ("nested_two_deep", {"d": {"M": {"a": {"M": {}}}}}, 409_567, + lambda n: {"UpdateExpression": "SET d.a.b = :v", "ExpressionAttributeValues": {":v": _s(n)}}), + ("map_operand", None, 409_572, + lambda n: {"UpdateExpression": "SET p = :v", "ExpressionAttributeValues": {":v": {"M": {"a": _s(n)}}}}), + ("list_index_siblings_not_charged_cap_binds", {"p": {"L": [{"S": "s" * 50}, {"S": "seed"}]}}, 409_541, + lambda n: {"UpdateExpression": "SET p[1] = :v", "ExpressionAttributeValues": {":v": _s(n)}}), + ("if_not_exists_charges_the_operand", None, 409_522, + lambda n: {"UpdateExpression": "SET p = if_not_exists(p, :v)", "ExpressionAttributeValues": {":v": _s(n)}}), + ("if_not_exists_charges_the_operand_when_present", {"p": {"S": "q"}}, 409_522, + lambda n: {"UpdateExpression": "SET p = if_not_exists(p, :v)", "ExpressionAttributeValues": {":v": _s(n)}}), + ("if_not_exists_ten_byte_names", None, 409_504, + lambda n: {"UpdateExpression": "SET pppppppppp = if_not_exists(pppppppppp, :v)", "ExpressionAttributeValues": {":v": _s(n)}}), + ("if_not_exists_other_inner_path", None, 409_513, + lambda n: {"UpdateExpression": "SET p = if_not_exists(qqqqqqqqqq, :v)", "ExpressionAttributeValues": {":v": _s(n)}}), + ("if_not_exists_nested_inner_path", {"d": {"M": {}}}, 409_504, + lambda n: {"UpdateExpression": "SET p = if_not_exists(d.leaf, :v)", "ExpressionAttributeValues": {":v": _s(n)}}), + ("if_not_exists_nested_target", {"d": {"M": {}}}, 409_496, + lambda n: {"UpdateExpression": "SET d.leaf = if_not_exists(d.leaf, :v)", "ExpressionAttributeValues": {":v": _s(n)}}), + ("if_not_exists_indexed_inner_path", {"d": {"L": [{"S": "q"}]}}, 409_507, + lambda n: {"UpdateExpression": "SET p = if_not_exists(d[0], :v)", "ExpressionAttributeValues": {":v": _s(n)}}), + ("list_append_two_values", None, 409_531, + lambda n: {"UpdateExpression": "SET p = list_append(:v, :w)", "ExpressionAttributeValues": {":v": {"L": [_s(n)]}, ":w": {"L": [{"S": "y"}]}}}), + ("list_append_path_value_not_charged", {"p": {"L": [{"S": "q"}]}}, 409_520, + lambda n: {"UpdateExpression": "SET p = list_append(p, :v)", "ExpressionAttributeValues": {":v": {"L": [_s(n)]}}}), + ("list_append_to_big_list_cap_binds", "big_list", 409_590, + lambda n: {"UpdateExpression": "SET p = list_append(p, :v)", "ExpressionAttributeValues": {":v": {"L": [{"S": "y"}]}}}), + ("nested_functions", None, 409_478, + lambda n: {"UpdateExpression": "SET p = list_append(if_not_exists(p, :e), :v)", "ExpressionAttributeValues": {":e": {"L": []}, ":v": {"L": [_s(n)]}}}), + ("addition_with_path_operand", {"n": {"N": "1"}}, 409_505, + lambda n: {"UpdateExpression": "SET p = :v, n = n + :w", "ExpressionAttributeValues": {":v": _s(n), ":w": {"N": "1"}}}), + ("addition_with_two_values", None, 409_519, + lambda n: {"UpdateExpression": "SET p = :v, n = :w + :x", "ExpressionAttributeValues": {":v": _s(n), ":w": {"N": "1"}, ":x": {"N": "1"}}}), + ("subtraction_with_two_values", None, 409_516, + lambda n: {"UpdateExpression": "SET p = :v, n = :w - :x", "ExpressionAttributeValues": {":v": _s(n), ":w": {"N": "1"}, ":x": {"N": "1"}}}), + ("add_number", None, 409_555, + lambda n: {"UpdateExpression": "SET p = :v ADD n :w", "ExpressionAttributeValues": {":v": _s(n), ":w": {"N": "1"}}}), + ("add_to_big_set_cap_binds", "big_set", 409_595, + lambda n: {"UpdateExpression": "ADD s :v", "ExpressionAttributeValues": {":v": {"SS": ["y"]}}}), + ("remove_nested_path", {"d": {"M": {"leaf": {"S": "y"}}}}, 409_567, + lambda n: {"UpdateExpression": "SET p = :v REMOVE d.leaf", "ExpressionAttributeValues": {":v": _s(n)}}), + ("remove_through_list_index", {"d": {"L": [{"S": "y"}]}}, 409_570, + lambda n: {"UpdateExpression": "SET p = :v REMOVE d[0]", "ExpressionAttributeValues": {":v": _s(n)}}), + ("delete_operand_not_charged", {"s": {"SS": ["a", "b"]}}, 409_575, + lambda n: {"UpdateExpression": "SET p = :v DELETE s :w", "ExpressionAttributeValues": {":v": _s(n), ":w": {"SS": ["a"]}}}), + ("legacy_put", None, 409_596, + lambda n: {"AttributeUpdates": {"p": {"Value": _s(n), "Action": "PUT"}}}), + ("legacy_two_puts", None, 409_594, + lambda n: {"AttributeUpdates": {"p": {"Value": _s(n), "Action": "PUT"}, "q": {"Value": {"S": "y"}, "Action": "PUT"}}}), + ("legacy_put_ten_byte_name", None, 409_587, + lambda n: {"AttributeUpdates": {"pppppppppp": {"Value": _s(n), "Action": "PUT"}}}), + ("legacy_put_map", None, 409_591, + lambda n: {"AttributeUpdates": {"p": {"Value": {"M": {"a": _s(n)}}, "Action": "PUT"}}}), + ("legacy_put_and_delete_charges_the_finished_set", {"s": {"SS": ["a", "b"]}}, 409_594, + lambda n: {"AttributeUpdates": {"p": {"Value": _s(n), "Action": "PUT"}, "s": {"Value": {"SS": ["a"]}, "Action": "DELETE"}}}), + ("legacy_put_and_remove_charges_nothing_for_the_removed", {"s": {"SS": ["a", "b"]}}, 409_596, + lambda n: {"AttributeUpdates": {"p": {"Value": _s(n), "Action": "PUT"}, "s": {"Action": "DELETE"}}}), + ] + + @staticmethod + def _seed(seed, n): + if seed == "big_list": + return {"p": {"L": [_s(n)]}} + if seed == "big_set": + return {"s": {"SS": ["x" * n]}} + return seed + + @pytest.mark.parametrize("name,seed,ceiling,build", SHAPES, ids=[r[0] for r in SHAPES]) + def test_largest_accepted_padding(self, dynamodb_client, table, name, seed, ceiling, build): + key = {"pk": {"S": "k"}} + for n, ok in ((ceiling, True), (ceiling + 1, False)): + dynamodb_client.delete_item(TableName=table, Key=key) + seeded = self._seed(seed, n) + try: + if seeded: + dynamodb_client.put_item(TableName=table, Item={"pk": {"S": "k"}, **seeded}) + dynamodb_client.update_item(TableName=table, Key=key, **build(n)) + accepted = True + except ClientError as exc: + assert exc.response["Error"]["Code"] == "ValidationException" + assert "size" in exc.response["Error"]["Message"], exc.response["Error"]["Message"] + accepted = False + assert accepted is ok, f"{name}: padding {n} {'accepted' if accepted else 'refused'}" + + +@on_default_backends_only +class TestTransactedUpdateSizeRule: + """A transacted Update is sized on the finished item and cancels with the update wording.""" + + def _txn(self, client, table, finished): + pad = finished - (2 + 1 + 1) + client.transact_write_items(TransactItems=[{ + "Update": { + "TableName": table, "Key": {"pk": {"S": "t"}}, + "UpdateExpression": "SET p = :v", + "ExpressionAttributeValues": {":v": {"S": "x" * pad}}, + } + }]) + + def test_exactly_the_limit_is_stored(self, dynamodb_client, table): + dynamodb_client.delete_item(TableName=table, Key={"pk": {"S": "t"}}) + self._txn(dynamodb_client, table, MAX) + got = dynamodb_client.get_item(TableName=table, Key={"pk": {"S": "t"}}, ConsistentRead=True) + assert len(got["Item"]["p"]["S"]) == MAX - 4 + + def test_one_byte_over_cancels_with_the_update_wording(self, dynamodb_client, table): + dynamodb_client.delete_item(TableName=table, Key={"pk": {"S": "t"}}) + with pytest.raises(ClientError) as exc_info: + self._txn(dynamodb_client, table, MAX + 1) + resp = exc_info.value.response + assert resp["Error"]["Code"] == "TransactionCanceledException" + reasons = resp["CancellationReasons"] + assert reasons[0]["Code"] == "ValidationError" + assert reasons[0]["Message"] == UPDATE_WORDING + + +@on_default_backends_only +class TestEmptyBinaryIndexKeyOnUpdate: + """UpdateItem setting a GSI key to an empty binary is refused with the update-form message.""" + + def test_update_refuses_empty_binary_index_key(self, dynamodb_client, gsi_table): + dynamodb_client.put_item(TableName=gsi_table, Item={"pk": {"S": "eb"}, "v": {"S": "1"}}) + with pytest.raises(ClientError) as exc_info: + dynamodb_client.update_item( + TableName=gsi_table, Key={"pk": {"S": "eb"}}, + UpdateExpression="SET bidx = :b", + ExpressionAttributeValues={":b": {"B": b""}}, + ) + assert _err(exc_info)["Code"] == "ValidationException" + assert _err(exc_info)["Message"] == ( + "One or more parameter values are not valid. The update expression attempted to " + "update a secondary index key to a value that is not supported. The AttributeValue " + "for a key attribute cannot contain an empty binary value." + ) + got = dynamodb_client.get_item(TableName=gsi_table, Key={"pk": {"S": "eb"}}, ConsistentRead=True) + assert "bidx" not in got["Item"]