From 20621876fe84c867895d44c3ff6d08f80e5bee1d Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Sun, 25 Oct 2026 09:34:39 +0000 Subject: [PATCH 1/3] fix(scan): check the numbers in ExpressionAttributeValues at request time Scan did not check that the N values in ExpressionAttributeValues are numbers, so a filter on {"N": "bob"}, a nested one in an L, M or NS, an overflow, an underflow, a 39 digit value or an empty N answered 200 with Count 0. The service refuses each with the same ValidationException text Query uses, whether or not an expression reads the value. Move the early Scan checks to a request-time function on the raw body, in the order measured on 2026-10-25: the Limit, Segment and TotalSegments ranges (one aggregated message, "Value at 'Limit'" with no value echoed), the expression and non-expression parameter mix, the ReturnConsumedCapacity enum, every number in ExpressionAttributeValues, then the Select enum. Both enums invalid now answers the ReturnConsumedCapacity error alone, as the service does; the earlier aggregation of the two no longer matched it. The Segment pairing checks stay in the handler after the table lookup. The two Scan tests that asserted the old segment range text now assert the measured text. --- crates/engine/src/lib.rs | 3 +- crates/engine/src/query_key_values.rs | 4 +- crates/engine/src/scan.rs | 88 +------ crates/engine/src/scan_request.rs | 133 ++++++++++ crates/engine/src/scan_request/tests.rs | 329 ++++++++++++++++++++++++ tests/test_query_scan.py | 20 +- 6 files changed, 488 insertions(+), 89 deletions(-) create mode 100644 crates/engine/src/scan_request.rs create mode 100644 crates/engine/src/scan_request/tests.rs diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 2cb47da7f..a8ab1d5fc 100755 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -32,6 +32,7 @@ pub mod query_key_values; mod read_helpers; mod request_limits; mod scan; +pub mod scan_request; mod search_vectors; pub mod stream_capture; mod streams; @@ -217,7 +218,7 @@ pub(crate) const RETURN_VALUES_MEMBERS: &[&str] = /// Pre-validate enum fields in a JSON body and return a combined error if /// multiple are invalid, in the order listed. Whether an operation aggregates /// its enum violations or stops at the first is per operation on the service -/// (PutItem/DeleteItem/Scan aggregate; UpdateItem and Query stop at the +/// (PutItem and DeleteItem aggregate; UpdateItem, Query and Scan stop at the /// first), so callers choose by passing one list or several sequential calls. pub(crate) fn validate_enum_fields( body: &serde_json::Value, diff --git a/crates/engine/src/query_key_values.rs b/crates/engine/src/query_key_values.rs index 54886bf96..42d10ee79 100644 --- a/crates/engine/src/query_key_values.rs +++ b/crates/engine/src/query_key_values.rs @@ -181,12 +181,12 @@ fn query_request_checks(body: &Value) -> Result<(), DynamoDbError> { /// Whether `field` is in the body with a value other than `null`, as a typed /// `Option` field would deserialize to `Some`. -fn present(body: &Value, field: &str) -> bool { +pub(crate) fn present(body: &Value, field: &str) -> bool { body.get(field).is_some_and(|v| !v.is_null()) } /// Whether `field` is a non-empty JSON object or array in the body. -fn non_empty(body: &Value, field: &str) -> bool { +pub(crate) fn non_empty(body: &Value, field: &str) -> bool { match body.get(field) { Some(Value::Object(map)) => !map.is_empty(), Some(Value::Array(items)) => !items.is_empty(), diff --git a/crates/engine/src/scan.rs b/crates/engine/src/scan.rs index 1c1de2274..2a2214d84 100755 --- a/crates/engine/src/scan.rs +++ b/crates/engine/src/scan.rs @@ -41,58 +41,19 @@ pub async fn handle_scan( body: Value, ctx: &OperationContext, ) -> Result { - // Pre-scanned before typed deserialization. Scan AGGREGATES its invalid - // enums with Select's clause ahead of ReturnConsumedCapacity's, the - // opposite of Query on both counts (Query stops at the first, RCC-first). - // Measured 2026-08-24 (us-east-1): both invalid answers "2 validation - // errors detected: ... 'select' ...; ... 'returnConsumedCapacity' ...". - crate::validate_enum_fields( - &body, - &[ - crate::EnumField { - json_name: "Select", - valid: &[ - "SPECIFIC_ATTRIBUTES", - "COUNT", - "ALL_ATTRIBUTES", - "ALL_PROJECTED_ATTRIBUTES", - ], - clause: crate::EnumClause::Named("select"), - }, - crate::EnumField { - json_name: "ReturnConsumedCapacity", - valid: &["INDEXES", "TOTAL", "NONE"], - clause: crate::EnumClause::Named("returnConsumedCapacity"), - }, - ], - )?; + // Request-time checks on the raw body, in the service's order: the Limit, + // Segment and TotalSegments ranges, the expression and non-expression + // parameter mix, the ReturnConsumedCapacity enum, every number in + // ExpressionAttributeValues, the Select enum. They run before typed + // deserialization so a field that does not deserialize does not stop the + // ones before it; see `scan_request`. + crate::scan_request::validate_scan_request_body(&body)?; let input: ScanInput = serde_json::from_value(body).map_err(crate::deserialize_error)?; // Validate Filter/Projection expressions before the existence - // check; index resolution, Segment/Limit/Select and key checks stay after. - - // --- Legacy vs expression mutual exclusivity checks --- - let has_fe = input.filter_expression.is_some(); - let has_sf = input.scan_filter.as_ref().is_some_and(|m| !m.is_empty()); - let has_pe = input.projection_expression.is_some(); - let has_atg = input - .attributes_to_get - .as_ref() - .is_some_and(|a| !a.is_empty()); + // check; index resolution, Segment pairing, Select and key checks stay after. let has_cond_op = input.conditional_operator.is_some(); - extenddb_core::validation::validate_no_expression_param_mixing( - &[ - ("AttributesToGet", has_atg), - ("ScanFilter", has_sf), - ("ConditionalOperator", has_cond_op), - ], - &[ - ("ProjectionExpression", has_pe), - ("FilterExpression", has_fe), - ], - )?; - let maps = build_expression_maps( input.expression_attribute_names.as_ref(), input.expression_attribute_values.as_ref(), @@ -299,7 +260,9 @@ pub async fn handle_scan( )?; } - // Validate Segment/TotalSegments — DynamoDB returns different messages per direction + // Validate Segment/TotalSegments pairing. The range of each field is + // checked at request time in validate_scan_request_body; the service + // reports a missing partner with a message per direction. match (input.segment, input.total_segments) { (Some(_), None) => { return Err(DynamoDbError::ValidationException( @@ -314,26 +277,6 @@ pub async fn handle_scan( )); } (Some(seg), Some(total)) => { - if total < 1 { - return Err(DynamoDbError::ValidationException( - "The parameter TotalSegments should be greater than or equal to 1".to_owned(), - )); - } - if total > 1_000_000 { - return Err(DynamoDbError::ValidationException(format!( - "1 validation error detected: Value '{total}' at 'totalSegments' failed to satisfy constraint: Member must have value less than or equal to 1000000" - ))); - } - if seg < 0 { - return Err(DynamoDbError::ValidationException(format!( - "1 validation error detected: Value '{seg}' at 'segment' failed to satisfy constraint: Member must have value greater than or equal to 0" - ))); - } - if seg > 999_999 { - return Err(DynamoDbError::ValidationException(format!( - "1 validation error detected: Value '{seg}' at 'segment' failed to satisfy constraint: Member must have value less than or equal to 999999" - ))); - } if seg >= total { return Err(DynamoDbError::ValidationException(format!( "The Segment parameter is zero-based and must be less than parameter TotalSegments: \ @@ -344,14 +287,7 @@ pub async fn handle_scan( (None, None) => {} } - // Validate Limit >= 1 - if let Some(limit) = input.limit - && limit < 1 - { - return Err(DynamoDbError::ValidationException(format!( - "1 validation error detected: Value '{limit}' at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1" - ))); - } + // Limit >= 1 is checked at request time in validate_scan_request_body. // For index scans, build a key_info that reflects the index's key schema. let scan_key_info = if let Some(ref idx) = index_info { diff --git a/crates/engine/src/scan_request.rs b/crates/engine/src/scan_request.rs new file mode 100644 index 000000000..e1f89f410 --- /dev/null +++ b/crates/engine/src/scan_request.rs @@ -0,0 +1,133 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The `Scan` checks that run on the request body alone, before the body is +//! deserialized and before the table is looked up, in the service's order. +//! +//! Measured 2026-10-25 (us-east-1, each request twice): +//! +//! 1. the range constraints on `Limit` (at least 1), `Segment` (0 to 999999) +//! and `TotalSegments` (1 to 1000000), aggregated into one message in that +//! order with no value echoed: `2 validation errors detected: Value at +//! 'Limit' failed to satisfy constraint: Member must have value greater than +//! or equal to 1; Value at 'Segment' ...`. Each field is checked when +//! present, whether or not the other segment field is; +//! 2. the expression and non-expression parameter mix; +//! 3. the `ReturnConsumedCapacity` enum; +//! 4. every number in `ExpressionAttributeValues`, `NS` members and values +//! nested in `L` and `M` included, whether a `FilterExpression` reads the +//! value, nothing reads it, or the request carries no expression at all; +//! 5. the `Select` enum. Both enums invalid answers the `ReturnConsumedCapacity` +//! error alone, so the two are not aggregated. +//! +//! Each of these is reported ahead of the next one, ahead of the table lookup, +//! and ahead of the checks the handler makes after deserializing the body +//! (Segment without TotalSegments and the reverse, Segment not below +//! TotalSegments, the undefined placeholder, unused value, filter syntax, +//! IndexName, ConsistentRead, Select with ProjectionExpression and +//! ExclusiveStartKey checks). + +use std::collections::HashMap; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::types::AttributeValue; +use serde_json::Value; + +use crate::query_key_values::{non_empty, present, validate_expression_attribute_value_numbers}; + +/// The `Scan` checks that run on the raw request body, in the service's order: +/// the `Limit`, `Segment` and `TotalSegments` ranges, the expression and +/// non-expression parameter mix, the `ReturnConsumedCapacity` enum, every +/// number in `ExpressionAttributeValues`, then the `Select` enum. +/// +/// The checks read the raw body, so a field that does not deserialize (an +/// invalid `Select`, for one) does not stop the ones before it. A field of the +/// wrong JSON shape is left to the deserializer, which runs next. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` with the service's text. +pub fn validate_scan_request_body(body: &Value) -> Result<(), DynamoDbError> { + validate_ranges(body)?; + + extenddb_core::validation::validate_no_expression_param_mixing( + &[ + ("AttributesToGet", non_empty(body, "AttributesToGet")), + ("ScanFilter", non_empty(body, "ScanFilter")), + ("ConditionalOperator", present(body, "ConditionalOperator")), + ], + &[ + ( + "ProjectionExpression", + present(body, "ProjectionExpression"), + ), + ("FilterExpression", present(body, "FilterExpression")), + ], + )?; + + crate::validate_enum_fields( + body, + &[crate::EnumField { + json_name: "ReturnConsumedCapacity", + valid: &["INDEXES", "TOTAL", "NONE"], + clause: crate::EnumClause::Named("returnConsumedCapacity"), + }], + )?; + + if let Some(raw) = body.get("ExpressionAttributeValues") + && let Ok(values) = serde_json::from_value::>(raw.clone()) + { + validate_expression_attribute_value_numbers(&values)?; + } + + crate::validate_enum_fields( + body, + &[crate::EnumField { + json_name: "Select", + valid: &[ + "SPECIFIC_ATTRIBUTES", + "COUNT", + "ALL_ATTRIBUTES", + "ALL_PROJECTED_ATTRIBUTES", + ], + clause: crate::EnumClause::Named("select"), + }], + ) +} + +/// The range constraints on `Limit`, `Segment` and `TotalSegments`, aggregated +/// in that order as the service reports them. The field name is capitalised +/// and the value is not echoed, unlike the `select` and `segment` style the +/// other `1 validation error detected` messages use. +fn validate_ranges(body: &Value) -> Result<(), DynamoDbError> { + let mut errors: Vec = Vec::new(); + let mut check = |field: &str, below: i64, above: i64| { + let Some(value) = body.get(field).and_then(Value::as_i64) else { + return; + }; + if value < below { + errors.push(format!( + "Value at '{field}' failed to satisfy constraint: Member must have value greater than or equal to {below}" + )); + } else if value > above { + errors.push(format!( + "Value at '{field}' failed to satisfy constraint: Member must have value less than or equal to {above}" + )); + } + }; + check("Limit", 1, i64::MAX); + check("Segment", 0, 999_999); + check("TotalSegments", 1, 1_000_000); + if errors.is_empty() { + return Ok(()); + } + let count = errors.len(); + Err(DynamoDbError::ValidationException(format!( + "{count} validation error{} detected: {}", + if count == 1 { "" } else { "s" }, + errors.join("; ") + ))) +} + +#[cfg(test)] +mod tests; diff --git a/crates/engine/src/scan_request/tests.rs b/crates/engine/src/scan_request/tests.rs new file mode 100644 index 000000000..80af8ae17 --- /dev/null +++ b/crates/engine/src/scan_request/tests.rs @@ -0,0 +1,329 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Every expected text below was measured against the service on 2026-10-25 +//! (us-east-1, each request twice). + +use extenddb_core::error::DynamoDbError; +use serde_json::{Value, json}; + +use super::validate_scan_request_body; + +const NUMERIC_BOB: &str = "The parameter cannot be converted to a numeric value: bob"; +const LIMIT_RANGE: &str = "Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1"; +const RCC_ENUM: &str = "1 validation error detected: Value 'NOPE' at 'returnConsumedCapacity' failed to satisfy constraint: Member must satisfy enum value set: [INDEXES, TOTAL, NONE]"; +const SELECT_ENUM: &str = "1 validation error detected: Value 'NOPE' at 'select' failed to satisfy constraint: Member must satisfy enum value set: [SPECIFIC_ATTRIBUTES, COUNT, ALL_ATTRIBUTES, ALL_PROJECTED_ATTRIBUTES]"; + +fn message(result: Result<(), DynamoDbError>) -> Option { + match result { + Ok(()) => None, + Err(DynamoDbError::ValidationException(msg)) => Some(msg), + Err(other) => panic!("unexpected error {other:?}"), + } +} + +fn check(body: &Value) -> Option { + message(validate_scan_request_body(body)) +} + +/// A Scan whose filter reads `:n = {"N": "bob"}`. +fn bad_number() -> Value { + json!({ + "TableName": "t", + "FilterExpression": "a = :n", + "ExpressionAttributeValues": {":n": {"N": "bob"}}, + }) +} + +fn with(mut body: Value, field: &str, value: Value) -> Value { + body[field] = value; + body +} + +#[test] +fn numbers_are_checked_whether_or_not_an_expression_reads_them() { + assert_eq!(check(&bad_number()).as_deref(), Some(NUMERIC_BOB)); + // Nothing reads :n; the filter reads :p. + assert_eq!( + check(&json!({ + "TableName": "t", + "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, ":n": {"N": "bob"}}, + })) + .as_deref(), + Some(NUMERIC_BOB) + ); + // No expression at all, a ScanFilter only, a ProjectionExpression only. + for body in [ + json!({"TableName": "t", "ExpressionAttributeValues": {":n": {"N": "bob"}}}), + json!({ + "TableName": "t", + "ScanFilter": {"a": {"ComparisonOperator": "EQ", "AttributeValueList": [{"S": "x"}]}}, + "ExpressionAttributeValues": {":n": {"N": "bob"}}, + }), + json!({ + "TableName": "t", + "ProjectionExpression": "a", + "ExpressionAttributeValues": {":n": {"N": "bob"}}, + }), + ] { + assert_eq!(check(&body).as_deref(), Some(NUMERIC_BOB), "{body}"); + } +} + +#[test] +fn nested_numbers_and_the_other_number_faults_are_reported_with_the_service_text() { + let cases = [ + (json!({"L": [{"S": "a"}, {"N": "bob"}]}), NUMERIC_BOB), + (json!({"M": {"a": {"N": "bob"}}}), NUMERIC_BOB), + (json!({"NS": ["1", "bob"]}), NUMERIC_BOB), + ( + json!({"M": {"a": {"L": [{"M": {"b": {"N": "bob"}}}]}}}), + NUMERIC_BOB, + ), + ( + json!({"N": "1e200"}), + "Number overflow. Attempting to store a number with magnitude larger than supported range", + ), + ( + json!({"N": "1e-200"}), + "Number underflow. Attempting to store a number with magnitude smaller than supported range", + ), + ( + json!({"N": "1".repeat(39)}), + "Attempting to store more than 38 significant digits in a Number", + ), + ( + json!({"N": ""}), + "The parameter cannot be converted to a numeric value: ", + ), + ( + json!({"N": " 1"}), + "The parameter cannot be converted to a numeric value: 1", + ), + ]; + for (value, expected) in cases { + let body = json!({ + "TableName": "t", + "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, ":n": value}, + }); + assert_eq!(check(&body).as_deref(), Some(expected), "{body}"); + } + // A well-formed exponent passes. + assert!( + check(&json!({ + "TableName": "t", + "FilterExpression": "a = :n", + "ExpressionAttributeValues": {":n": {"N": "1.0e1"}}, + })) + .is_none() + ); +} + +#[test] +fn limit_segment_and_total_segments_ranges_come_first_and_aggregate() { + let base = bad_number(); + assert_eq!( + check(&with(base.clone(), "Limit", json!(0))).as_deref(), + Some(&*format!("1 validation error detected: {LIMIT_RANGE}")) + ); + assert_eq!( + check(&with(base.clone(), "Limit", json!(-1))).as_deref(), + Some(&*format!("1 validation error detected: {LIMIT_RANGE}")) + ); + // Each segment field is range checked when present, with or without the + // other one. + for (field, value, text) in [ + ( + "Segment", + -1, + "Value at 'Segment' failed to satisfy constraint: Member must have value greater than or equal to 0", + ), + ( + "Segment", + 1_000_000, + "Value at 'Segment' failed to satisfy constraint: Member must have value less than or equal to 999999", + ), + ( + "TotalSegments", + 0, + "Value at 'TotalSegments' failed to satisfy constraint: Member must have value greater than or equal to 1", + ), + ( + "TotalSegments", + 1_000_001, + "Value at 'TotalSegments' failed to satisfy constraint: Member must have value less than or equal to 1000000", + ), + ] { + assert_eq!( + check(&with(base.clone(), field, json!(value))).as_deref(), + Some(&*format!("1 validation error detected: {text}")), + "{field} {value}" + ); + let partner = if field == "Segment" { + ("TotalSegments", 2) + } else { + ("Segment", 0) + }; + let paired = with( + with(base.clone(), field, json!(value)), + partner.0, + json!(partner.1), + ); + assert_eq!( + check(&paired).as_deref(), + Some(&*format!("1 validation error detected: {text}")), + "{field} {value} with {}", + partner.0 + ); + } + // Limit, Segment, TotalSegments, in that order, one message. + let three = with( + with( + with(json!({"TableName": "t"}), "Limit", json!(0)), + "Segment", + json!(-1), + ), + "TotalSegments", + json!(1_000_001), + ); + assert_eq!( + check(&three).as_deref(), + Some( + "3 validation errors detected: Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1; Value at 'Segment' failed to satisfy constraint: Member must have value greater than or equal to 0; Value at 'TotalSegments' failed to satisfy constraint: Member must have value less than or equal to 1000000" + ) + ); + // Values at the bounds pass. + assert!( + check( + &json!({"TableName": "t", "Limit": 1, "Segment": 999_999, "TotalSegments": 1_000_000}) + ) + .is_none() + ); + // A range fault beats the parameter mix and the ReturnConsumedCapacity enum. + let mix = with( + with(base.clone(), "Segment", json!(-1)), + "ScanFilter", + json!({"a": {"ComparisonOperator": "EQ", "AttributeValueList": [{"S": "x"}]}}), + ); + assert!(check(&mix).is_some_and(|m| m.contains("Value at 'Segment'"))); + let rcc = with( + with(base, "Limit", json!(0)), + "ReturnConsumedCapacity", + json!("NOPE"), + ); + assert_eq!( + check(&rcc).as_deref(), + Some(&*format!("1 validation error detected: {LIMIT_RANGE}")) + ); +} + +#[test] +fn parameter_mix_beats_the_enum_and_the_numbers() { + let scan_filter = + json!({"a": {"ComparisonOperator": "EQ", "AttributeValueList": [{"S": "x"}]}}); + let mix = with(bad_number(), "ScanFilter", scan_filter.clone()); + assert_eq!( + check(&mix).as_deref(), + Some( + "Can not use both expression and non-expression parameters in the same request: \ + Non-expression parameters: {ScanFilter} Expression parameters: {FilterExpression}" + ) + ); + assert_eq!( + check(&with(mix, "ReturnConsumedCapacity", json!("NOPE"))).as_deref(), + Some( + "Can not use both expression and non-expression parameters in the same request: \ + Non-expression parameters: {ScanFilter} Expression parameters: {FilterExpression}" + ) + ); + let atg = with( + with(bad_number(), "AttributesToGet", json!(["a"])), + "ProjectionExpression", + json!("a"), + ); + assert_eq!( + check(&atg).as_deref(), + Some( + "Can not use both expression and non-expression parameters in the same request: \ + Non-expression parameters: {AttributesToGet} Expression parameters: {ProjectionExpression, FilterExpression}" + ) + ); + assert_eq!( + check(&with(bad_number(), "ConditionalOperator", json!("AND"))).as_deref(), + Some( + "Can not use both expression and non-expression parameters in the same request: \ + Non-expression parameters: {ConditionalOperator} Expression parameters: {FilterExpression}" + ) + ); + // ConditionalOperator with no expression does not join the mix; the + // numbers are reported. + assert_eq!( + check(&json!({ + "TableName": "t", + "ConditionalOperator": "AND", + "ExpressionAttributeValues": {":n": {"N": "bob"}}, + })) + .as_deref(), + Some(NUMERIC_BOB) + ); + // An empty ScanFilter or AttributesToGet does not join the mix. + assert!( + check(&json!({ + "TableName": "t", + "ScanFilter": {}, + "AttributesToGet": [], + "FilterExpression": "a = :n", + "ExpressionAttributeValues": {":n": {"N": "1"}}, + })) + .is_none() + ); +} + +#[test] +fn enum_checks_sit_on_either_side_of_the_numbers() { + assert_eq!( + check(&with(bad_number(), "ReturnConsumedCapacity", json!("NOPE"))).as_deref(), + Some(RCC_ENUM) + ); + assert_eq!( + check(&with(bad_number(), "Select", json!("NOPE"))).as_deref(), + Some(NUMERIC_BOB) + ); + // Both invalid: ReturnConsumedCapacity alone, not aggregated with Select. + let both = with( + with(json!({"TableName": "t"}), "Select", json!("NOPE")), + "ReturnConsumedCapacity", + json!("NOPE"), + ); + assert_eq!(check(&both).as_deref(), Some(RCC_ENUM)); + // With the numbers valid, an invalid Select is reported. + assert_eq!( + check(&json!({"TableName": "t", "Select": "NOPE"})).as_deref(), + Some(SELECT_ENUM) + ); + // A valid request passes. + assert!( + check(&json!({ + "TableName": "t", + "Select": "COUNT", + "ReturnConsumedCapacity": "TOTAL", + "Limit": 5, + "FilterExpression": "a = :n", + "ExpressionAttributeValues": {":n": {"N": "5"}}, + })) + .is_none() + ); +} + +#[test] +fn values_that_do_not_deserialize_are_left_to_the_deserializer() { + // A string Limit and a malformed ExpressionAttributeValues are not range + // or number faults here; the typed deserializer reports them next. + assert!(check(&json!({"TableName": "t", "Limit": "0"})).is_none()); + assert!( + check(&json!({"TableName": "t", "ExpressionAttributeValues": {":n": {"X": "bob"}}})) + .is_none() + ); +} diff --git a/tests/test_query_scan.py b/tests/test_query_scan.py index 44a82f4a2..04019c990 100755 --- a/tests/test_query_scan.py +++ b/tests/test_query_scan.py @@ -614,17 +614,17 @@ def test_scan_segment_at_max_accepted(self, dynamodb_client, scan_table): def test_scan_segment_one_over_max_rejected( self, dynamodb_client_no_validation, scan_table ): - """Segment > 999999 returns the Coral-format ValidationException.""" + """Segment > 999999 returns the range ValidationException (measured 2026-10-25).""" with pytest.raises(ClientError) as exc_info: dynamodb_client_no_validation.scan( TableName=scan_table, Segment=1_000_000, TotalSegments=1_000_000 ) err = exc_info.value.response["Error"] assert err["Code"] == "ValidationException" - msg = err["Message"] - assert "1 validation error detected" in msg - assert "'1000000' at 'segment'" in msg - assert "Member must have value less than or equal to 999999" in msg + assert err["Message"] == ( + "1 validation error detected: Value at 'Segment' failed to satisfy " + "constraint: Member must have value less than or equal to 999999" + ) def test_scan_total_segments_at_max_accepted(self, dynamodb_client, scan_table): """TotalSegments=1000000 with Segment=0 is at the documented bound.""" @@ -635,17 +635,17 @@ def test_scan_total_segments_at_max_accepted(self, dynamodb_client, scan_table): def test_scan_total_segments_one_over_max_rejected( self, dynamodb_client_no_validation, scan_table ): - """TotalSegments > 1000000 returns the Coral-format ValidationException.""" + """TotalSegments > 1000000 returns the range ValidationException (measured 2026-10-25).""" with pytest.raises(ClientError) as exc_info: dynamodb_client_no_validation.scan( TableName=scan_table, Segment=0, TotalSegments=1_000_001 ) err = exc_info.value.response["Error"] assert err["Code"] == "ValidationException" - msg = err["Message"] - assert "1 validation error detected" in msg - assert "'1000001' at 'totalSegments'" in msg - assert "Member must have value less than or equal to 1000000" in msg + assert err["Message"] == ( + "1 validation error detected: Value at 'TotalSegments' failed to satisfy " + "constraint: Member must have value less than or equal to 1000000" + ) def test_scan_all_projected_attributes_without_index_rejected( self, dynamodb_client, scan_table From 6f1e28421351cd9857070e1bfc59793207735de3 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Sun, 25 Oct 2026 09:36:47 +0000 Subject: [PATCH 2/3] test(scan): wire tests for the request-time number check and its position Every N value shape the service refuses on Scan (a word, a nested L, M or NS member, overflow, underflow, 39 digits, empty, a leading space), read by the filter or by nothing, and the order against every other early check: the Limit, Segment and TotalSegments ranges, the parameter mix and the ReturnConsumedCapacity enum ahead of it; the Select enum, the Segment pairing, the table lookup, IndexName, ConsistentRead, Select with ProjectionExpression, ExclusiveStartKey, undefined placeholder and filter syntax checks behind it. Expected texts measured on 2026-10-25. --- tests/test_scan_number_values.py | 330 +++++++++++++++++++++++++++++++ 1 file changed, 330 insertions(+) create mode 100644 tests/test_scan_number_values.py diff --git a/tests/test_scan_number_values.py b/tests/test_scan_number_values.py new file mode 100644 index 000000000..df354ef4d --- /dev/null +++ b/tests/test_scan_number_values.py @@ -0,0 +1,330 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Scan checks the numbers in ``ExpressionAttributeValues`` at request time. + +An ``N`` value that is not a number, an ``N`` nested in an ``L`` or ``M``, an +``NS`` member, an overflow, an underflow, a value with more than 38 +significant digits or an empty ``N`` is refused with ``ValidationException`` +before any row is read, whether a ``FilterExpression`` reads the value, +nothing reads it, or the request carries no expression at all. Every expected +answer below was measured against the service on 2026-10-25 (09:23 to 09:28 +UTC) with each request sent twice. + +The rows also pin where the check falls against the other early Scan checks: +after the ``Limit``, ``Segment`` and ``TotalSegments`` range constraints (one +aggregated message, ``Value at 'Limit'`` with no value echoed), the expression +and non-expression parameter mix and the ``ReturnConsumedCapacity`` enum; and +before the ``Select`` enum, the Segment pairing checks, the table lookup, the +IndexName, ConsistentRead, Select with ProjectionExpression, +ExclusiveStartKey, undefined placeholder, unused value and filter syntax +checks. + +Dual-target: runs against the service and against extenddb with the same +assertions. + +Run: + EXTENDDB_TEST_ENDPOINT=https://localhost:18443 \\ + EXTENDDB_ADMIN_USER=admin EXTENDDB_ADMIN_PASSWORD= \\ + pytest tests/test_scan_number_values.py -v +""" + +from __future__ import annotations + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +NUMERIC = "The parameter cannot be converted to a numeric value: " +OVERFLOW = ( + "Number overflow. Attempting to store a number with magnitude larger than " + "supported range" +) +UNDERFLOW = ( + "Number underflow. Attempting to store a number with magnitude smaller than " + "supported range" +) +TOO_MANY_DIGITS = "Attempting to store more than 38 significant digits in a Number" +LIMIT_RANGE = ( + "Value at 'Limit' failed to satisfy constraint: " + "Member must have value greater than or equal to 1" +) +RCC_ENUM = ( + "1 validation error detected: Value 'NOPE' at 'returnConsumedCapacity' failed " + "to satisfy constraint: Member must satisfy enum value set: [INDEXES, TOTAL, NONE]" +) +SELECT_ENUM = ( + "1 validation error detected: Value 'NOPE' at 'select' failed to satisfy " + "constraint: Member must satisfy enum value set: " + "[SPECIFIC_ATTRIBUTES, COUNT, ALL_ATTRIBUTES, ALL_PROJECTED_ATTRIBUTES]" +) + +S = {"AttributeType": "S"} +SX = {"S": "x"} +NB = {"N": "bob"} +NB_FILTER = {"FilterExpression": "a = :n", "ExpressionAttributeValues": {":n": NB}} +OK_FILTER = {"FilterExpression": "a = :n", "ExpressionAttributeValues": {":n": {"N": "1"}}} +SCAN_FILTER = {"a": {"ComparisonOperator": "EQ", "AttributeValueList": [SX]}} + + +def _err(exc_info) -> tuple[str, str]: + err = exc_info.value.response["Error"] + return err["Code"], err["Message"] + + +def _scan_error(client, **kwargs) -> tuple[str, str]: + with pytest.raises(ClientError) as exc_info: + client.scan(**kwargs) + return _err(exc_info) + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + """pk S, sk S; GSI gsi1 on gpk S (hash only).""" + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", **S}, + {"AttributeName": "sk", **S}, + {"AttributeName": "gpk", **S}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + GlobalSecondaryIndexes=[{ + "IndexName": "gsi1", + "KeySchema": [{"AttributeName": "gpk", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}, + }], + ) as name: + for pk in ("alice", "bob"): + dynamodb_client.put_item(TableName=name, Item={ + "pk": {"S": pk}, "sk": {"S": "1"}, "gpk": {"S": pk}, "a": {"N": "1"}}) + yield name + + +class TestNumberValues: + """Every number in ExpressionAttributeValues is checked.""" + + @pytest.mark.parametrize("value, expected", [ + pytest.param(NB, NUMERIC + "bob", id="N-bob"), + pytest.param({"L": [SX, NB]}, NUMERIC + "bob", id="L-with-N-bob"), + pytest.param({"M": {"a": NB}}, NUMERIC + "bob", id="M-with-N-bob"), + pytest.param({"NS": ["1", "bob"]}, NUMERIC + "bob", id="NS-with-bob"), + pytest.param({"M": {"a": {"L": [{"M": {"b": NB}}]}}}, NUMERIC + "bob", id="M-L-M-N-bob"), + pytest.param({"N": "1e200"}, OVERFLOW, id="overflow"), + pytest.param({"N": "1e-200"}, UNDERFLOW, id="underflow"), + pytest.param({"N": "1" * 39}, TOO_MANY_DIGITS, id="39-digits"), + pytest.param({"N": ""}, NUMERIC, id="empty"), + pytest.param({"N": " 1"}, NUMERIC + " 1", id="leading-space"), + ]) + def test_value_read_by_the_filter(self, dynamodb_client, table, value, expected): + assert _scan_error( + dynamodb_client, TableName=table, + FilterExpression="a = :n", ExpressionAttributeValues={":n": value}, + ) == ("ValidationException", expected) + + @pytest.mark.parametrize("value, expected", [ + pytest.param(NB, NUMERIC + "bob", id="N-bob"), + pytest.param({"L": [SX, NB]}, NUMERIC + "bob", id="L-with-N-bob"), + pytest.param({"M": {"a": NB}}, NUMERIC + "bob", id="M-with-N-bob"), + pytest.param({"NS": ["1", "bob"]}, NUMERIC + "bob", id="NS-with-bob"), + pytest.param({"M": {"a": {"L": [{"M": {"b": NB}}]}}}, NUMERIC + "bob", id="M-L-M-N-bob"), + pytest.param({"N": "1e200"}, OVERFLOW, id="overflow"), + pytest.param({"N": "1" * 39}, TOO_MANY_DIGITS, id="39-digits"), + pytest.param({"N": ""}, NUMERIC, id="empty"), + ]) + def test_value_nothing_reads(self, dynamodb_client, table, value, expected): + """The filter reads :p; :n is checked all the same, ahead of the unused value check.""" + assert _scan_error( + dynamodb_client, TableName=table, + FilterExpression="a = :p", ExpressionAttributeValues={":p": SX, ":n": value}, + ) == ("ValidationException", expected) + + def test_no_expression_at_all(self, dynamodb_client, table): + assert _scan_error( + dynamodb_client, TableName=table, ExpressionAttributeValues={":n": NB}, + ) == ("ValidationException", NUMERIC + "bob") + + def test_with_scan_filter_only(self, dynamodb_client, table): + assert _scan_error( + dynamodb_client, TableName=table, ScanFilter=SCAN_FILTER, + ExpressionAttributeValues={":n": NB}, + ) == ("ValidationException", NUMERIC + "bob") + + def test_with_projection_expression_only(self, dynamodb_client, table): + assert _scan_error( + dynamodb_client, TableName=table, ProjectionExpression="a", + ExpressionAttributeValues={":n": NB}, + ) == ("ValidationException", NUMERIC + "bob") + + def test_with_conditional_operator_and_no_filter(self, dynamodb_client, table): + """ConditionalOperator alone does not join the parameter mix; the number is reported.""" + assert _scan_error( + dynamodb_client, TableName=table, ConditionalOperator="AND", + ExpressionAttributeValues={":n": NB}, + ) == ("ValidationException", NUMERIC + "bob") + + def test_two_bad_numbers_report_one_of_them(self, dynamodb_client, table): + """The service's choice between two bad values varied between two runs; either is accepted.""" + code, msg = _scan_error( + dynamodb_client, TableName=table, FilterExpression="a = :a", + ExpressionAttributeValues={":a": {"N": "one"}, ":b": {"N": "two"}}, + ) + assert code == "ValidationException" + assert msg in (NUMERIC + "one", NUMERIC + "two") + + def test_well_formed_numbers_pass(self, dynamodb_client, table): + for value in ("1", "1.0e1"): + response = dynamodb_client.scan( + TableName=table, FilterExpression="a = :n", + ExpressionAttributeValues={":n": {"N": value}}, + ) + assert "Count" in response, value + + +class TestChecksBeforeTheNumbers: + """Range constraints, the parameter mix and ReturnConsumedCapacity win over a bad number.""" + + @pytest.mark.parametrize("limit", [0, -1]) + def test_limit_range(self, dynamodb_client_no_validation, table, limit): + assert _scan_error( + dynamodb_client_no_validation, TableName=table, Limit=limit, **NB_FILTER, + ) == ("ValidationException", "1 validation error detected: " + LIMIT_RANGE) + + @pytest.mark.parametrize("kwargs, text", [ + pytest.param({"Segment": 0, "TotalSegments": 0}, + "Value at 'TotalSegments' failed to satisfy constraint: " + "Member must have value greater than or equal to 1", id="TotalSegments-0"), + pytest.param({"Segment": 0, "TotalSegments": 1_000_001}, + "Value at 'TotalSegments' failed to satisfy constraint: " + "Member must have value less than or equal to 1000000", id="TotalSegments-1000001"), + pytest.param({"Segment": -1, "TotalSegments": 2}, + "Value at 'Segment' failed to satisfy constraint: " + "Member must have value greater than or equal to 0", id="Segment-minus-1"), + pytest.param({"Segment": 1_000_000, "TotalSegments": 1_000_000}, + "Value at 'Segment' failed to satisfy constraint: " + "Member must have value less than or equal to 999999", id="Segment-1000000"), + pytest.param({"Segment": -1}, + "Value at 'Segment' failed to satisfy constraint: " + "Member must have value greater than or equal to 0", id="Segment-minus-1-alone"), + pytest.param({"TotalSegments": 0}, + "Value at 'TotalSegments' failed to satisfy constraint: " + "Member must have value greater than or equal to 1", id="TotalSegments-0-alone"), + ]) + def test_segment_ranges(self, dynamodb_client_no_validation, table, kwargs, text): + assert _scan_error( + dynamodb_client_no_validation, TableName=table, **kwargs, **NB_FILTER, + ) == ("ValidationException", "1 validation error detected: " + text) + + def test_ranges_aggregate_limit_then_segment_then_total_segments( + self, dynamodb_client_no_validation, table, + ): + assert _scan_error( + dynamodb_client_no_validation, TableName=table, + Limit=0, Segment=-1, TotalSegments=1_000_001, + ) == ("ValidationException", ( + "3 validation errors detected: " + LIMIT_RANGE + "; " + "Value at 'Segment' failed to satisfy constraint: " + "Member must have value greater than or equal to 0; " + "Value at 'TotalSegments' failed to satisfy constraint: " + "Member must have value less than or equal to 1000000" + )) + + def test_limit_range_beats_the_missing_table(self, dynamodb_client_no_validation): + assert _scan_error( + dynamodb_client_no_validation, TableName="extenddb-test-no-such-table", Limit=0, + ) == ("ValidationException", "1 validation error detected: " + LIMIT_RANGE) + + @pytest.mark.parametrize("extra, non_expression, expression", [ + pytest.param({"ScanFilter": SCAN_FILTER}, "ScanFilter", "FilterExpression", id="ScanFilter"), + pytest.param({"AttributesToGet": ["a"], "ProjectionExpression": "a"}, "AttributesToGet", + "ProjectionExpression, FilterExpression", id="AttributesToGet"), + pytest.param({"ConditionalOperator": "AND"}, "ConditionalOperator", "FilterExpression", + id="ConditionalOperator"), + ]) + def test_parameter_mix(self, dynamodb_client, table, extra, non_expression, expression): + assert _scan_error(dynamodb_client, TableName=table, **extra, **NB_FILTER) == ( + "ValidationException", + "Can not use both expression and non-expression parameters in the same request: " + f"Non-expression parameters: {{{non_expression}}} Expression parameters: {{{expression}}}", + ) + + def test_parameter_mix_beats_return_consumed_capacity(self, dynamodb_client_no_validation, table): + assert _scan_error( + dynamodb_client_no_validation, TableName=table, ScanFilter=SCAN_FILTER, + ReturnConsumedCapacity="NOPE", **OK_FILTER, + ) == ("ValidationException", ( + "Can not use both expression and non-expression parameters in the same request: " + "Non-expression parameters: {ScanFilter} Expression parameters: {FilterExpression}" + )) + + def test_limit_range_beats_parameter_mix(self, dynamodb_client_no_validation, table): + assert _scan_error( + dynamodb_client_no_validation, TableName=table, ScanFilter=SCAN_FILTER, Limit=0, **OK_FILTER, + ) == ("ValidationException", "1 validation error detected: " + LIMIT_RANGE) + + def test_return_consumed_capacity_enum(self, dynamodb_client_no_validation, table): + assert _scan_error( + dynamodb_client_no_validation, TableName=table, ReturnConsumedCapacity="NOPE", **NB_FILTER, + ) == ("ValidationException", RCC_ENUM) + + def test_both_enums_invalid_report_return_consumed_capacity_alone( + self, dynamodb_client_no_validation, table, + ): + for extra in ({}, NB_FILTER): + assert _scan_error( + dynamodb_client_no_validation, TableName=table, + Select="NOPE", ReturnConsumedCapacity="NOPE", **extra, + ) == ("ValidationException", RCC_ENUM) + + def test_limit_range_beats_both_enums(self, dynamodb_client_no_validation, table): + assert _scan_error( + dynamodb_client_no_validation, TableName=table, + Limit=0, Select="NOPE", ReturnConsumedCapacity="NOPE", + ) == ("ValidationException", "1 validation error detected: " + LIMIT_RANGE) + + +class TestChecksAfterTheNumbers: + """A bad number wins over every check the handler makes later.""" + + def test_select_enum(self, dynamodb_client_no_validation, table): + assert _scan_error( + dynamodb_client_no_validation, TableName=table, Select="NOPE", **NB_FILTER, + ) == ("ValidationException", NUMERIC + "bob") + assert _scan_error( + dynamodb_client_no_validation, TableName=table, Select="NOPE", **OK_FILTER, + ) == ("ValidationException", SELECT_ENUM) + + @pytest.mark.parametrize("kwargs", [ + pytest.param({"Segment": 0}, id="Segment-without-TotalSegments"), + pytest.param({"TotalSegments": 2}, id="TotalSegments-without-Segment"), + pytest.param({"Segment": 5, "TotalSegments": 2}, id="Segment-not-below-TotalSegments"), + pytest.param({"IndexName": "nope"}, id="unknown-IndexName"), + pytest.param({"IndexName": "gsi1", "ConsistentRead": True}, id="ConsistentRead-on-GSI"), + pytest.param({"Select": "COUNT", "ProjectionExpression": "a"}, id="COUNT-with-ProjectionExpression"), + pytest.param({"ExclusiveStartKey": {"wrong": SX}}, id="ExclusiveStartKey-off-schema"), + ]) + def test_handler_checks(self, dynamodb_client, table, kwargs): + assert _scan_error( + dynamodb_client, TableName=table, **kwargs, **NB_FILTER, + ) == ("ValidationException", NUMERIC + "bob") + + def test_missing_table(self, dynamodb_client): + assert _scan_error( + dynamodb_client, TableName="extenddb-test-no-such-table", **NB_FILTER, + ) == ("ValidationException", NUMERIC + "bob") + + def test_undefined_placeholder(self, dynamodb_client, table): + assert _scan_error( + dynamodb_client, TableName=table, FilterExpression="a = :zz", + ExpressionAttributeValues={":n": NB}, + ) == ("ValidationException", NUMERIC + "bob") + + def test_filter_syntax_error(self, dynamodb_client, table): + assert _scan_error( + dynamodb_client, TableName=table, FilterExpression="a = = :n", + ExpressionAttributeValues={":n": NB}, + ) == ("ValidationException", NUMERIC + "bob") From d676c6ea43b9ea4cbf1521aef1f9db259d746051 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Sun, 25 Oct 2026 12:20:49 +0000 Subject: [PATCH 3/3] fix(scan): report range faults one at a time, shape faults first, and accept Unicode decimal digits The range checks on Limit, Segment and TotalSegments reported every fault in one aggregated message with a capitalised field name and no value for all three fields. The service uses that form for a Limit inside the int32 range and saturates the value into the older form below it (Limit -2147483649 answers Value '-2147483648' at 'limit'); the Segment and TotalSegments texts and the aggregation are a separate question. The checks now report the first fault in the order Limit, Segment, TotalSegments; Limit follows the measured int32 rule and the two segment fields keep the texts of the previous release. The two assertions in tests/test_query_scan.py go back to the previous text. A field of the wrong JSON type in another field (a string ExpressionAttributeValues beside Limit 0, a string TotalSegments beside Segment -1, a string ExpressionAttributeNames beside a malformed N, an object AttributesToGet beside ProjectionExpression) answered the check's ValidationException where the service and the previous release answer SerializationException. When a check fails the body is deserialized as the typed input and a SerializationException from that is returned instead, with the helper the Query path uses. A Unicode decimal digit in an N value passes the request-time check shared with Query, as it does on the service. The filter compares the value as the request wrote it, as on the previous release. The unit test for the parameter mix no longer asserts that an empty ScanFilter beside a FilterExpression passes, which the service refuses as a parameter mix; the engine's handling of an empty ScanFilter and an empty AttributesToGet is unchanged and covered by a separate issue. Measured against the service on 2026-10-25 (10:38, 10:55 and 11:31 UTC), each request twice. --- crates/engine/src/scan.rs | 3 +- crates/engine/src/scan_request.rs | 115 ++++++++++------ crates/engine/src/scan_request/tests.rs | 159 ++++++++++++++++++---- tests/test_query_scan.py | 20 +-- tests/test_scan_number_values.py | 174 +++++++++++++++++++----- 5 files changed, 361 insertions(+), 110 deletions(-) diff --git a/crates/engine/src/scan.rs b/crates/engine/src/scan.rs index 2a2214d84..9a8e15296 100755 --- a/crates/engine/src/scan.rs +++ b/crates/engine/src/scan.rs @@ -46,7 +46,8 @@ pub async fn handle_scan( // parameter mix, the ReturnConsumedCapacity enum, every number in // ExpressionAttributeValues, the Select enum. They run before typed // deserialization so a field that does not deserialize does not stop the - // ones before it; see `scan_request`. + // ones before it; a field of the wrong JSON type anywhere in the body is + // reported ahead of them; see `scan_request`. crate::scan_request::validate_scan_request_body(&body)?; let input: ScanInput = serde_json::from_value(body).map_err(crate::deserialize_error)?; diff --git a/crates/engine/src/scan_request.rs b/crates/engine/src/scan_request.rs index e1f89f410..2214d0058 100644 --- a/crates/engine/src/scan_request.rs +++ b/crates/engine/src/scan_request.rs @@ -7,16 +7,21 @@ //! Measured 2026-10-25 (us-east-1, each request twice): //! //! 1. the range constraints on `Limit` (at least 1), `Segment` (0 to 999999) -//! and `TotalSegments` (1 to 1000000), aggregated into one message in that -//! order with no value echoed: `2 validation errors detected: Value at -//! 'Limit' failed to satisfy constraint: Member must have value greater than -//! or equal to 1; Value at 'Segment' ...`. Each field is checked when -//! present, whether or not the other segment field is; +//! and `TotalSegments` (1 to 1000000), each checked when present, whether or +//! not the other segment field is. A `Limit` below 1 inside the int32 range +//! answers `1 validation error detected: Value at 'Limit' failed to satisfy +//! constraint: Member must have value greater than or equal to 1`; below +//! int32 the service saturates the value and answers the older form +//! `Value '-2147483648' at 'limit' ...`. The service aggregates several +//! range faults into one message; this module reports the first in the +//! order `Limit`, `Segment`, `TotalSegments`, with the `Segment` and +//! `TotalSegments` texts the previous release used; //! 2. the expression and non-expression parameter mix; //! 3. the `ReturnConsumedCapacity` enum; //! 4. every number in `ExpressionAttributeValues`, `NS` members and values //! nested in `L` and `M` included, whether a `FilterExpression` reads the -//! value, nothing reads it, or the request carries no expression at all; +//! value, nothing reads it, or the request carries no expression at all; a +//! Unicode decimal digit counts as the digit it stands for; //! 5. the `Select` enum. Both enums invalid answers the `ReturnConsumedCapacity` //! error alone, so the two are not aggregated. //! @@ -25,15 +30,18 @@ //! (Segment without TotalSegments and the reverse, Segment not below //! TotalSegments, the undefined placeholder, unused value, filter syntax, //! IndexName, ConsistentRead, Select with ProjectionExpression and -//! ExclusiveStartKey checks). +//! ExclusiveStartKey checks). A field of the wrong JSON type anywhere in the +//! body is reported ahead of all of them, as a `SerializationException`. use std::collections::HashMap; use extenddb_core::error::DynamoDbError; -use extenddb_core::types::AttributeValue; +use extenddb_core::types::{AttributeValue, ScanInput}; use serde_json::Value; -use crate::query_key_values::{non_empty, present, validate_expression_attribute_value_numbers}; +use crate::query_key_values::{ + non_empty, present, shape_fault_first, validate_expression_attribute_value_numbers, +}; /// The `Scan` checks that run on the raw request body, in the service's order: /// the `Limit`, `Segment` and `TotalSegments` ranges, the expression and @@ -41,13 +49,24 @@ use crate::query_key_values::{non_empty, present, validate_expression_attribute_ /// number in `ExpressionAttributeValues`, then the `Select` enum. /// /// The checks read the raw body, so a field that does not deserialize (an -/// invalid `Select`, for one) does not stop the ones before it. A field of the -/// wrong JSON shape is left to the deserializer, which runs next. +/// invalid `Select`, for one) does not stop the ones before it. When a check +/// fails, the body is deserialized as a [`ScanInput`] and a +/// `SerializationException` from that (a field of the wrong JSON shape +/// anywhere in the body) is the answer instead, as the service reports a shape +/// fault ahead of every validation; see [`shape_fault_first`]. A body that +/// passes every check is not deserialized here. /// /// # Errors /// -/// Returns `DynamoDbError::ValidationException` with the service's text. +/// Returns `DynamoDbError::ValidationException` with the service's text, or +/// `DynamoDbError::SerializationException` for a body of the wrong shape. pub fn validate_scan_request_body(body: &Value) -> Result<(), DynamoDbError> { + scan_request_checks(body).map_err(|check| shape_fault_first::(body, check)) +} + +/// The `Scan` checks in the service's order; see +/// [`validate_scan_request_body`]. +fn scan_request_checks(body: &Value) -> Result<(), DynamoDbError> { validate_ranges(body)?; extenddb_core::validation::validate_no_expression_param_mixing( @@ -95,38 +114,56 @@ pub fn validate_scan_request_body(body: &Value) -> Result<(), DynamoDbError> { ) } -/// The range constraints on `Limit`, `Segment` and `TotalSegments`, aggregated -/// in that order as the service reports them. The field name is capitalised -/// and the value is not echoed, unlike the `select` and `segment` style the -/// other `1 validation error detected` messages use. +/// The range constraints on `Limit`, `Segment` and `TotalSegments`, the first +/// fault in that order. +/// +/// `Limit` answers the service's text: inside the int32 range the field is +/// capitalised and the value is not echoed; below int32 the service saturates +/// the value and uses the older `Value '' at 'limit'` form (measured +/// 2026-10-25, `Limit: -2147483649`). `Segment` and `TotalSegments` keep the +/// texts of the previous release. fn validate_ranges(body: &Value) -> Result<(), DynamoDbError> { - let mut errors: Vec = Vec::new(); - let mut check = |field: &str, below: i64, above: i64| { - let Some(value) = body.get(field).and_then(Value::as_i64) else { - return; + let get = |field: &str| body.get(field).and_then(Value::as_i64); + if let Some(limit) = get("Limit") + && limit < 1 + { + let text = if limit >= i64::from(i32::MIN) { + "Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1".to_owned() + } else { + format!( + "Value '{}' at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1", + i32::MIN + ) }; - if value < below { - errors.push(format!( - "Value at '{field}' failed to satisfy constraint: Member must have value greater than or equal to {below}" - )); - } else if value > above { - errors.push(format!( - "Value at '{field}' failed to satisfy constraint: Member must have value less than or equal to {above}" + return Err(DynamoDbError::ValidationException(format!( + "1 validation error detected: {text}" + ))); + } + if let Some(segment) = get("Segment") { + if segment < 0 { + return Err(DynamoDbError::ValidationException(format!( + "1 validation error detected: Value '{segment}' at 'segment' failed to satisfy constraint: Member must have value greater than or equal to 0" + ))); + } + if segment > 999_999 { + return Err(DynamoDbError::ValidationException(format!( + "1 validation error detected: Value '{segment}' at 'segment' failed to satisfy constraint: Member must have value less than or equal to 999999" + ))); + } + } + if let Some(total) = get("TotalSegments") { + if total < 1 { + return Err(DynamoDbError::ValidationException( + "The parameter TotalSegments should be greater than or equal to 1".to_owned(), )); } - }; - check("Limit", 1, i64::MAX); - check("Segment", 0, 999_999); - check("TotalSegments", 1, 1_000_000); - if errors.is_empty() { - return Ok(()); + if total > 1_000_000 { + return Err(DynamoDbError::ValidationException(format!( + "1 validation error detected: Value '{total}' at 'totalSegments' failed to satisfy constraint: Member must have value less than or equal to 1000000" + ))); + } } - let count = errors.len(); - Err(DynamoDbError::ValidationException(format!( - "{count} validation error{} detected: {}", - if count == 1 { "" } else { "s" }, - errors.join("; ") - ))) + Ok(()) } #[cfg(test)] diff --git a/crates/engine/src/scan_request/tests.rs b/crates/engine/src/scan_request/tests.rs index 80af8ae17..2ae845559 100644 --- a/crates/engine/src/scan_request/tests.rs +++ b/crates/engine/src/scan_request/tests.rs @@ -1,8 +1,12 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! Every expected text below was measured against the service on 2026-10-25 -//! (us-east-1, each request twice). +//! The expected texts below were measured against the service on 2026-10-25 +//! (us-east-1, each request twice), except where a test says otherwise: the +//! `Segment` and `TotalSegments` range texts are the previous release's (the +//! service aggregates range faults into one message with a different form, a +//! separate issue), and the deserializer test pins this module's hand-off, not +//! a service text. use extenddb_core::error::DynamoDbError; use serde_json::{Value, json}; @@ -122,7 +126,7 @@ fn nested_numbers_and_the_other_number_faults_are_reported_with_the_service_text } #[test] -fn limit_segment_and_total_segments_ranges_come_first_and_aggregate() { +fn limit_segment_and_total_segments_ranges_come_first() { let base = bad_number(); assert_eq!( check(&with(base.clone(), "Limit", json!(0))).as_deref(), @@ -132,33 +136,45 @@ fn limit_segment_and_total_segments_ranges_come_first_and_aggregate() { check(&with(base.clone(), "Limit", json!(-1))).as_deref(), Some(&*format!("1 validation error detected: {LIMIT_RANGE}")) ); + // The int32 minimum itself gets the capitalised form; below it the service + // saturates the value and uses the older form (measured 2026-10-25). + assert_eq!( + check(&with(base.clone(), "Limit", json!(-2_147_483_648_i64))).as_deref(), + Some(&*format!("1 validation error detected: {LIMIT_RANGE}")) + ); + assert_eq!( + check(&with(base.clone(), "Limit", json!(-2_147_483_649_i64))).as_deref(), + Some( + "1 validation error detected: Value '-2147483648' at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1" + ) + ); // Each segment field is range checked when present, with or without the - // other one. + // other one (the service's position; the texts are the previous release's). for (field, value, text) in [ ( "Segment", -1, - "Value at 'Segment' failed to satisfy constraint: Member must have value greater than or equal to 0", + "1 validation error detected: Value '-1' at 'segment' failed to satisfy constraint: Member must have value greater than or equal to 0", ), ( "Segment", 1_000_000, - "Value at 'Segment' failed to satisfy constraint: Member must have value less than or equal to 999999", + "1 validation error detected: Value '1000000' at 'segment' failed to satisfy constraint: Member must have value less than or equal to 999999", ), ( "TotalSegments", 0, - "Value at 'TotalSegments' failed to satisfy constraint: Member must have value greater than or equal to 1", + "The parameter TotalSegments should be greater than or equal to 1", ), ( "TotalSegments", 1_000_001, - "Value at 'TotalSegments' failed to satisfy constraint: Member must have value less than or equal to 1000000", + "1 validation error detected: Value '1000001' at 'totalSegments' failed to satisfy constraint: Member must have value less than or equal to 1000000", ), ] { assert_eq!( check(&with(base.clone(), field, json!(value))).as_deref(), - Some(&*format!("1 validation error detected: {text}")), + Some(text), "{field} {value}" ); let partner = if field == "Segment" { @@ -173,12 +189,13 @@ fn limit_segment_and_total_segments_ranges_come_first_and_aggregate() { ); assert_eq!( check(&paired).as_deref(), - Some(&*format!("1 validation error detected: {text}")), + Some(text), "{field} {value} with {}", partner.0 ); } - // Limit, Segment, TotalSegments, in that order, one message. + // Several range faults: the first in the order Limit, Segment, + // TotalSegments is reported. let three = with( with( with(json!({"TableName": "t"}), "Limit", json!(0)), @@ -190,10 +207,14 @@ fn limit_segment_and_total_segments_ranges_come_first_and_aggregate() { ); assert_eq!( check(&three).as_deref(), - Some( - "3 validation errors detected: Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1; Value at 'Segment' failed to satisfy constraint: Member must have value greater than or equal to 0; Value at 'TotalSegments' failed to satisfy constraint: Member must have value less than or equal to 1000000" - ) + Some(&*format!("1 validation error detected: {LIMIT_RANGE}")) + ); + let two = with( + with(json!({"TableName": "t"}), "Segment", json!(-1)), + "TotalSegments", + json!(1_000_001), ); + assert!(check(&two).is_some_and(|m| m.contains("at 'segment'"))); // Values at the bounds pass. assert!( check( @@ -207,7 +228,7 @@ fn limit_segment_and_total_segments_ranges_come_first_and_aggregate() { "ScanFilter", json!({"a": {"ComparisonOperator": "EQ", "AttributeValueList": [{"S": "x"}]}}), ); - assert!(check(&mix).is_some_and(|m| m.contains("Value at 'Segment'"))); + assert!(check(&mix).is_some_and(|m| m.contains("at 'segment'"))); let rcc = with( with(base, "Limit", json!(0)), "ReturnConsumedCapacity", @@ -268,16 +289,17 @@ fn parameter_mix_beats_the_enum_and_the_numbers() { .as_deref(), Some(NUMERIC_BOB) ); - // An empty ScanFilter or AttributesToGet does not join the mix. - assert!( + // An empty ScanFilter alone is accepted, and an unused N bob beside it + // answers the number (measured 2026-10-25). + assert!(check(&json!({"TableName": "t", "ScanFilter": {}})).is_none()); + assert_eq!( check(&json!({ "TableName": "t", "ScanFilter": {}, - "AttributesToGet": [], - "FilterExpression": "a = :n", - "ExpressionAttributeValues": {":n": {"N": "1"}}, + "ExpressionAttributeValues": {":n": {"N": "bob"}}, })) - .is_none() + .as_deref(), + Some(NUMERIC_BOB) ); } @@ -318,12 +340,101 @@ fn enum_checks_sit_on_either_side_of_the_numbers() { } #[test] -fn values_that_do_not_deserialize_are_left_to_the_deserializer() { - // A string Limit and a malformed ExpressionAttributeValues are not range - // or number faults here; the typed deserializer reports them next. +fn a_shape_fault_anywhere_in_the_body_is_reported_first() { + // Measured 2026-10-25: a field of the wrong + // JSON type in another field answers SerializationException ahead of the + // range, mix and number checks. + let shape_faults = [ + json!({"TableName": "t", "Limit": 0, "ExpressionAttributeValues": "x"}), + json!({"TableName": "t", "Segment": -1, "TotalSegments": "2"}), + with(bad_number(), "ExpressionAttributeNames", json!("x")), + json!({"TableName": "t", "AttributesToGet": {"a": 1}, "ProjectionExpression": "a"}), + json!({"TableName": "t", "ScanFilter": [1], "FilterExpression": "a = :n", + "ExpressionAttributeValues": {":n": {"N": "1"}}}), + json!({"TableName": "t", "Limit": 0, "Select": 5}), + ]; + for body in shape_faults { + assert!( + matches!( + validate_scan_request_body(&body), + Err(DynamoDbError::SerializationException(_)) + ), + "{body}" + ); + } + // A string enum that is not a member is the checks' answer (not a shape + // fault), and a ValidationException the deserializer raises (an empty NS) + // does not preempt a check. + assert_eq!( + check(&with( + with(bad_number(), "Limit", json!(0)), + "Select", + json!("NOPE") + )) + .as_deref(), + Some(&*format!("1 validation error detected: {LIMIT_RANGE}")) + ); + assert_eq!( + check(&json!({ + "TableName": "t", "Limit": 0, + "ExpressionAttributeValues": {":s": {"NS": []}}, + })) + .as_deref(), + Some(&*format!("1 validation error detected: {LIMIT_RANGE}")) + ); + // A body that passes every check is left to the handler's deserializer: a + // string Limit and a malformed AttributeValue are not range or number + // faults here. assert!(check(&json!({"TableName": "t", "Limit": "0"})).is_none()); assert!( check(&json!({"TableName": "t", "ExpressionAttributeValues": {":n": {"X": "bob"}}})) .is_none() ); } + +#[test] +fn unicode_decimal_digits_pass_the_number_check() { + // Measured 2026-10-25: fullwidth and Arabic-Indic digits are read as digits, thirty nine + // Devanagari ones answer the 38 digit message, a superscript, a fraction, a + // fullwidth minus and a fullwidth full stop are refused with the value as + // sent. + for raw in ["\u{ff11}", "\u{661}", "\u{ff17}", "\u{ff11}e2"] { + assert!( + check(&with( + bad_number(), + "ExpressionAttributeValues", + json!({":n": {"N": raw}}) + )) + .is_none(), + "{raw:?}" + ); + } + assert!( + check(&json!({ + "TableName": "t", "FilterExpression": "a = :p", + "ExpressionAttributeValues": {":p": {"S": "x"}, ":s": {"NS": ["1", "\u{ff12}"]}}, + })) + .is_none() + ); + assert_eq!( + check(&with( + bad_number(), + "ExpressionAttributeValues", + json!({":n": {"N": "\u{967}".repeat(39)}}) + )) + .as_deref(), + Some("Attempting to store more than 38 significant digits in a Number") + ); + for raw in ["\u{b2}", "\u{bd}", "\u{ff0d}1", "1\u{ff0e}5"] { + assert_eq!( + check(&with( + bad_number(), + "ExpressionAttributeValues", + json!({":n": {"N": raw}}) + )) + .as_deref(), + Some(format!("The parameter cannot be converted to a numeric value: {raw}").as_str()), + "{raw:?}" + ); + } +} diff --git a/tests/test_query_scan.py b/tests/test_query_scan.py index 04019c990..44a82f4a2 100755 --- a/tests/test_query_scan.py +++ b/tests/test_query_scan.py @@ -614,17 +614,17 @@ def test_scan_segment_at_max_accepted(self, dynamodb_client, scan_table): def test_scan_segment_one_over_max_rejected( self, dynamodb_client_no_validation, scan_table ): - """Segment > 999999 returns the range ValidationException (measured 2026-10-25).""" + """Segment > 999999 returns the Coral-format ValidationException.""" with pytest.raises(ClientError) as exc_info: dynamodb_client_no_validation.scan( TableName=scan_table, Segment=1_000_000, TotalSegments=1_000_000 ) err = exc_info.value.response["Error"] assert err["Code"] == "ValidationException" - assert err["Message"] == ( - "1 validation error detected: Value at 'Segment' failed to satisfy " - "constraint: Member must have value less than or equal to 999999" - ) + msg = err["Message"] + assert "1 validation error detected" in msg + assert "'1000000' at 'segment'" in msg + assert "Member must have value less than or equal to 999999" in msg def test_scan_total_segments_at_max_accepted(self, dynamodb_client, scan_table): """TotalSegments=1000000 with Segment=0 is at the documented bound.""" @@ -635,17 +635,17 @@ def test_scan_total_segments_at_max_accepted(self, dynamodb_client, scan_table): def test_scan_total_segments_one_over_max_rejected( self, dynamodb_client_no_validation, scan_table ): - """TotalSegments > 1000000 returns the range ValidationException (measured 2026-10-25).""" + """TotalSegments > 1000000 returns the Coral-format ValidationException.""" with pytest.raises(ClientError) as exc_info: dynamodb_client_no_validation.scan( TableName=scan_table, Segment=0, TotalSegments=1_000_001 ) err = exc_info.value.response["Error"] assert err["Code"] == "ValidationException" - assert err["Message"] == ( - "1 validation error detected: Value at 'TotalSegments' failed to satisfy " - "constraint: Member must have value less than or equal to 1000000" - ) + msg = err["Message"] + assert "1 validation error detected" in msg + assert "'1000001' at 'totalSegments'" in msg + assert "Member must have value less than or equal to 1000000" in msg def test_scan_all_projected_attributes_without_index_rejected( self, dynamodb_client, scan_table diff --git a/tests/test_scan_number_values.py b/tests/test_scan_number_values.py index df354ef4d..89ab066aa 100644 --- a/tests/test_scan_number_values.py +++ b/tests/test_scan_number_values.py @@ -8,13 +8,18 @@ significant digits or an empty ``N`` is refused with ``ValidationException`` before any row is read, whether a ``FilterExpression`` reads the value, nothing reads it, or the request carries no expression at all. Every expected -answer below was measured against the service on 2026-10-25 (09:23 to 09:28 -UTC) with each request sent twice. +answer below was measured against the service on 2026-10-25 (09:23 to 09:28, +10:38, 10:55 and 11:31 UTC) with each request sent twice. A Unicode decimal +digit in an ``N`` value counts as the digit it stands for; a field of the +wrong JSON type anywhere in the body answers ``SerializationException`` ahead +of every check. The rows also pin where the check falls against the other early Scan checks: -after the ``Limit``, ``Segment`` and ``TotalSegments`` range constraints (one -aggregated message, ``Value at 'Limit'`` with no value echoed), the expression -and non-expression parameter mix and the ``ReturnConsumedCapacity`` enum; and +after the ``Limit``, ``Segment`` and ``TotalSegments`` range constraints +(``Value at 'Limit'`` with no value echoed for a ``Limit`` inside the int32 +range; the ``Segment`` and ``TotalSegments`` rows assert the position of the +check, not its text), the expression and non-expression parameter mix and the +``ReturnConsumedCapacity`` enum; and before the ``Select`` enum, the Segment pairing checks, the table lookup, the IndexName, ConsistentRead, Select with ProjectionExpression, ExclusiveStartKey, undefined placeholder, unused value and filter syntax @@ -194,44 +199,45 @@ def test_limit_range(self, dynamodb_client_no_validation, table, limit): dynamodb_client_no_validation, TableName=table, Limit=limit, **NB_FILTER, ) == ("ValidationException", "1 validation error detected: " + LIMIT_RANGE) - @pytest.mark.parametrize("kwargs, text", [ - pytest.param({"Segment": 0, "TotalSegments": 0}, - "Value at 'TotalSegments' failed to satisfy constraint: " - "Member must have value greater than or equal to 1", id="TotalSegments-0"), - pytest.param({"Segment": 0, "TotalSegments": 1_000_001}, - "Value at 'TotalSegments' failed to satisfy constraint: " - "Member must have value less than or equal to 1000000", id="TotalSegments-1000001"), - pytest.param({"Segment": -1, "TotalSegments": 2}, - "Value at 'Segment' failed to satisfy constraint: " - "Member must have value greater than or equal to 0", id="Segment-minus-1"), - pytest.param({"Segment": 1_000_000, "TotalSegments": 1_000_000}, - "Value at 'Segment' failed to satisfy constraint: " - "Member must have value less than or equal to 999999", id="Segment-1000000"), - pytest.param({"Segment": -1}, - "Value at 'Segment' failed to satisfy constraint: " - "Member must have value greater than or equal to 0", id="Segment-minus-1-alone"), - pytest.param({"TotalSegments": 0}, - "Value at 'TotalSegments' failed to satisfy constraint: " - "Member must have value greater than or equal to 1", id="TotalSegments-0-alone"), + @pytest.mark.parametrize("kwargs, field", [ + pytest.param({"Segment": 0, "TotalSegments": 0}, "totalsegments", id="TotalSegments-0"), + pytest.param({"Segment": 0, "TotalSegments": 1_000_001}, "totalsegments", id="TotalSegments-1000001"), + pytest.param({"Segment": -1, "TotalSegments": 2}, "segment", id="Segment-minus-1"), + pytest.param({"Segment": 1_000_000, "TotalSegments": 1_000_000}, "segment", id="Segment-1000000"), + pytest.param({"Segment": -1}, "segment", id="Segment-minus-1-alone"), + pytest.param({"TotalSegments": 0}, "totalsegments", id="TotalSegments-0-alone"), ]) - def test_segment_ranges(self, dynamodb_client_no_validation, table, kwargs, text): - assert _scan_error( - dynamodb_client_no_validation, TableName=table, **kwargs, **NB_FILTER, - ) == ("ValidationException", "1 validation error detected: " + text) + def test_segment_ranges(self, dynamodb_client_no_validation, table, kwargs, field): + # The range fault names the field and is reported ahead of the number; + # the text itself differs between the service and extenddb. + code, msg = _scan_error(dynamodb_client_no_validation, TableName=table, **kwargs, **NB_FILTER) + assert code == "ValidationException" + assert field in msg.lower().replace("'", "") + assert "bob" not in msg - def test_ranges_aggregate_limit_then_segment_then_total_segments( + def test_limit_range_is_reported_ahead_of_the_segment_ranges_and_the_number( self, dynamodb_client_no_validation, table, ): - assert _scan_error( + code, msg = _scan_error( dynamodb_client_no_validation, TableName=table, - Limit=0, Segment=-1, TotalSegments=1_000_001, + Limit=0, Segment=-1, TotalSegments=1_000_001, **NB_FILTER, + ) + assert code == "ValidationException" + assert msg.endswith(LIMIT_RANGE) or msg.startswith("3 validation errors detected: " + LIMIT_RANGE) + assert "bob" not in msg + + def test_limit_below_int32_answers_the_saturated_value(self, dynamodb_client_no_validation, table): + # Measured 2026-10-25 10:38 and 10:55 UTC: outside int32 the service + # echoes the saturated value in the older form, and beats the number. + assert _scan_error( + dynamodb_client_no_validation, TableName=table, Limit=-2_147_483_649, **NB_FILTER, ) == ("ValidationException", ( - "3 validation errors detected: " + LIMIT_RANGE + "; " - "Value at 'Segment' failed to satisfy constraint: " - "Member must have value greater than or equal to 0; " - "Value at 'TotalSegments' failed to satisfy constraint: " - "Member must have value less than or equal to 1000000" + "1 validation error detected: Value '-2147483648' at 'limit' failed to " + "satisfy constraint: Member must have value greater than or equal to 1" )) + assert _scan_error( + dynamodb_client_no_validation, TableName=table, Limit=-2_147_483_648, **NB_FILTER, + ) == ("ValidationException", "1 validation error detected: " + LIMIT_RANGE) def test_limit_range_beats_the_missing_table(self, dynamodb_client_no_validation): assert _scan_error( @@ -328,3 +334,99 @@ def test_filter_syntax_error(self, dynamodb_client, table): dynamodb_client, TableName=table, FilterExpression="a = = :n", ExpressionAttributeValues={":n": NB}, ) == ("ValidationException", NUMERIC + "bob") + + +FW_ONE = "\uff11" # fullwidth digit one +FW_SEVEN = "\uff17" # fullwidth digit seven +AR_ONE = "\u0661" # Arabic-Indic digit one +DEVA_ONE = "\u0967" # Devanagari digit one +# Superscript two, vulgar half, fullwidth minus then 1, 1 fullwidth full stop 5. +NOT_DIGITS = ["\u00b2", "\u00bd", "\uff0d1", "1\uff0e5"] + + +class TestUnicodeDecimalDigits: + """A Unicode decimal digit (general category Nd) in an N value passes the + number check as the digit it stands for; other characters are refused with + the value echoed as sent (measured 2026-10-25 10:38 and 10:55 UTC). The + service also reads such a value as the digit when it compares; this change + does not, so only the acceptance is asserted.""" + + @pytest.mark.parametrize("raw", [FW_ONE, AR_ONE], ids=["fullwidth-1", "arabic-indic-1"]) + def test_digit_in_the_filter_is_accepted(self, dynamodb_client, table, raw): + # The request is accepted and every row is scanned. + resp = dynamodb_client.scan( + TableName=table, FilterExpression="a = :n", + ExpressionAttributeValues={":n": {"N": raw}}) + assert resp["ScannedCount"] == 2 + + @pytest.mark.parametrize("raw", [FW_SEVEN, FW_ONE + "e2"], ids=["fullwidth-7", "fullwidth-1e2"]) + def test_digit_that_matches_no_row(self, dynamodb_client, table, raw): + # Accepted; no stored item has a equal to 7 or 100. + resp = dynamodb_client.scan( + TableName=table, FilterExpression="a = :n", + ExpressionAttributeValues={":n": {"N": raw}}) + assert (resp["Count"], resp["ScannedCount"]) == (0, 2) + + def test_digit_in_a_number_set_passes_the_check(self, dynamodb_client, table): + assert _scan_error( + dynamodb_client, TableName=table, FilterExpression="a = :p", + ExpressionAttributeValues={":p": SX, ":s": {"NS": ["1", "\uff12"]}}, + ) == ("ValidationException", + "Value provided in ExpressionAttributeValues unused in expressions: keys: {:s}") + + def test_39_devanagari_digits_are_39_digits(self, dynamodb_client, table): + assert _scan_error( + dynamodb_client, TableName=table, FilterExpression="a = :n", + ExpressionAttributeValues={":n": {"N": DEVA_ONE * 39}}, + ) == ("ValidationException", TOO_MANY_DIGITS) + + @pytest.mark.parametrize("raw", NOT_DIGITS, ids=["superscript", "fraction", "fullwidth-minus", "fullwidth-point"]) + def test_characters_that_are_not_digits_are_refused(self, dynamodb_client, table, raw): + assert _scan_error( + dynamodb_client, TableName=table, FilterExpression="a = :n", + ExpressionAttributeValues={":n": {"N": raw}}, + ) == ("ValidationException", NUMERIC + raw) + + +def _raw_scan(client, body: dict): + """POST ``body`` as the Scan JSON exactly as written, signed with the + client's credentials, so a field of the wrong JSON type reaches the server + (boto3 would refuse or reshape it).""" + import json + + from botocore.awsrequest import AWSRequest + + req = AWSRequest(method="POST", url=client.meta.endpoint_url, data=json.dumps(body), + headers={"Content-Type": "application/x-amz-json-1.0", + "X-Amz-Target": "DynamoDB_20120810.Scan"}) + client._request_signer.sign("Scan", req) + resp = client._endpoint.http_session.send(req.prepare()) + return resp.status_code, json.loads(resp.content.decode("utf-8", "replace") or "{}") + + +SHAPE_FAULTS = [ + ("Limit 0 with ExpressionAttributeValues string", {"Limit": 0, "ExpressionAttributeValues": "x"}), + ("Segment -1 with TotalSegments string", {"Segment": -1, "TotalSegments": "2"}), + ("ExpressionAttributeNames string with N bob", {"ExpressionAttributeNames": "x", **NB_FILTER}), + ("AttributesToGet object with ProjectionExpression", + {"AttributesToGet": {"a": 1}, "ProjectionExpression": "a"}), +] + + +class TestShapeFaultsAreReportedFirst: + """A field of the wrong JSON type anywhere in the body answers + SerializationException ahead of the range, parameter mix and number checks + (measured 2026-10-25 10:38 UTC). The message texts differ between the + service and extenddb, so only the error type is asserted.""" + + @pytest.mark.parametrize("row,body", SHAPE_FAULTS, ids=[r[0] for r in SHAPE_FAULTS]) + def test_rows(self, dynamodb_client, table, row, body): + status, payload = _raw_scan(dynamodb_client, {"TableName": table, **body}) + assert status == 400 + assert payload.get("__type", "").endswith("SerializationException"), payload + + def test_control_without_a_shape_fault_answers_the_check(self, dynamodb_client, table): + status, payload = _raw_scan(dynamodb_client, {"TableName": table, "Limit": 0, **NB_FILTER}) + assert status == 400 + assert payload.get("__type", "").endswith("ValidationException") + assert payload.get("message", payload.get("Message")) == "1 validation error detected: " + LIMIT_RANGE