From 18f76f0ae25ea6a763098c7540de7791b69222b5 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Sun, 25 Oct 2026 16:15:40 +0000 Subject: [PATCH 1/5] fix(expression): report a character outside the grammar as the service does, without a panic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tokenizer behind condition and filter expressions read the offending byte as a char and sliced the input five bytes either side of it. When that range ended inside a multi-byte character (`#v = ééé`, `#v = 日本語`) the slice panicked, the request task ended and the connection closed; the server's key condition parse ahead of authorization calls the same tokenizer, so a Query key condition did the same for every principal. Other multi-byte input answered a token made of the first byte read as Latin-1. The tokenizer behind update, projection and the engine's key condition parse did not panic and answered the character with one byte after it as the window. The error now names the character as sent and shows the service's window: the source text from the start of the previous token to the end of the next one, with each character outside the grammar counted as one token (measured 2026-10-25, each request twice: `#v = ééé` answers `token: "é", near: "= éé"`, `abéc = :v` answers `near: "abéc"`, `#v = $x` answers `near: "= $x"`). Both tokenizer entry points share the rule. --- crates/core/src/expression/tokenizer.rs | 132 ++++++++++++++++++++---- 1 file changed, 111 insertions(+), 21 deletions(-) diff --git a/crates/core/src/expression/tokenizer.rs b/crates/core/src/expression/tokenizer.rs index ca57fd833..13e55caed 100755 --- a/crates/core/src/expression/tokenizer.rs +++ b/crates/core/src/expression/tokenizer.rs @@ -64,7 +64,10 @@ pub fn tokenize_with_limit(input: &str, max_tokens: usize) -> Result, let bytes = input.as_bytes(); let mut i = 0; + let mut prev_start: Option = None; while i < bytes.len() { + let tok_start = i; + let before = tokens.len(); match bytes[i] { b' ' | b'\t' | b'\n' | b'\r' => i += 1, b'(' => { @@ -178,16 +181,16 @@ pub fn tokenize_with_limit(input: &str, max_tokens: usize) -> Result, max_tokens, )?; } - other => { - let ch = other as char; - let near_start = i.saturating_sub(5); - let near_end = std::cmp::min(i + 5, input.len()); - let near = &input[near_start..near_end]; + _ => { + let (ch, near) = unknown_character(input, i, prev_start); return Err(validation_err(&format!( "Syntax error; token: \"{ch}\", near: \"{near}\"" ))); } } + if tokens.len() > before { + prev_start = Some(tok_start); + } } Ok(tokens) @@ -211,7 +214,10 @@ pub fn tokenize_for( let bytes = input.as_bytes(); let mut i = 0; + let mut prev_start: Option = None; while i < bytes.len() { + let tok_start = i; + let before = tokens.len(); match bytes[i] { b' ' | b'\t' | b'\n' | b'\r' => i += 1, b'(' => { @@ -325,27 +331,15 @@ pub fn tokenize_for( )?; } _ => { - // Safety: use chars() to safely extract the character at this - // byte position, avoiding panics on multi-byte UTF-8 input. - let ch = input[i..].chars().next().unwrap_or('?'); - let ch_len = ch.len_utf8(); - let near_end = std::cmp::min(i + ch_len + 1, input.len()); - // Clamp near_end to a valid char boundary - let near_end = if near_end <= input.len() { - let mut end = near_end; - while end < input.len() && !input.is_char_boundary(end) { - end += 1; - } - if end > input.len() { input.len() } else { end } - } else { - input.len() - }; - let near = &input[i..near_end]; + let (ch, near) = unknown_character(input, i, prev_start); return Err(DynamoDbError::ValidationException(format!( "Invalid {kind}: Syntax error; token: \"{ch}\", near: \"{near}\"" ))); } } + if tokens.len() > before { + prev_start = Some(tok_start); + } } Ok(tokens) } @@ -372,6 +366,55 @@ fn is_ident_char(c: u8) -> bool { c.is_ascii_alphanumeric() || c == b'_' } +/// The token text and the `near` window the service reports for a character +/// the expression grammar does not accept, at byte offset `at` of `input`. +/// +/// Measured 2026-10-25 on FilterExpression, ConditionExpression, +/// KeyConditionExpression, UpdateExpression and ProjectionExpression: the +/// token is the character as sent (a whole code point, `é` or an emoji), and +/// the window is the source text from the start of the previous token to the +/// end of the next one, whitespace kept, each dropped when absent. Every +/// character outside the grammar is a token of its own, so `#v = ééé` answers +/// `token: "é", near: "= éé"`, and a name stops at the first such character, +/// so `abéc = :v` answers `near: "abéc"`. +/// +/// `prev_start` is the byte offset where the previous token starts. +fn unknown_character(input: &str, at: usize, prev_start: Option) -> (String, String) { + let ch = input[at..].chars().next().unwrap_or('?'); + let after = at + ch.len_utf8(); + let end = next_token_end(input, after).unwrap_or(after); + let start = prev_start.unwrap_or(at); + (ch.to_string(), input[start..end].to_owned()) +} + +/// The byte offset where the next token after `from` ends, skipping the +/// tokenizer's whitespace, or `None` at the end of input. Mirrors the token +/// rules of [`tokenize_with_limit`], with a character outside the grammar +/// counted as one token. +fn next_token_end(input: &str, from: usize) -> Option { + let bytes = input.as_bytes(); + let mut i = from; + while i < bytes.len() && matches!(bytes[i], b' ' | b'\t' | b'\n' | b'\r') { + i += 1; + } + let first = *bytes.get(i)?; + let run = |mut j: usize, pred: fn(u8) -> bool| { + while j < bytes.len() && pred(bytes[j]) { + j += 1; + } + j + }; + Some(match first { + b'<' if matches!(bytes.get(i + 1), Some(b'>' | b'=')) => i + 2, + b'>' if bytes.get(i + 1) == Some(&b'=') => i + 2, + b'(' | b')' | b'[' | b']' | b',' | b'.' | b'+' | b'-' | b'=' | b'<' | b'>' => i + 1, + b'#' | b':' => run(i + 1, is_ident_char), + c if is_ident_start(c) => run(i, is_ident_char), + c if c.is_ascii_digit() => run(i, |b| b.is_ascii_digit()), + _ => i + input[i..].chars().next().map_or(1, char::len_utf8), + }) +} + fn keyword_or_ident(word: &str) -> Token { match word.to_ascii_uppercase().as_str() { "AND" => Token::And, @@ -586,4 +629,51 @@ mod tests { matches!(err, DynamoDbError::ValidationException(ref msg) if msg.contains("Syntax error")) ); } + + /// The token and window the service reports for a character outside the + /// grammar (measured 2026-10-25), as `(expression, token, near)`. + const UNKNOWN_CHARACTER_ROWS: &[(&str, &str, &str)] = &[ + ("#v = ééé", "é", "= éé"), + ("#v = é", "é", "= é"), + ("é = :v", "é", "é ="), + ("#v = :v AND é", "é", "AND é"), + ("#v = é #v", "é", "= é #v"), + ("#v = 日本語", "日", "= 日本"), + ("#v = 😀x", "😀", "= 😀x"), + ("#v = $x", "$", "= $x"), + ("#v = @", "@", "= @"), + ("#v = aé", "é", "aé"), + ("abéc = :v", "é", "abéc"), + ("#longname = é", "é", "= é"), + ("#v = é é", "é", "= é é"), + ("(#v = é)", "é", "= é)"), + ("contains(#v, é)", "é", ", é)"), + ("#v = :v !", "!", ":v !"), + ("#v =\té", "é", "=\té"), + ("#v =é", "é", "=é"), + ("pk = :v AND é", "é", "AND é"), + ("SET #v = éé", "é", "= éé"), + ("#v, $x", "$", ", $x"), + ]; + + #[test] + fn unknown_character_reports_the_character_and_the_service_window() { + for (expr, token, near) in UNKNOWN_CHARACTER_ROWS { + let expected = format!("Syntax error; token: \"{token}\", near: \"{near}\""); + let Err(DynamoDbError::ValidationException(msg)) = tokenize(expr) else { + panic!("{expr}: expected a syntax error"); + }; + assert_eq!(msg, expected, "{expr}"); + let Err(DynamoDbError::ValidationException(msg)) = + tokenize_for(expr, 4096, super::super::ExpressionKind::Filter) + else { + panic!("{expr}: expected a syntax error"); + }; + assert_eq!( + msg, + format!("Invalid FilterExpression: {expected}"), + "{expr}" + ); + } + } } From 91e1fe71b306147dc2606621ac028b7d500543e0 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Sun, 25 Oct 2026 16:15:56 +0000 Subject: [PATCH 2/5] refactor(expression): move the tokenizer tests into their own file Pure move, using the #[path] pattern evaluator_tests.rs uses; tokenizer.rs had grown past 600 lines with the previous commit. --- crates/core/src/expression/tokenizer.rs | 244 +----------------- crates/core/src/expression/tokenizer_tests.rs | 242 +++++++++++++++++ 2 files changed, 244 insertions(+), 242 deletions(-) create mode 100644 crates/core/src/expression/tokenizer_tests.rs diff --git a/crates/core/src/expression/tokenizer.rs b/crates/core/src/expression/tokenizer.rs index 13e55caed..a1143ea9f 100755 --- a/crates/core/src/expression/tokenizer.rs +++ b/crates/core/src/expression/tokenizer.rs @@ -435,245 +435,5 @@ fn validation_err(msg: &str) -> DynamoDbError { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn tokenize_simple_comparison() { - let tokens = tokenize("price > :min").unwrap(); - assert_eq!( - tokens, - vec![ - Token::Ident("price".into()), - Token::Gt, - Token::Placeholder("min".into()), - ] - ); - } - - #[test] - fn tokenize_name_ref_and_placeholder() { - let tokens = tokenize("#n = :v").unwrap(); - assert_eq!( - tokens, - vec![ - Token::NameRef("n".into()), - Token::Eq, - Token::Placeholder("v".into()) - ] - ); - } - - #[test] - fn tokenize_all_operators() { - let tokens = tokenize("= <> < <= > >=").unwrap(); - assert_eq!( - tokens, - vec![ - Token::Eq, - Token::Ne, - Token::Lt, - Token::Le, - Token::Gt, - Token::Ge - ] - ); - } - - #[test] - fn tokenize_keywords_case_insensitive() { - let tokens = tokenize("SET REMOVE add DELETE And Or Not Between In").unwrap(); - assert_eq!( - tokens, - vec![ - Token::Set, - Token::Remove, - Token::Add, - Token::Delete, - Token::And, - Token::Or, - Token::Not, - Token::Between, - Token::In, - ] - ); - } - - #[test] - fn tokenize_path_with_index() { - let tokens = tokenize("items[0].name").unwrap(); - assert_eq!( - tokens, - vec![ - Token::Ident("items".into()), - Token::LBracket, - Token::Ident("0".into()), - Token::RBracket, - Token::Dot, - Token::Ident("name".into()), - ] - ); - } - - #[test] - fn tokenize_arithmetic() { - let tokens = tokenize("price + :tax - :discount").unwrap(); - assert_eq!( - tokens, - vec![ - Token::Ident("price".into()), - Token::Plus, - Token::Placeholder("tax".into()), - Token::Minus, - Token::Placeholder("discount".into()), - ] - ); - } - - #[test] - fn tokenize_empty_name_ref_rejected() { - let err = tokenize("# = :v").unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("empty name reference")) - ); - } - - #[test] - fn tokenize_empty_placeholder_rejected() { - let err = tokenize("a = :").unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("empty value placeholder")) - ); - } - - #[test] - fn tokenize_invalid_character_rejected() { - let err = tokenize("a @ b").unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("Syntax error; token:")) - ); - } - - #[test] - fn tokenize_exceeds_token_limit() { - let err = tokenize_with_limit("a = b AND c = d", 3).unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("maximum token count")) - ); - } - - #[test] - fn tokenize_whitespace_variants() { - let tokens = tokenize("a\t=\n:v\r").unwrap(); - assert_eq!( - tokens, - vec![ - Token::Ident("a".into()), - Token::Eq, - Token::Placeholder("v".into()) - ] - ); - } - - #[test] - fn tokenize_empty_input() { - let tokens = tokenize("").unwrap(); - assert!(tokens.is_empty()); - } - - #[test] - fn tokenize_function_call() { - let tokens = tokenize("attribute_exists(#pk)").unwrap(); - assert_eq!( - tokens, - vec![ - Token::Ident("attribute_exists".into()), - Token::LParen, - Token::NameRef("pk".into()), - Token::RParen, - ] - ); - } - - #[test] - fn tokenize_underscore_ident() { - let tokens = tokenize("_my_attr = :v").unwrap(); - assert_eq!( - tokens, - vec![ - Token::Ident("_my_attr".into()), - Token::Eq, - Token::Placeholder("v".into()), - ] - ); - } - - #[test] - fn tokenize_for_multibyte_utf8_does_not_panic() { - // Emoji (4-byte UTF-8) should produce a validation error, not a panic. - let err = - tokenize_for("a = 😀", 4096, super::super::ExpressionKind::Condition).unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(ref msg) if msg.contains("Syntax error")) - ); - - // Accented character (2-byte UTF-8) - let err = tokenize_for("café", 4096, super::super::ExpressionKind::Filter).unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(ref msg) if msg.contains("Syntax error")) - ); - - // Multi-byte at the very end - let err = tokenize_for("x + ñ", 4096, super::super::ExpressionKind::Update).unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(ref msg) if msg.contains("Syntax error")) - ); - } - - /// The token and window the service reports for a character outside the - /// grammar (measured 2026-10-25), as `(expression, token, near)`. - const UNKNOWN_CHARACTER_ROWS: &[(&str, &str, &str)] = &[ - ("#v = ééé", "é", "= éé"), - ("#v = é", "é", "= é"), - ("é = :v", "é", "é ="), - ("#v = :v AND é", "é", "AND é"), - ("#v = é #v", "é", "= é #v"), - ("#v = 日本語", "日", "= 日本"), - ("#v = 😀x", "😀", "= 😀x"), - ("#v = $x", "$", "= $x"), - ("#v = @", "@", "= @"), - ("#v = aé", "é", "aé"), - ("abéc = :v", "é", "abéc"), - ("#longname = é", "é", "= é"), - ("#v = é é", "é", "= é é"), - ("(#v = é)", "é", "= é)"), - ("contains(#v, é)", "é", ", é)"), - ("#v = :v !", "!", ":v !"), - ("#v =\té", "é", "=\té"), - ("#v =é", "é", "=é"), - ("pk = :v AND é", "é", "AND é"), - ("SET #v = éé", "é", "= éé"), - ("#v, $x", "$", ", $x"), - ]; - - #[test] - fn unknown_character_reports_the_character_and_the_service_window() { - for (expr, token, near) in UNKNOWN_CHARACTER_ROWS { - let expected = format!("Syntax error; token: \"{token}\", near: \"{near}\""); - let Err(DynamoDbError::ValidationException(msg)) = tokenize(expr) else { - panic!("{expr}: expected a syntax error"); - }; - assert_eq!(msg, expected, "{expr}"); - let Err(DynamoDbError::ValidationException(msg)) = - tokenize_for(expr, 4096, super::super::ExpressionKind::Filter) - else { - panic!("{expr}: expected a syntax error"); - }; - assert_eq!( - msg, - format!("Invalid FilterExpression: {expected}"), - "{expr}" - ); - } - } -} +#[path = "tokenizer_tests.rs"] +mod tests; diff --git a/crates/core/src/expression/tokenizer_tests.rs b/crates/core/src/expression/tokenizer_tests.rs new file mode 100644 index 000000000..1bcdc7bd0 --- /dev/null +++ b/crates/core/src/expression/tokenizer_tests.rs @@ -0,0 +1,242 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +use super::*; + +#[test] +fn tokenize_simple_comparison() { + let tokens = tokenize("price > :min").unwrap(); + assert_eq!( + tokens, + vec![ + Token::Ident("price".into()), + Token::Gt, + Token::Placeholder("min".into()), + ] + ); +} + +#[test] +fn tokenize_name_ref_and_placeholder() { + let tokens = tokenize("#n = :v").unwrap(); + assert_eq!( + tokens, + vec![ + Token::NameRef("n".into()), + Token::Eq, + Token::Placeholder("v".into()) + ] + ); +} + +#[test] +fn tokenize_all_operators() { + let tokens = tokenize("= <> < <= > >=").unwrap(); + assert_eq!( + tokens, + vec![ + Token::Eq, + Token::Ne, + Token::Lt, + Token::Le, + Token::Gt, + Token::Ge + ] + ); +} + +#[test] +fn tokenize_keywords_case_insensitive() { + let tokens = tokenize("SET REMOVE add DELETE And Or Not Between In").unwrap(); + assert_eq!( + tokens, + vec![ + Token::Set, + Token::Remove, + Token::Add, + Token::Delete, + Token::And, + Token::Or, + Token::Not, + Token::Between, + Token::In, + ] + ); +} + +#[test] +fn tokenize_path_with_index() { + let tokens = tokenize("items[0].name").unwrap(); + assert_eq!( + tokens, + vec![ + Token::Ident("items".into()), + Token::LBracket, + Token::Ident("0".into()), + Token::RBracket, + Token::Dot, + Token::Ident("name".into()), + ] + ); +} + +#[test] +fn tokenize_arithmetic() { + let tokens = tokenize("price + :tax - :discount").unwrap(); + assert_eq!( + tokens, + vec![ + Token::Ident("price".into()), + Token::Plus, + Token::Placeholder("tax".into()), + Token::Minus, + Token::Placeholder("discount".into()), + ] + ); +} + +#[test] +fn tokenize_empty_name_ref_rejected() { + let err = tokenize("# = :v").unwrap_err(); + assert!( + matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("empty name reference")) + ); +} + +#[test] +fn tokenize_empty_placeholder_rejected() { + let err = tokenize("a = :").unwrap_err(); + assert!( + matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("empty value placeholder")) + ); +} + +#[test] +fn tokenize_invalid_character_rejected() { + let err = tokenize("a @ b").unwrap_err(); + assert!( + matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("Syntax error; token:")) + ); +} + +#[test] +fn tokenize_exceeds_token_limit() { + let err = tokenize_with_limit("a = b AND c = d", 3).unwrap_err(); + assert!( + matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("maximum token count")) + ); +} + +#[test] +fn tokenize_whitespace_variants() { + let tokens = tokenize("a\t=\n:v\r").unwrap(); + assert_eq!( + tokens, + vec![ + Token::Ident("a".into()), + Token::Eq, + Token::Placeholder("v".into()) + ] + ); +} + +#[test] +fn tokenize_empty_input() { + let tokens = tokenize("").unwrap(); + assert!(tokens.is_empty()); +} + +#[test] +fn tokenize_function_call() { + let tokens = tokenize("attribute_exists(#pk)").unwrap(); + assert_eq!( + tokens, + vec![ + Token::Ident("attribute_exists".into()), + Token::LParen, + Token::NameRef("pk".into()), + Token::RParen, + ] + ); +} + +#[test] +fn tokenize_underscore_ident() { + let tokens = tokenize("_my_attr = :v").unwrap(); + assert_eq!( + tokens, + vec![ + Token::Ident("_my_attr".into()), + Token::Eq, + Token::Placeholder("v".into()), + ] + ); +} + +#[test] +fn tokenize_for_multibyte_utf8_does_not_panic() { + // Emoji (4-byte UTF-8) should produce a validation error, not a panic. + let err = tokenize_for("a = 😀", 4096, super::super::ExpressionKind::Condition).unwrap_err(); + assert!( + matches!(err, DynamoDbError::ValidationException(ref msg) if msg.contains("Syntax error")) + ); + + // Accented character (2-byte UTF-8) + let err = tokenize_for("café", 4096, super::super::ExpressionKind::Filter).unwrap_err(); + assert!( + matches!(err, DynamoDbError::ValidationException(ref msg) if msg.contains("Syntax error")) + ); + + // Multi-byte at the very end + let err = tokenize_for("x + ñ", 4096, super::super::ExpressionKind::Update).unwrap_err(); + assert!( + matches!(err, DynamoDbError::ValidationException(ref msg) if msg.contains("Syntax error")) + ); +} + +/// The token and window the service reports for a character outside the +/// grammar (measured 2026-10-25), as `(expression, token, near)`. +const UNKNOWN_CHARACTER_ROWS: &[(&str, &str, &str)] = &[ + ("#v = ééé", "é", "= éé"), + ("#v = é", "é", "= é"), + ("é = :v", "é", "é ="), + ("#v = :v AND é", "é", "AND é"), + ("#v = é #v", "é", "= é #v"), + ("#v = 日本語", "日", "= 日本"), + ("#v = 😀x", "😀", "= 😀x"), + ("#v = $x", "$", "= $x"), + ("#v = @", "@", "= @"), + ("#v = aé", "é", "aé"), + ("abéc = :v", "é", "abéc"), + ("#longname = é", "é", "= é"), + ("#v = é é", "é", "= é é"), + ("(#v = é)", "é", "= é)"), + ("contains(#v, é)", "é", ", é)"), + ("#v = :v !", "!", ":v !"), + ("#v =\té", "é", "=\té"), + ("#v =é", "é", "=é"), + ("pk = :v AND é", "é", "AND é"), + ("SET #v = éé", "é", "= éé"), + ("#v, $x", "$", ", $x"), +]; + +#[test] +fn unknown_character_reports_the_character_and_the_service_window() { + for (expr, token, near) in UNKNOWN_CHARACTER_ROWS { + let expected = format!("Syntax error; token: \"{token}\", near: \"{near}\""); + let Err(DynamoDbError::ValidationException(msg)) = tokenize(expr) else { + panic!("{expr}: expected a syntax error"); + }; + assert_eq!(msg, expected, "{expr}"); + let Err(DynamoDbError::ValidationException(msg)) = + tokenize_for(expr, 4096, super::super::ExpressionKind::Filter) + else { + panic!("{expr}: expected a syntax error"); + }; + assert_eq!( + msg, + format!("Invalid FilterExpression: {expected}"), + "{expr}" + ); + } +} From 8ae42251d0155ce7f84f0d47cc7396785f5f2e44 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Sun, 25 Oct 2026 16:22:57 +0000 Subject: [PATCH 3/5] test(expression): wire tests for a character outside the expression grammar Every row is a request measured against the service on 2026-10-25, each sent twice: Scan filters with multi-byte and ASCII characters outside the grammar in each position, and one or two rows each for PutItem conditions, Query key conditions, UpdateItem update expressions and GetItem projections. --- tests/test_expression_unknown_character.py | 170 +++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 tests/test_expression_unknown_character.py diff --git a/tests/test_expression_unknown_character.py b/tests/test_expression_unknown_character.py new file mode 100644 index 000000000..136347eb9 --- /dev/null +++ b/tests/test_expression_unknown_character.py @@ -0,0 +1,170 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""A character outside the expression grammar. + +Amazon DynamoDB answers ``Invalid : Syntax error; token: "", near: +""``: the token is the character as sent, a whole code point, and the +window runs from the start of the previous token to the end of the next one, +with each character outside the grammar counted as one token. The write paths +carry a ``1 validation error detected: `` prefix that ExtendDB does not emit; +those assertions compare the text after it. + +Every expected text was measured against Amazon DynamoDB on 2026-10-25 +(us-east-1), each request twice. + +Dual-target against Amazon DynamoDB and extenddb. +""" + +from __future__ import annotations + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +PREFIX = "1 validation error detected: " +NAMES = {"#v": "v"} +VALUES = {":v": {"S": "x"}} +ABSENT = {"pk": {"S": "absent"}, "sk": {"S": "absent"}} + + +def _message(func) -> str: + with pytest.raises(ClientError) as exc_info: + func() + err = exc_info.value.response["Error"] + assert err["Code"] == "ValidationException", err + msg = err["Message"] + return msg[len(PREFIX) :] if msg.startswith(PREFIX) else msg + + +def syntax(kind: str, token: str, near: str) -> str: + return f'Invalid {kind}: Syntax error; token: "{token}", near: "{near}"' + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + dynamodb_client.put_item( + TableName=name, Item={"pk": {"S": "p1"}, "sk": {"S": "s1"}, "v": {"S": "x"}} + ) + yield name + + +@pytest.mark.parametrize( + "expr, token, near", + [ + ("#v = \u00e9\u00e9\u00e9", "\u00e9", "= \u00e9\u00e9"), + ("#v = \u00e9", "\u00e9", "= \u00e9"), + ("\u00e9 = :v", "\u00e9", "\u00e9 ="), + ("#v = :v AND \u00e9", "\u00e9", "AND \u00e9"), + ("#v = \u00e9 #v", "\u00e9", "= \u00e9 #v"), + ("#v = \u65e5\u672c\u8a9e", "\u65e5", "= \u65e5\u672c"), + ("#v = \U0001F600x", "\U0001F600", "= \U0001F600x"), + ("#v = $x", "$", "= $x"), + ("#v = @", "@", "= @"), + ("#v = a\u00e9", "\u00e9", "a\u00e9"), + ("ab\u00e9c = :v", "\u00e9", "ab\u00e9c"), + ("#longname = \u00e9", "\u00e9", "= \u00e9"), + ("#v = \u00e9 \u00e9", "\u00e9", "= \u00e9 \u00e9"), + ("(#v = \u00e9)", "\u00e9", "= \u00e9)"), + ("contains(#v, \u00e9)", "\u00e9", ", \u00e9)"), + ("#v = :v !", "!", ":v !"), + ("#v =\t\u00e9", "\u00e9", "=\t\u00e9"), + ("#v =\u00e9", "\u00e9", "=\u00e9"), + ], +) +def test_scan_filter(dynamodb_client, table, expr, token, near): + msg = _message( + lambda: dynamodb_client.scan( + TableName=table, + Limit=1, + FilterExpression=expr, + ExpressionAttributeNames=NAMES, + ExpressionAttributeValues=VALUES, + ) + ) + assert msg == syntax("FilterExpression", token, near) + + +@pytest.mark.parametrize( + "condition, token, near", + [ + ("#v = \u00e9\u00e9\u00e9", "\u00e9", "= \u00e9\u00e9"), + ("#v = $x", "$", "= $x"), + ], +) +def test_put_item_condition(dynamodb_client, table, condition, token, near): + msg = _message( + lambda: dynamodb_client.put_item( + TableName=table, + Item=ABSENT, + ConditionExpression="attribute_exists(pk) AND " + condition, + ExpressionAttributeNames=NAMES, + ExpressionAttributeValues=VALUES, + ) + ) + assert msg == syntax("ConditionExpression", token, near) + + +@pytest.mark.parametrize( + "kce, token, near", + [ + ("pk = \u00e9", "\u00e9", "= \u00e9"), + ("pk = :v AND \u00e9", "\u00e9", "AND \u00e9"), + ], +) +def test_query_key_condition(dynamodb_client, table, kce, token, near): + msg = _message( + lambda: dynamodb_client.query( + TableName=table, Limit=1, KeyConditionExpression=kce, ExpressionAttributeValues=VALUES + ) + ) + assert msg == syntax("KeyConditionExpression", token, near) + + +@pytest.mark.parametrize( + "update, token, near", + [ + ("SET #v = \u00e9\u00e9", "\u00e9", "= \u00e9\u00e9"), + ("SET #v = $x", "$", "= $x"), + ], +) +def test_update_expression(dynamodb_client, table, update, token, near): + msg = _message( + lambda: dynamodb_client.update_item( + TableName=table, + Key=ABSENT, + UpdateExpression=update, + ConditionExpression="attribute_exists(pk)", + ExpressionAttributeNames=NAMES, + ) + ) + assert msg == syntax("UpdateExpression", token, near) + + +@pytest.mark.parametrize( + "projection, token, near", + [ + ("#v, \u00e9", "\u00e9", ", \u00e9"), + ("#v, $x", "$", ", $x"), + ], +) +def test_projection_expression(dynamodb_client, table, projection, token, near): + msg = _message( + lambda: dynamodb_client.get_item( + TableName=table, Key=ABSENT, ProjectionExpression=projection, + ExpressionAttributeNames=NAMES, + ) + ) + assert msg == syntax("ProjectionExpression", token, near) From 5a02bf673c62e5b2b73dfb1a8158aa65f89cf5a8 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Sun, 25 Oct 2026 17:23:01 +0000 Subject: [PATCH 4/5] fix(expression): start the window at the start of the expression and drop U+0000, as the service does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When no token comes before the character, the service starts the window at the start of the expression and keeps the leading whitespace (` é = :v` answers `near: " é ="`), and it removes U+0000 from the token and the window (`#v = \0x` answers `token: "", near: "= x"`). Measured 2026-10-25 at 16:34, 16:38 and 16:43 UTC, each request twice. The unit table gains those rows and the rows that pin the next-token rules for two-character operators, name runs and digit runs. --- crates/core/src/expression/tokenizer.rs | 14 ++++++++-- crates/core/src/expression/tokenizer_tests.rs | 26 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/crates/core/src/expression/tokenizer.rs b/crates/core/src/expression/tokenizer.rs index a1143ea9f..2809709c9 100755 --- a/crates/core/src/expression/tokenizer.rs +++ b/crates/core/src/expression/tokenizer.rs @@ -378,13 +378,23 @@ fn is_ident_char(c: u8) -> bool { /// `token: "é", near: "= éé"`, and a name stops at the first such character, /// so `abéc = :v` answers `near: "abéc"`. /// +/// When no token comes before the character the window starts at the start +/// of the expression, leading whitespace kept (` é = :v` answers +/// `near: " é ="`), and U+0000 is removed from the token and the window +/// (`#v = \0x` answers `token: "", near: "= x"`). +/// /// `prev_start` is the byte offset where the previous token starts. fn unknown_character(input: &str, at: usize, prev_start: Option) -> (String, String) { let ch = input[at..].chars().next().unwrap_or('?'); let after = at + ch.len_utf8(); let end = next_token_end(input, after).unwrap_or(after); - let start = prev_start.unwrap_or(at); - (ch.to_string(), input[start..end].to_owned()) + let start = prev_start.unwrap_or(0); + let token = if ch == '\0' { + String::new() + } else { + ch.to_string() + }; + (token, input[start..end].replace('\0', "")) } /// The byte offset where the next token after `from` ends, skipping the diff --git a/crates/core/src/expression/tokenizer_tests.rs b/crates/core/src/expression/tokenizer_tests.rs index 1bcdc7bd0..e8b2cf8a8 100644 --- a/crates/core/src/expression/tokenizer_tests.rs +++ b/crates/core/src/expression/tokenizer_tests.rs @@ -218,6 +218,32 @@ const UNKNOWN_CHARACTER_ROWS: &[(&str, &str, &str)] = &[ ("pk = :v AND é", "é", "AND é"), ("SET #v = éé", "é", "= éé"), ("#v, $x", "$", ", $x"), + // No token before the character: the window starts at the start. + (" é = :v", "é", " é ="), + ("\té = :v", "é", "\té ="), + ("\n\né = :v", "é", "\n\né ="), + (" é", "é", " é"), + (" $ = :v", "$", " $ ="), + (" éé = :v", "é", " éé"), + (" é SET #v = :v", "é", " é SET"), + (" #v = :v AND é", "é", "AND é"), + (" (é)", "é", "(é)"), + // U+0000 is removed from the token and the window. + ("#v = :v AND \0", "", "AND "), + ("#v = \0x", "", "= x"), + ("\0#v = :v", "", "#v"), + ("#v = é\0", "é", "= é"), + ("#v\0 = :v", "", "#v ="), + ("#v = :v AND \0\0", "", "AND "), + ("a\0b = :v", "", "ab"), + // The next token: two-character operators, a name run, a digit run. + ("#v é<> :v", "é", "#v é<>"), + ("#v é<= :v", "é", "#v é<="), + ("#v é>= :v", "é", "#v é>="), + ("#v <>é", "é", "<>é"), + ("#v = é abc", "é", "= é abc"), + ("#v[é12] = :v", "é", "[é12"), + ("#v BETWEEN é AND :v", "é", "BETWEEN é AND"), ]; #[test] From 4b079afa99e5e96e918a67f2af50510230274b8e Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Sun, 25 Oct 2026 17:33:04 +0000 Subject: [PATCH 5/5] test(expression): wire rows for the window start, U+0000, the next-token rules and the Query paths Rows measured on 2026-10-25 at 16:34, 16:38, 16:43 and 16:54 UTC, each request twice: a character with no token before it, U+0000, two-character operators, name runs and digit runs after the character, and the panic shape on a Query key condition, a Query filter and an UpdateItem condition. The prefix the service adds on write paths is stripped only on those paths. --- tests/test_expression_unknown_character.py | 87 ++++++++++++++++++---- 1 file changed, 74 insertions(+), 13 deletions(-) diff --git a/tests/test_expression_unknown_character.py b/tests/test_expression_unknown_character.py index 136347eb9..3b8ace02a 100644 --- a/tests/test_expression_unknown_character.py +++ b/tests/test_expression_unknown_character.py @@ -3,12 +3,14 @@ """A character outside the expression grammar. -Amazon DynamoDB answers ``Invalid : Syntax error; token: "", near: -""``: the token is the character as sent, a whole code point, and the -window runs from the start of the previous token to the end of the next one, -with each character outside the grammar counted as one token. The write paths -carry a ``1 validation error detected: `` prefix that ExtendDB does not emit; -those assertions compare the text after it. +When the character is the first fault in the expression, Amazon DynamoDB +answers ``Invalid : Syntax error; token: "", near: ""``: the +token is the character as sent, a whole code point, and the window runs from +the start of the previous token (or of the expression, when no token comes +before it) to the end of the next one, with each character outside the +grammar counted as one token. U+0000 is removed from the token and the window. +The write paths carry a ``1 validation error detected: `` prefix that +ExtendDB does not emit; those assertions compare the text after it. Every expected text was measured against Amazon DynamoDB on 2026-10-25 (us-east-1), each request twice. @@ -34,7 +36,10 @@ def _message(func) -> str: func() err = exc_info.value.response["Error"] assert err["Code"] == "ValidationException", err - msg = err["Message"] + return err["Message"] + + +def _after_prefix(msg: str) -> str: return msg[len(PREFIX) :] if msg.startswith(PREFIX) else msg @@ -82,6 +87,20 @@ def table(dynamodb_client): ("#v = :v !", "!", ":v !"), ("#v =\t\u00e9", "\u00e9", "=\t\u00e9"), ("#v =\u00e9", "\u00e9", "=\u00e9"), + # No token before the character: the window starts at the start. + (" \u00e9 = :v", "\u00e9", " \u00e9 ="), + ("\t\u00e9 = :v", "\u00e9", "\t\u00e9 ="), + (" $ = :v", "$", " $ ="), + (" (\u00e9)", "\u00e9", "(\u00e9)"), + # U+0000 is removed from the token and the window. + ("#v = :v AND \u0000", "", "AND "), + ("#v = \u0000x", "", "= x"), + ("a\u0000b = :v", "", "ab"), + # The next token: two-character operators, a name run, a digit run. + ("#v \u00e9<> :v", "\u00e9", "#v \u00e9<>"), + ("#v \u00e9>= :v", "\u00e9", "#v \u00e9>="), + ("#v = \u00e9 abc", "\u00e9", "= \u00e9 abc"), + ("#v[\u00e912] = :v", "\u00e9", "[\u00e912"), ], ) def test_scan_filter(dynamodb_client, table, expr, token, near): @@ -114,7 +133,35 @@ def test_put_item_condition(dynamodb_client, table, condition, token, near): ExpressionAttributeValues=VALUES, ) ) - assert msg == syntax("ConditionExpression", token, near) + assert _after_prefix(msg) == syntax("ConditionExpression", token, near) + + +def test_update_item_condition(dynamodb_client, table): + msg = _message( + lambda: dynamodb_client.update_item( + TableName=table, + Key=ABSENT, + UpdateExpression="SET #v = :v", + ConditionExpression="attribute_exists(pk) AND #v = \u00e9\u00e9\u00e9", + ExpressionAttributeNames=NAMES, + ExpressionAttributeValues=VALUES, + ) + ) + assert _after_prefix(msg) == syntax("ConditionExpression", "\u00e9", "= \u00e9\u00e9") + + +def test_query_filter(dynamodb_client, table): + msg = _message( + lambda: dynamodb_client.query( + TableName=table, + Limit=1, + KeyConditionExpression="pk = :v", + FilterExpression="#v = \u00e9\u00e9\u00e9", + ExpressionAttributeNames=NAMES, + ExpressionAttributeValues=VALUES, + ) + ) + assert msg == syntax("FilterExpression", "\u00e9", "= \u00e9\u00e9") @pytest.mark.parametrize( @@ -122,6 +169,11 @@ def test_put_item_condition(dynamodb_client, table, condition, token, near): [ ("pk = \u00e9", "\u00e9", "= \u00e9"), ("pk = :v AND \u00e9", "\u00e9", "AND \u00e9"), + # The server reads the key condition before authorization. + ("pk = \u00e9\u00e9\u00e9", "\u00e9", "= \u00e9\u00e9"), + (" \u00e9 = :v", "\u00e9", " \u00e9 ="), + ("pk = \u0000", "", "= "), + ("pk \u00e9<= :v", "\u00e9", "pk \u00e9<="), ], ) def test_query_key_condition(dynamodb_client, table, kce, token, near): @@ -134,13 +186,17 @@ def test_query_key_condition(dynamodb_client, table, kce, token, near): @pytest.mark.parametrize( - "update, token, near", + "update, token, near, values", [ - ("SET #v = \u00e9\u00e9", "\u00e9", "= \u00e9\u00e9"), - ("SET #v = $x", "$", "= $x"), + ("SET #v = \u00e9\u00e9", "\u00e9", "= \u00e9\u00e9", None), + ("SET #v = $x", "$", "= $x", None), + (" \u00e9 SET #v = :v", "\u00e9", " \u00e9 SET", VALUES), + ("SET #v = \u0000", "", "= ", VALUES), + ("SET #v = :v \u00e9<> :v", "\u00e9", ":v \u00e9<>", VALUES), ], ) -def test_update_expression(dynamodb_client, table, update, token, near): +def test_update_expression(dynamodb_client, table, update, token, near, values): + extra = {"ExpressionAttributeValues": values} if values else {} msg = _message( lambda: dynamodb_client.update_item( TableName=table, @@ -148,9 +204,10 @@ def test_update_expression(dynamodb_client, table, update, token, near): UpdateExpression=update, ConditionExpression="attribute_exists(pk)", ExpressionAttributeNames=NAMES, + **extra, ) ) - assert msg == syntax("UpdateExpression", token, near) + assert _after_prefix(msg) == syntax("UpdateExpression", token, near) @pytest.mark.parametrize( @@ -158,6 +215,10 @@ def test_update_expression(dynamodb_client, table, update, token, near): [ ("#v, \u00e9", "\u00e9", ", \u00e9"), ("#v, $x", "$", ", $x"), + ("#v, \u00e9\u00e9\u00e9", "\u00e9", ", \u00e9\u00e9"), + (" \u00e9", "\u00e9", " \u00e9"), + ("#v, \u0000", "", ", "), + ("#v \u00e9>=", "\u00e9", "#v \u00e9>="), ], ) def test_projection_expression(dynamodb_client, table, projection, token, near):