diff --git a/crates/core/src/expression/tokenizer.rs b/crates/core/src/expression/tokenizer.rs index ca57fd833..2809709c9 100755 --- a/crates/core/src/expression/tokenizer.rs +++ b/crates/core/src/expression/tokenizer.rs @@ -64,7 +64,10 @@ pub fn tokenize_with_limit(input: &str, max_tokens: usize) -> Result, let bytes = input.as_bytes(); let mut i = 0; + let mut prev_start: Option = None; while i < bytes.len() { + let tok_start = i; + let before = tokens.len(); match bytes[i] { b' ' | b'\t' | b'\n' | b'\r' => i += 1, b'(' => { @@ -178,16 +181,16 @@ pub fn tokenize_with_limit(input: &str, max_tokens: usize) -> Result, max_tokens, )?; } - other => { - let ch = other as char; - let near_start = i.saturating_sub(5); - let near_end = std::cmp::min(i + 5, input.len()); - let near = &input[near_start..near_end]; + _ => { + let (ch, near) = unknown_character(input, i, prev_start); return Err(validation_err(&format!( "Syntax error; token: \"{ch}\", near: \"{near}\"" ))); } } + if tokens.len() > before { + prev_start = Some(tok_start); + } } Ok(tokens) @@ -211,7 +214,10 @@ pub fn tokenize_for( let bytes = input.as_bytes(); let mut i = 0; + let mut prev_start: Option = None; while i < bytes.len() { + let tok_start = i; + let before = tokens.len(); match bytes[i] { b' ' | b'\t' | b'\n' | b'\r' => i += 1, b'(' => { @@ -325,27 +331,15 @@ pub fn tokenize_for( )?; } _ => { - // Safety: use chars() to safely extract the character at this - // byte position, avoiding panics on multi-byte UTF-8 input. - let ch = input[i..].chars().next().unwrap_or('?'); - let ch_len = ch.len_utf8(); - let near_end = std::cmp::min(i + ch_len + 1, input.len()); - // Clamp near_end to a valid char boundary - let near_end = if near_end <= input.len() { - let mut end = near_end; - while end < input.len() && !input.is_char_boundary(end) { - end += 1; - } - if end > input.len() { input.len() } else { end } - } else { - input.len() - }; - let near = &input[i..near_end]; + let (ch, near) = unknown_character(input, i, prev_start); return Err(DynamoDbError::ValidationException(format!( "Invalid {kind}: Syntax error; token: \"{ch}\", near: \"{near}\"" ))); } } + if tokens.len() > before { + prev_start = Some(tok_start); + } } Ok(tokens) } @@ -372,6 +366,65 @@ fn is_ident_char(c: u8) -> bool { c.is_ascii_alphanumeric() || c == b'_' } +/// The token text and the `near` window the service reports for a character +/// the expression grammar does not accept, at byte offset `at` of `input`. +/// +/// Measured 2026-10-25 on FilterExpression, ConditionExpression, +/// KeyConditionExpression, UpdateExpression and ProjectionExpression: the +/// token is the character as sent (a whole code point, `é` or an emoji), and +/// the window is the source text from the start of the previous token to the +/// end of the next one, whitespace kept, each dropped when absent. Every +/// character outside the grammar is a token of its own, so `#v = ééé` answers +/// `token: "é", near: "= éé"`, and a name stops at the first such character, +/// so `abéc = :v` answers `near: "abéc"`. +/// +/// When no token comes before the character the window starts at the start +/// of the expression, leading whitespace kept (` é = :v` answers +/// `near: " é ="`), and U+0000 is removed from the token and the window +/// (`#v = \0x` answers `token: "", near: "= x"`). +/// +/// `prev_start` is the byte offset where the previous token starts. +fn unknown_character(input: &str, at: usize, prev_start: Option) -> (String, String) { + let ch = input[at..].chars().next().unwrap_or('?'); + let after = at + ch.len_utf8(); + let end = next_token_end(input, after).unwrap_or(after); + let start = prev_start.unwrap_or(0); + let token = if ch == '\0' { + String::new() + } else { + ch.to_string() + }; + (token, input[start..end].replace('\0', "")) +} + +/// The byte offset where the next token after `from` ends, skipping the +/// tokenizer's whitespace, or `None` at the end of input. Mirrors the token +/// rules of [`tokenize_with_limit`], with a character outside the grammar +/// counted as one token. +fn next_token_end(input: &str, from: usize) -> Option { + let bytes = input.as_bytes(); + let mut i = from; + while i < bytes.len() && matches!(bytes[i], b' ' | b'\t' | b'\n' | b'\r') { + i += 1; + } + let first = *bytes.get(i)?; + let run = |mut j: usize, pred: fn(u8) -> bool| { + while j < bytes.len() && pred(bytes[j]) { + j += 1; + } + j + }; + Some(match first { + b'<' if matches!(bytes.get(i + 1), Some(b'>' | b'=')) => i + 2, + b'>' if bytes.get(i + 1) == Some(&b'=') => i + 2, + b'(' | b')' | b'[' | b']' | b',' | b'.' | b'+' | b'-' | b'=' | b'<' | b'>' => i + 1, + b'#' | b':' => run(i + 1, is_ident_char), + c if is_ident_start(c) => run(i, is_ident_char), + c if c.is_ascii_digit() => run(i, |b| b.is_ascii_digit()), + _ => i + input[i..].chars().next().map_or(1, char::len_utf8), + }) +} + fn keyword_or_ident(word: &str) -> Token { match word.to_ascii_uppercase().as_str() { "AND" => Token::And, @@ -392,198 +445,5 @@ fn validation_err(msg: &str) -> DynamoDbError { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn tokenize_simple_comparison() { - let tokens = tokenize("price > :min").unwrap(); - assert_eq!( - tokens, - vec![ - Token::Ident("price".into()), - Token::Gt, - Token::Placeholder("min".into()), - ] - ); - } - - #[test] - fn tokenize_name_ref_and_placeholder() { - let tokens = tokenize("#n = :v").unwrap(); - assert_eq!( - tokens, - vec![ - Token::NameRef("n".into()), - Token::Eq, - Token::Placeholder("v".into()) - ] - ); - } - - #[test] - fn tokenize_all_operators() { - let tokens = tokenize("= <> < <= > >=").unwrap(); - assert_eq!( - tokens, - vec![ - Token::Eq, - Token::Ne, - Token::Lt, - Token::Le, - Token::Gt, - Token::Ge - ] - ); - } - - #[test] - fn tokenize_keywords_case_insensitive() { - let tokens = tokenize("SET REMOVE add DELETE And Or Not Between In").unwrap(); - assert_eq!( - tokens, - vec![ - Token::Set, - Token::Remove, - Token::Add, - Token::Delete, - Token::And, - Token::Or, - Token::Not, - Token::Between, - Token::In, - ] - ); - } - - #[test] - fn tokenize_path_with_index() { - let tokens = tokenize("items[0].name").unwrap(); - assert_eq!( - tokens, - vec![ - Token::Ident("items".into()), - Token::LBracket, - Token::Ident("0".into()), - Token::RBracket, - Token::Dot, - Token::Ident("name".into()), - ] - ); - } - - #[test] - fn tokenize_arithmetic() { - let tokens = tokenize("price + :tax - :discount").unwrap(); - assert_eq!( - tokens, - vec![ - Token::Ident("price".into()), - Token::Plus, - Token::Placeholder("tax".into()), - Token::Minus, - Token::Placeholder("discount".into()), - ] - ); - } - - #[test] - fn tokenize_empty_name_ref_rejected() { - let err = tokenize("# = :v").unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("empty name reference")) - ); - } - - #[test] - fn tokenize_empty_placeholder_rejected() { - let err = tokenize("a = :").unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("empty value placeholder")) - ); - } - - #[test] - fn tokenize_invalid_character_rejected() { - let err = tokenize("a @ b").unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("Syntax error; token:")) - ); - } - - #[test] - fn tokenize_exceeds_token_limit() { - let err = tokenize_with_limit("a = b AND c = d", 3).unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("maximum token count")) - ); - } - - #[test] - fn tokenize_whitespace_variants() { - let tokens = tokenize("a\t=\n:v\r").unwrap(); - assert_eq!( - tokens, - vec![ - Token::Ident("a".into()), - Token::Eq, - Token::Placeholder("v".into()) - ] - ); - } - - #[test] - fn tokenize_empty_input() { - let tokens = tokenize("").unwrap(); - assert!(tokens.is_empty()); - } - - #[test] - fn tokenize_function_call() { - let tokens = tokenize("attribute_exists(#pk)").unwrap(); - assert_eq!( - tokens, - vec![ - Token::Ident("attribute_exists".into()), - Token::LParen, - Token::NameRef("pk".into()), - Token::RParen, - ] - ); - } - - #[test] - fn tokenize_underscore_ident() { - let tokens = tokenize("_my_attr = :v").unwrap(); - assert_eq!( - tokens, - vec![ - Token::Ident("_my_attr".into()), - Token::Eq, - Token::Placeholder("v".into()), - ] - ); - } - - #[test] - fn tokenize_for_multibyte_utf8_does_not_panic() { - // Emoji (4-byte UTF-8) should produce a validation error, not a panic. - let err = - tokenize_for("a = 😀", 4096, super::super::ExpressionKind::Condition).unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(ref msg) if msg.contains("Syntax error")) - ); - - // Accented character (2-byte UTF-8) - let err = tokenize_for("café", 4096, super::super::ExpressionKind::Filter).unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(ref msg) if msg.contains("Syntax error")) - ); - - // Multi-byte at the very end - let err = tokenize_for("x + ñ", 4096, super::super::ExpressionKind::Update).unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(ref msg) if msg.contains("Syntax error")) - ); - } -} +#[path = "tokenizer_tests.rs"] +mod tests; diff --git a/crates/core/src/expression/tokenizer_tests.rs b/crates/core/src/expression/tokenizer_tests.rs new file mode 100644 index 000000000..e8b2cf8a8 --- /dev/null +++ b/crates/core/src/expression/tokenizer_tests.rs @@ -0,0 +1,268 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +use super::*; + +#[test] +fn tokenize_simple_comparison() { + let tokens = tokenize("price > :min").unwrap(); + assert_eq!( + tokens, + vec![ + Token::Ident("price".into()), + Token::Gt, + Token::Placeholder("min".into()), + ] + ); +} + +#[test] +fn tokenize_name_ref_and_placeholder() { + let tokens = tokenize("#n = :v").unwrap(); + assert_eq!( + tokens, + vec![ + Token::NameRef("n".into()), + Token::Eq, + Token::Placeholder("v".into()) + ] + ); +} + +#[test] +fn tokenize_all_operators() { + let tokens = tokenize("= <> < <= > >=").unwrap(); + assert_eq!( + tokens, + vec![ + Token::Eq, + Token::Ne, + Token::Lt, + Token::Le, + Token::Gt, + Token::Ge + ] + ); +} + +#[test] +fn tokenize_keywords_case_insensitive() { + let tokens = tokenize("SET REMOVE add DELETE And Or Not Between In").unwrap(); + assert_eq!( + tokens, + vec![ + Token::Set, + Token::Remove, + Token::Add, + Token::Delete, + Token::And, + Token::Or, + Token::Not, + Token::Between, + Token::In, + ] + ); +} + +#[test] +fn tokenize_path_with_index() { + let tokens = tokenize("items[0].name").unwrap(); + assert_eq!( + tokens, + vec![ + Token::Ident("items".into()), + Token::LBracket, + Token::Ident("0".into()), + Token::RBracket, + Token::Dot, + Token::Ident("name".into()), + ] + ); +} + +#[test] +fn tokenize_arithmetic() { + let tokens = tokenize("price + :tax - :discount").unwrap(); + assert_eq!( + tokens, + vec![ + Token::Ident("price".into()), + Token::Plus, + Token::Placeholder("tax".into()), + Token::Minus, + Token::Placeholder("discount".into()), + ] + ); +} + +#[test] +fn tokenize_empty_name_ref_rejected() { + let err = tokenize("# = :v").unwrap_err(); + assert!( + matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("empty name reference")) + ); +} + +#[test] +fn tokenize_empty_placeholder_rejected() { + let err = tokenize("a = :").unwrap_err(); + assert!( + matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("empty value placeholder")) + ); +} + +#[test] +fn tokenize_invalid_character_rejected() { + let err = tokenize("a @ b").unwrap_err(); + assert!( + matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("Syntax error; token:")) + ); +} + +#[test] +fn tokenize_exceeds_token_limit() { + let err = tokenize_with_limit("a = b AND c = d", 3).unwrap_err(); + assert!( + matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("maximum token count")) + ); +} + +#[test] +fn tokenize_whitespace_variants() { + let tokens = tokenize("a\t=\n:v\r").unwrap(); + assert_eq!( + tokens, + vec![ + Token::Ident("a".into()), + Token::Eq, + Token::Placeholder("v".into()) + ] + ); +} + +#[test] +fn tokenize_empty_input() { + let tokens = tokenize("").unwrap(); + assert!(tokens.is_empty()); +} + +#[test] +fn tokenize_function_call() { + let tokens = tokenize("attribute_exists(#pk)").unwrap(); + assert_eq!( + tokens, + vec![ + Token::Ident("attribute_exists".into()), + Token::LParen, + Token::NameRef("pk".into()), + Token::RParen, + ] + ); +} + +#[test] +fn tokenize_underscore_ident() { + let tokens = tokenize("_my_attr = :v").unwrap(); + assert_eq!( + tokens, + vec![ + Token::Ident("_my_attr".into()), + Token::Eq, + Token::Placeholder("v".into()), + ] + ); +} + +#[test] +fn tokenize_for_multibyte_utf8_does_not_panic() { + // Emoji (4-byte UTF-8) should produce a validation error, not a panic. + let err = tokenize_for("a = 😀", 4096, super::super::ExpressionKind::Condition).unwrap_err(); + assert!( + matches!(err, DynamoDbError::ValidationException(ref msg) if msg.contains("Syntax error")) + ); + + // Accented character (2-byte UTF-8) + let err = tokenize_for("café", 4096, super::super::ExpressionKind::Filter).unwrap_err(); + assert!( + matches!(err, DynamoDbError::ValidationException(ref msg) if msg.contains("Syntax error")) + ); + + // Multi-byte at the very end + let err = tokenize_for("x + ñ", 4096, super::super::ExpressionKind::Update).unwrap_err(); + assert!( + matches!(err, DynamoDbError::ValidationException(ref msg) if msg.contains("Syntax error")) + ); +} + +/// The token and window the service reports for a character outside the +/// grammar (measured 2026-10-25), as `(expression, token, near)`. +const UNKNOWN_CHARACTER_ROWS: &[(&str, &str, &str)] = &[ + ("#v = ééé", "é", "= éé"), + ("#v = é", "é", "= é"), + ("é = :v", "é", "é ="), + ("#v = :v AND é", "é", "AND é"), + ("#v = é #v", "é", "= é #v"), + ("#v = 日本語", "日", "= 日本"), + ("#v = 😀x", "😀", "= 😀x"), + ("#v = $x", "$", "= $x"), + ("#v = @", "@", "= @"), + ("#v = aé", "é", "aé"), + ("abéc = :v", "é", "abéc"), + ("#longname = é", "é", "= é"), + ("#v = é é", "é", "= é é"), + ("(#v = é)", "é", "= é)"), + ("contains(#v, é)", "é", ", é)"), + ("#v = :v !", "!", ":v !"), + ("#v =\té", "é", "=\té"), + ("#v =é", "é", "=é"), + ("pk = :v AND é", "é", "AND é"), + ("SET #v = éé", "é", "= éé"), + ("#v, $x", "$", ", $x"), + // No token before the character: the window starts at the start. + (" é = :v", "é", " é ="), + ("\té = :v", "é", "\té ="), + ("\n\né = :v", "é", "\n\né ="), + (" é", "é", " é"), + (" $ = :v", "$", " $ ="), + (" éé = :v", "é", " éé"), + (" é SET #v = :v", "é", " é SET"), + (" #v = :v AND é", "é", "AND é"), + (" (é)", "é", "(é)"), + // U+0000 is removed from the token and the window. + ("#v = :v AND \0", "", "AND "), + ("#v = \0x", "", "= x"), + ("\0#v = :v", "", "#v"), + ("#v = é\0", "é", "= é"), + ("#v\0 = :v", "", "#v ="), + ("#v = :v AND \0\0", "", "AND "), + ("a\0b = :v", "", "ab"), + // The next token: two-character operators, a name run, a digit run. + ("#v é<> :v", "é", "#v é<>"), + ("#v é<= :v", "é", "#v é<="), + ("#v é>= :v", "é", "#v é>="), + ("#v <>é", "é", "<>é"), + ("#v = é abc", "é", "= é abc"), + ("#v[é12] = :v", "é", "[é12"), + ("#v BETWEEN é AND :v", "é", "BETWEEN é AND"), +]; + +#[test] +fn unknown_character_reports_the_character_and_the_service_window() { + for (expr, token, near) in UNKNOWN_CHARACTER_ROWS { + let expected = format!("Syntax error; token: \"{token}\", near: \"{near}\""); + let Err(DynamoDbError::ValidationException(msg)) = tokenize(expr) else { + panic!("{expr}: expected a syntax error"); + }; + assert_eq!(msg, expected, "{expr}"); + let Err(DynamoDbError::ValidationException(msg)) = + tokenize_for(expr, 4096, super::super::ExpressionKind::Filter) + else { + panic!("{expr}: expected a syntax error"); + }; + assert_eq!( + msg, + format!("Invalid FilterExpression: {expected}"), + "{expr}" + ); + } +} diff --git a/tests/test_expression_unknown_character.py b/tests/test_expression_unknown_character.py new file mode 100644 index 000000000..3b8ace02a --- /dev/null +++ b/tests/test_expression_unknown_character.py @@ -0,0 +1,231 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""A character outside the expression grammar. + +When the character is the first fault in the expression, Amazon DynamoDB +answers ``Invalid : Syntax error; token: "", near: ""``: the +token is the character as sent, a whole code point, and the window runs from +the start of the previous token (or of the expression, when no token comes +before it) to the end of the next one, with each character outside the +grammar counted as one token. U+0000 is removed from the token and the window. +The write paths carry a ``1 validation error detected: `` prefix that +ExtendDB does not emit; those assertions compare the text after it. + +Every expected text was measured against Amazon DynamoDB on 2026-10-25 +(us-east-1), each request twice. + +Dual-target against Amazon DynamoDB and extenddb. +""" + +from __future__ import annotations + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +PREFIX = "1 validation error detected: " +NAMES = {"#v": "v"} +VALUES = {":v": {"S": "x"}} +ABSENT = {"pk": {"S": "absent"}, "sk": {"S": "absent"}} + + +def _message(func) -> str: + with pytest.raises(ClientError) as exc_info: + func() + err = exc_info.value.response["Error"] + assert err["Code"] == "ValidationException", err + return err["Message"] + + +def _after_prefix(msg: str) -> str: + return msg[len(PREFIX) :] if msg.startswith(PREFIX) else msg + + +def syntax(kind: str, token: str, near: str) -> str: + return f'Invalid {kind}: Syntax error; token: "{token}", near: "{near}"' + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + dynamodb_client.put_item( + TableName=name, Item={"pk": {"S": "p1"}, "sk": {"S": "s1"}, "v": {"S": "x"}} + ) + yield name + + +@pytest.mark.parametrize( + "expr, token, near", + [ + ("#v = \u00e9\u00e9\u00e9", "\u00e9", "= \u00e9\u00e9"), + ("#v = \u00e9", "\u00e9", "= \u00e9"), + ("\u00e9 = :v", "\u00e9", "\u00e9 ="), + ("#v = :v AND \u00e9", "\u00e9", "AND \u00e9"), + ("#v = \u00e9 #v", "\u00e9", "= \u00e9 #v"), + ("#v = \u65e5\u672c\u8a9e", "\u65e5", "= \u65e5\u672c"), + ("#v = \U0001F600x", "\U0001F600", "= \U0001F600x"), + ("#v = $x", "$", "= $x"), + ("#v = @", "@", "= @"), + ("#v = a\u00e9", "\u00e9", "a\u00e9"), + ("ab\u00e9c = :v", "\u00e9", "ab\u00e9c"), + ("#longname = \u00e9", "\u00e9", "= \u00e9"), + ("#v = \u00e9 \u00e9", "\u00e9", "= \u00e9 \u00e9"), + ("(#v = \u00e9)", "\u00e9", "= \u00e9)"), + ("contains(#v, \u00e9)", "\u00e9", ", \u00e9)"), + ("#v = :v !", "!", ":v !"), + ("#v =\t\u00e9", "\u00e9", "=\t\u00e9"), + ("#v =\u00e9", "\u00e9", "=\u00e9"), + # No token before the character: the window starts at the start. + (" \u00e9 = :v", "\u00e9", " \u00e9 ="), + ("\t\u00e9 = :v", "\u00e9", "\t\u00e9 ="), + (" $ = :v", "$", " $ ="), + (" (\u00e9)", "\u00e9", "(\u00e9)"), + # U+0000 is removed from the token and the window. + ("#v = :v AND \u0000", "", "AND "), + ("#v = \u0000x", "", "= x"), + ("a\u0000b = :v", "", "ab"), + # The next token: two-character operators, a name run, a digit run. + ("#v \u00e9<> :v", "\u00e9", "#v \u00e9<>"), + ("#v \u00e9>= :v", "\u00e9", "#v \u00e9>="), + ("#v = \u00e9 abc", "\u00e9", "= \u00e9 abc"), + ("#v[\u00e912] = :v", "\u00e9", "[\u00e912"), + ], +) +def test_scan_filter(dynamodb_client, table, expr, token, near): + msg = _message( + lambda: dynamodb_client.scan( + TableName=table, + Limit=1, + FilterExpression=expr, + ExpressionAttributeNames=NAMES, + ExpressionAttributeValues=VALUES, + ) + ) + assert msg == syntax("FilterExpression", token, near) + + +@pytest.mark.parametrize( + "condition, token, near", + [ + ("#v = \u00e9\u00e9\u00e9", "\u00e9", "= \u00e9\u00e9"), + ("#v = $x", "$", "= $x"), + ], +) +def test_put_item_condition(dynamodb_client, table, condition, token, near): + msg = _message( + lambda: dynamodb_client.put_item( + TableName=table, + Item=ABSENT, + ConditionExpression="attribute_exists(pk) AND " + condition, + ExpressionAttributeNames=NAMES, + ExpressionAttributeValues=VALUES, + ) + ) + assert _after_prefix(msg) == syntax("ConditionExpression", token, near) + + +def test_update_item_condition(dynamodb_client, table): + msg = _message( + lambda: dynamodb_client.update_item( + TableName=table, + Key=ABSENT, + UpdateExpression="SET #v = :v", + ConditionExpression="attribute_exists(pk) AND #v = \u00e9\u00e9\u00e9", + ExpressionAttributeNames=NAMES, + ExpressionAttributeValues=VALUES, + ) + ) + assert _after_prefix(msg) == syntax("ConditionExpression", "\u00e9", "= \u00e9\u00e9") + + +def test_query_filter(dynamodb_client, table): + msg = _message( + lambda: dynamodb_client.query( + TableName=table, + Limit=1, + KeyConditionExpression="pk = :v", + FilterExpression="#v = \u00e9\u00e9\u00e9", + ExpressionAttributeNames=NAMES, + ExpressionAttributeValues=VALUES, + ) + ) + assert msg == syntax("FilterExpression", "\u00e9", "= \u00e9\u00e9") + + +@pytest.mark.parametrize( + "kce, token, near", + [ + ("pk = \u00e9", "\u00e9", "= \u00e9"), + ("pk = :v AND \u00e9", "\u00e9", "AND \u00e9"), + # The server reads the key condition before authorization. + ("pk = \u00e9\u00e9\u00e9", "\u00e9", "= \u00e9\u00e9"), + (" \u00e9 = :v", "\u00e9", " \u00e9 ="), + ("pk = \u0000", "", "= "), + ("pk \u00e9<= :v", "\u00e9", "pk \u00e9<="), + ], +) +def test_query_key_condition(dynamodb_client, table, kce, token, near): + msg = _message( + lambda: dynamodb_client.query( + TableName=table, Limit=1, KeyConditionExpression=kce, ExpressionAttributeValues=VALUES + ) + ) + assert msg == syntax("KeyConditionExpression", token, near) + + +@pytest.mark.parametrize( + "update, token, near, values", + [ + ("SET #v = \u00e9\u00e9", "\u00e9", "= \u00e9\u00e9", None), + ("SET #v = $x", "$", "= $x", None), + (" \u00e9 SET #v = :v", "\u00e9", " \u00e9 SET", VALUES), + ("SET #v = \u0000", "", "= ", VALUES), + ("SET #v = :v \u00e9<> :v", "\u00e9", ":v \u00e9<>", VALUES), + ], +) +def test_update_expression(dynamodb_client, table, update, token, near, values): + extra = {"ExpressionAttributeValues": values} if values else {} + msg = _message( + lambda: dynamodb_client.update_item( + TableName=table, + Key=ABSENT, + UpdateExpression=update, + ConditionExpression="attribute_exists(pk)", + ExpressionAttributeNames=NAMES, + **extra, + ) + ) + assert _after_prefix(msg) == syntax("UpdateExpression", token, near) + + +@pytest.mark.parametrize( + "projection, token, near", + [ + ("#v, \u00e9", "\u00e9", ", \u00e9"), + ("#v, $x", "$", ", $x"), + ("#v, \u00e9\u00e9\u00e9", "\u00e9", ", \u00e9\u00e9"), + (" \u00e9", "\u00e9", " \u00e9"), + ("#v, \u0000", "", ", "), + ("#v \u00e9>=", "\u00e9", "#v \u00e9>="), + ], +) +def test_projection_expression(dynamodb_client, table, projection, token, near): + msg = _message( + lambda: dynamodb_client.get_item( + TableName=table, Key=ABSENT, ProjectionExpression=projection, + ExpressionAttributeNames=NAMES, + ) + ) + assert msg == syntax("ProjectionExpression", token, near)