From 98bca8ecb71e678e12c47a0bcecc45f0b2bbc0b3 Mon Sep 17 00:00:00 2001 From: Lee Hannigan Date: Mon, 26 Oct 2026 10:11:23 +0000 Subject: [PATCH] fix: answer TTL and tag calls during a table's DELETING window as the service does DescribeTimeToLive on a DELETING table answers 400 ValidationException "Cannot describe time to live while table is in DELETING state: Current table state is DELETING", and on a CREATING table the same sentence naming CREATING; it is served on ACTIVE and UPDATING tables. UpdateTimeToLive on a DELETING table runs the same rules as on an ACTIVE one. Amazon DynamoDB answers these calls this way (measured 2026-10-25 and 2026-10-26). On ExtendDB the UpdateTimeToLive write goes to the catalog row, which the delete removes. ExtendDB answered UpdateTimeToLive on a DELETING table with 400 ResourceInUseException "Table is not in ACTIVE state", because every backend's update_ttl admitted ACTIVE rows only, and DescribeTimeToLive with 200 on CREATING and DELETING tables, because nothing read the status. The engine reads the status with describe_table, since TableKeyInfo carries no status and MongoDB and Cassandra report a DELETING row as not found from the key-info read. The read is four catalog queries on PostgreSQL and SQLite and two on MongoDB and Cassandra, once per call. update_ttl admits DELETING beside ACTIVE in the conditional write on PostgreSQL, SQLite and MongoDB, and in the status check and both lightweight transactions on Cassandra. CREATING and UPDATING keep TableNotActive, and a missing table keeps TableNotFound. UpdateTimeToLive sends no index DDL for a table it read as DELETING, since the delete removes the data table or collection with its index. On PostgreSQL a disable's DROP INDEX waited with no lock timeout behind whatever held the data table and queued the table's data-plane requests behind it. On MongoDB an enable's create_index could create the collection again after the delete dropped it, and the collection outlived the table. MongoDB create_ttl_index also reads an ACTIVE catalog document only. A delete that commits between the status read and the write meets the index work, which the base refused at the write; on MongoDB the ACTIVE filter covers that order. TagResource and UntagResource on the ARN of a DELETING table answer 400 ResourceInUseException "Attempt to change a resource which is still in use: Table is being deleted: ", and ListTagsOfResource is served with the tags. On the table's stream ARN both writes and the read are served. Amazon DynamoDB answers this way (measured 2026-10-25 and 2026-10-26). ExtendDB answered ResourceNotFoundException for all three on MongoDB and Cassandra and accepted both writes on PostgreSQL and SQLite. An accepted TagResource on the table ARN writes an unconditioned tag row; on PostgreSQL the delete removes the ARN's tag rows before the catalog row, so a tag written between the two statements could outlive the table and be inherited by the next table of that name. The tag path reads the table's status before the key-info read, so all four backends reach the window's answers, and refuses the two writes when the ARN names the table. The key-info read still follows for every status but DELETING, so a tag call on an ACTIVE table sends seven catalog queries before its tag statements on PostgreSQL and SQLite, where it sent three, and four on MongoDB and Cassandra, where it sent two. The refusal removes the writes admitted during the window. A write whose status read came before the delete can still be written after the delete removed the tag rows, as before. ListTagsOfResource returns the tags for the whole window on PostgreSQL and SQLite. On MongoDB it returns no tags once the delete call has removed them; the service returns them for the whole window. A MongoDB storage test drives the tag and TTL handlers on a DELETING document, and devtools/run-mongodb-tests runs the crate's ignored deleting_window tests in CI. Closes #552 Closes #551 --- Cargo.lock | 2 + crates/core/src/error/messages.rs | 18 ++ crates/engine/src/lib.rs | 1 + crates/engine/src/table_lifecycle.rs | 64 +++++ crates/engine/src/tagging.rs | 164 ++++++++++- crates/engine/src/ttl.rs | 86 +++++- .../storage-cassandra/src/metadata_engine.rs | 10 +- .../tests/direct/deleting_window.rs | 49 ++++ crates/storage-mongodb/Cargo.toml | 3 + .../src/deleting_window_tests.rs | 258 ++++++++++++++++++ crates/storage-mongodb/src/metadata_engine.rs | 17 +- crates/storage-postgres/Cargo.toml | 3 + .../storage-postgres/src/metadata_engine.rs | 7 +- .../storage-postgres/tests/deleting_window.rs | 3 + .../tests/deleting_window/ttl.rs | 180 ++++++++++++ crates/storage-sqlite/src/delete_table.rs | 80 +++++- crates/storage-sqlite/src/metadata.rs | 7 +- devtools/run-mongodb-tests | 10 + .../test_deleting_window_control_plane.py | 174 +++++++++++- 19 files changed, 1105 insertions(+), 31 deletions(-) create mode 100644 crates/engine/src/table_lifecycle.rs create mode 100644 crates/storage-postgres/tests/deleting_window/ttl.rs diff --git a/Cargo.lock b/Cargo.lock index 003569679..e0ba802e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1448,6 +1448,7 @@ dependencies = [ "dashmap", "extenddb-auth", "extenddb-core", + "extenddb-engine", "extenddb-storage", "futures", "mongodb", @@ -1476,6 +1477,7 @@ dependencies = [ "crc32fast", "extenddb-auth", "extenddb-core", + "extenddb-engine", "extenddb-storage", "futures", "libc", diff --git a/crates/core/src/error/messages.rs b/crates/core/src/error/messages.rs index e5aedc057..8db9ee01b 100755 --- a/crates/core/src/error/messages.rs +++ b/crates/core/src/error/messages.rs @@ -44,6 +44,14 @@ pub enum ErrorMessageKey { /// Measured on the service 2026-10-24 for a second `DeleteTable` and for /// `UpdateTable` during the window. TableBeingDeleted, + /// `DescribeTimeToLive` refused because the table is `DELETING`. Measured + /// on the service 2026-10-25 on three tables, every call inside the window. + /// `UpdateTimeToLive` has no such refusal: it runs its ordinary rules. + DescribeTtlWhileDeleting, + /// `DescribeTimeToLive` refused because the table is `CREATING`: the + /// `DELETING` sentence with the state substituted. Measured on the service + /// 2026-10-26, five calls, each while `DescribeTable` read `CREATING`. + DescribeTtlWhileCreating, MissingKeySchema, KeySchemaTooMany, KeySchemaFirstNotHash, @@ -92,6 +100,16 @@ pub fn error_message(key: ErrorMessageKey, args: &[&str]) -> String { args[0] ) } + ErrorMessageKey::DescribeTtlWhileDeleting => { + "Cannot describe time to live while table is in DELETING state: \ + Current table state is DELETING" + .to_owned() + } + ErrorMessageKey::DescribeTtlWhileCreating => { + "Cannot describe time to live while table is in CREATING state: \ + Current table state is CREATING" + .to_owned() + } ErrorMessageKey::MissingKeySchema => { "1 validation error detected: Value null at 'keySchema' failed to satisfy constraint: \ Member must not be null" diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 74083fbfa..c960fa4c7 100755 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -33,6 +33,7 @@ mod scan; mod search_vectors; pub mod stream_capture; mod streams; +mod table_lifecycle; mod tagging; mod transact_get_items; mod transact_write_helpers; diff --git a/crates/engine/src/table_lifecycle.rs b/crates/engine/src/table_lifecycle.rs new file mode 100644 index 000000000..839df5117 --- /dev/null +++ b/crates/engine/src/table_lifecycle.rs @@ -0,0 +1,64 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The catalog's lifecycle status for a table, for handlers that decide on it. +//! +//! `TableKeyInfo` carries no status, and the key-info read reports a `DELETING` +//! row as not found on MongoDB and Cassandra, so a handler that has to tell a +//! `DELETING` table from a missing one cannot get the answer from it. The +//! description carries the status on every backend. The callers (the two TTL +//! handlers and the tag path) are control-plane calls, so the read is not on a +//! data-plane path. +//! +//! The read is `describe_table`: four catalog queries on PostgreSQL and SQLite +//! (`tables`, `indexes`, `vector_indexes`, `table_restores`) and two on MongoDB +//! and Cassandra (`tables`, `indexes`). `DescribeTimeToLive`, +//! `UpdateTimeToLive`, `TagResource`, `UntagResource` and `ListTagsOfResource` +//! each make it once per call. The tag path keeps the key-info read after it +//! for every status but `DELETING`, so a tag call on an `ACTIVE` table sends +//! seven queries before its tag statements on PostgreSQL and SQLite, where it +//! sent three, and four on MongoDB and Cassandra, where it sent two. +//! `DescribeTimeToLive` on an `ACTIVE` table sends five and three, where it +//! sent one. `UpdateTimeToLive` sends the four or two more, and on a +//! `DELETING` table it skips the TTL read after the write and the index +//! statements. + +use extenddb_core::error::DynamoDbError; +use extenddb_core::types::{DescribeTableInput, TableStatus}; +use extenddb_storage::error::StorageError; + +use crate::OperationContext; + +/// Read `table_name`'s current status from the catalog. +/// +/// # Errors +/// +/// Returns [`StorageError::TableNotFound`] when no catalog row exists, and +/// whatever the backend raised otherwise. The caller maps it, so each operation +/// keeps the message it already answers for a missing table. +pub(crate) async fn read_table_status( + table_name: &str, + ctx: &OperationContext, +) -> Result { + let desc = ctx + .storage + .describe_table( + &ctx.account_id, + DescribeTableInput { + table_name: table_name.to_owned(), + }, + ) + .await?; + Ok(desc.table_status) +} + +/// The service's refusal of a change to a table that is being deleted, worded +/// once in [`extenddb_core::error::ErrorMessageKey::TableBeingDeleted`] and +/// classified the way the control-plane mappers classify +/// [`StorageError::TableBeingDeleted`]. +pub(crate) fn being_deleted(table_name: &str) -> DynamoDbError { + DynamoDbError::ResourceInUseException(extenddb_core::error::error_message( + extenddb_core::error::ErrorMessageKey::TableBeingDeleted, + &[table_name], + )) +} diff --git a/crates/engine/src/tagging.rs b/crates/engine/src/tagging.rs index 432d75dd4..c0520f8b4 100755 --- a/crates/engine/src/tagging.rs +++ b/crates/engine/src/tagging.rs @@ -5,7 +5,8 @@ use extenddb_core::error::DynamoDbError; use extenddb_core::types::{ - ListTagsOfResourceInput, ListTagsOfResourceOutput, TagResourceInput, UntagResourceInput, + ListTagsOfResourceInput, ListTagsOfResourceOutput, TableStatus, TagResourceInput, + UntagResourceInput, }; use serde_json::Value; @@ -34,6 +35,15 @@ fn tag_subresource_not_table_or_stream(resource: &str) -> bool { !matches!(segs.next(), None | Some("") | Some("stream")) } +/// True for a stream ARN, `table//stream/