Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions .github/workflows/integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -300,17 +300,18 @@ jobs:
# storage layer directly against this job's PostgreSQL. The PutItem create
# race tests hold a create open in an outside transaction, the
# write-transaction conflict tests force a deadlock, and the backup restore
# cases need catalog rows the current binary would not write; all need
# direct access. The online GSI build tests reorder queue rows by hand and
# pause writers with a test hook. They build their own throwaway databases;
# the connection string is the server, not a database.
# cases need catalog rows the current binary would not write, and the
# DELETING window tests hold a table DELETING with no poller to finish it;
# all need direct access. The online GSI build tests reorder queue rows by
# hand and pause writers with a test hook. They build their own throwaway
# databases; the connection string is the server, not a database.
- name: Run PostgreSQL storage-level tests
env:
EXTENDDB_TEST_PG_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432
run: >-
cargo test --release -p extenddb-storage-postgres
--test vector_control_plane --test put_create_race --test twi_conflict
--test stream_shard_retention --test backup_restore
--test stream_shard_retention --test backup_restore --test deleting_window
--test gsi_online_build --test gsi_stale_writer

# The daemonized server logs to syslog; dump it so server-side failures
Expand Down
10 changes: 10 additions & 0 deletions crates/core/src/error/messages.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ pub enum ErrorMessageKey {
TableNotFound,
TableAlreadyExists,
TableInUse,
/// A control-plane change refused because the table is `DELETING`.
/// Measured on the service 2026-10-24 for a second `DeleteTable` and for
/// `UpdateTable` during the window.
TableBeingDeleted,
MissingKeySchema,
KeySchemaTooMany,
KeySchemaFirstNotHash,
Expand Down Expand Up @@ -82,6 +86,12 @@ pub fn error_message(key: ErrorMessageKey, args: &[&str]) -> String {
ErrorMessageKey::TableInUse => {
format!("Table is being created, updated, or deleted: {}", args[0])
}
ErrorMessageKey::TableBeingDeleted => {
format!(
"Attempt to change a resource which is still in use: Table is being deleted: {}",
args[0]
)
}
ErrorMessageKey::MissingKeySchema => {
"1 validation error detected: Value null at 'keySchema' failed to satisfy constraint: \
Member must not be null"
Expand Down
3 changes: 3 additions & 0 deletions crates/engine/src/create_table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,9 @@ pub fn storage_err_to_dynamo(e: extenddb_storage::error::StorageError) -> Dynamo
ErrorMessageKey::TableInUse,
&[&name],
)),
StorageError::TableBeingDeleted(name) => DynamoDbError::ResourceInUseException(
error_message(ErrorMessageKey::TableBeingDeleted, &[&name]),
),
// Only RestoreTableFromBackup raises this; its own mapping makes it
// TableInUseException. Elsewhere (a plain CreateTable into a name
// that is a CREATING restore target) the service's text is the one
Expand Down
25 changes: 25 additions & 0 deletions crates/engine/src/delete_table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,28 @@ pub async fn handle_delete_table(
};
serialize_output(&output)
}

#[cfg(test)]
mod tests {
use super::*;
use extenddb_storage::error::StorageError;

/// A second DeleteTable during the DELETING window: the measured sentence
/// (2026-10-24), byte for byte. The other non-ACTIVE statuses keep theirs.
#[test]
fn a_deleting_table_has_its_own_sentence() {
match storage_err_to_dynamo(StorageError::TableBeingDeleted("t".to_owned())) {
DynamoDbError::ResourceInUseException(msg) => assert_eq!(
msg,
"Attempt to change a resource which is still in use: Table is being deleted: t"
),
other => panic!("expected ResourceInUseException, got {other:?}"),
}
match storage_err_to_dynamo(StorageError::TableNotActive("t".to_owned())) {
DynamoDbError::ResourceInUseException(msg) => {
assert_eq!(msg, "Table is being created, updated, or deleted: t");
}
other => panic!("expected ResourceInUseException, got {other:?}"),
}
}
}
28 changes: 28 additions & 0 deletions crates/engine/src/ttl.rs
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,15 @@ fn storage_to_dynamo(e: StorageError) -> DynamoDbError {
StorageError::TableNotActive(name) => {
DynamoDbError::ResourceInUseException(format!("Table {name} is not in ACTIVE state"))
}
// No TTL path raises this today (the TTL metadata paths answer
// TableNotActive for every non-ACTIVE status), but the variant exists
// and a backend that raises it here must not become a 500.
StorageError::TableBeingDeleted(name) => {
DynamoDbError::ResourceInUseException(extenddb_core::error::error_message(
extenddb_core::error::ErrorMessageKey::TableBeingDeleted,
&[&name],
))
}
other => {
tracing::error!(internal_error = %other, "storage internal error");
DynamoDbError::InternalServerError("Internal server error".to_owned())
Expand All @@ -237,6 +246,25 @@ fn storage_to_dynamo(e: StorageError) -> DynamoDbError {
mod tests {
use super::*;

/// The TTL mapper renders a DELETING table with the service's sentence and
/// keeps the other non-ACTIVE statuses on theirs.
#[test]
fn a_deleting_table_has_its_own_sentence() {
match storage_to_dynamo(StorageError::TableBeingDeleted("t".to_owned())) {
DynamoDbError::ResourceInUseException(msg) => assert_eq!(
msg,
"Attempt to change a resource which is still in use: Table is being deleted: t"
),
other => panic!("expected ResourceInUseException, got {other:?}"),
}
match storage_to_dynamo(StorageError::TableNotActive("t".to_owned())) {
DynamoDbError::ResourceInUseException(msg) => {
assert_eq!(msg, "Table t is not in ACTIVE state");
}
other => panic!("expected ResourceInUseException, got {other:?}"),
}
}

#[test]
fn valid_ttl_attribute_names() {
assert!(validate_ttl_attribute_name("ttl").is_ok());
Expand Down
30 changes: 30 additions & 0 deletions crates/engine/src/update_table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,15 @@ fn update_table_err_to_dynamo(
StorageError::TableNotActive(name) => {
DynamoDbError::ResourceInUseException(format!("Table {name} is not in ACTIVE state"))
}
// The service words a DELETING table differently from the other
// non-ACTIVE statuses (measured 2026-10-24); the sentence lives with
// the engine's other message templates.
StorageError::TableBeingDeleted(name) => {
DynamoDbError::ResourceInUseException(extenddb_core::error::error_message(
extenddb_core::error::ErrorMessageKey::TableBeingDeleted,
&[&name],
))
}
StorageError::IndexAlreadyExists(name) => DynamoDbError::ValidationException(format!(
"One or more parameter values were invalid: Index already exists: {name}"
)),
Expand Down Expand Up @@ -500,4 +509,25 @@ mod tests {
update_table_err_to_dynamo(StorageError::Internal("disk on fire".to_owned()), "t");
assert!(matches!(err, DynamoDbError::InternalServerError(_)));
}

/// UpdateTable on a DELETING table: the measured sentence (2026-10-24),
/// byte for byte, and the other non-ACTIVE statuses keep theirs.
#[test]
fn a_deleting_table_has_its_own_sentence() {
let err = update_table_err_to_dynamo(StorageError::TableBeingDeleted("t".to_owned()), "t");
match err {
DynamoDbError::ResourceInUseException(msg) => assert_eq!(
msg,
"Attempt to change a resource which is still in use: Table is being deleted: t"
),
other => panic!("expected ResourceInUseException, got {other:?}"),
}
let err = update_table_err_to_dynamo(StorageError::TableNotActive("t".to_owned()), "t");
match err {
DynamoDbError::ResourceInUseException(msg) => {
assert_eq!(msg, "Table t is not in ACTIVE state");
}
other => panic!("expected ResourceInUseException, got {other:?}"),
}
}
}
9 changes: 5 additions & 4 deletions crates/server/src/key_info_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -160,10 +160,11 @@ impl CachedTableKeyInfoStore {
/// cached and is what the caller sees.
///
/// Every backend's `table_key_info` answers `TableNotFound` both for a name
/// the catalog does not have and for one it has in a state other than
/// `ACTIVE` (a `CreateTable` or `RestoreTableFromBackup` target that is still
/// `CREATING`; the data plane reports both as `ResourceNotFoundException`, as
/// the service does). Caching the second as a negative entry made reads on a
/// the catalog does not have and for one it has as `CREATING` (a `CreateTable`
/// or `RestoreTableFromBackup` target; the data plane reports both as
/// `ResourceNotFoundException`, as the service does). PostgreSQL and SQLite
/// answer the key info for a `DELETING` row, since the service serves the data
/// plane until the table is gone. Caching the second as a negative entry made reads on a
/// restored table fail for up to `negative_ttl` after `DescribeTable` reported
/// it `ACTIVE`: a probe during the copy stored the entry, and nothing
/// invalidated it at the transition (the service's first
Expand Down
14 changes: 12 additions & 2 deletions crates/storage-cassandra/src/delete_table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,17 @@ impl CassandraEngine {

match caller {
DeleteTableCaller::Client => {
// A DELETING row belongs to the deletion already under way
// (the abandoned-restore sweep sets it on this backend). The
// service refuses a second DeleteTable in that window as a
// table still in use (measured 2026-10-24); checked before
// deletion protection, as the SQL backends order it.
let status: String =
crate::cassandra_util::get_column(table_row, "table_status", "delete_table")?;
if status == "DELETING" {
return Err(StorageError::TableBeingDeleted(input.table_name.clone()));
}

// Check deletion protection
let deletion_protection: bool = table_row
.get_r_by_name("deletion_protection_enabled")
Expand All @@ -94,12 +105,11 @@ impl CassandraEngine {
// A restore target is owned by its copy until the copy
// activates it (or the sweep removes it). Deleting it mid-copy
// would race the copy's writes into dropped tables.
let status: Option<String> = table_row.get_by_name("table_status").ok().flatten();
let restoring_from: Option<String> = table_row
.get_by_name("restore_source_backup_arn")
.ok()
.flatten();
if status.as_deref() == Some("CREATING") && restoring_from.is_some() {
if status == "CREATING" && restoring_from.is_some() {
return Err(table_being_restored(&input.table_name));
}
}
Expand Down
5 changes: 5 additions & 0 deletions crates/storage-cassandra/src/update_table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ impl CassandraEngine {

let status: String =
crate::cassandra_util::get_column(&row, "table_status", "update_table")?;
// A DELETING table has its own sentence on the service (measured
// 2026-10-24); every other non-ACTIVE status keeps the existing one.
if status == "DELETING" {
return Err(StorageError::TableBeingDeleted(input.table_name.clone()));
}
if status != "ACTIVE" {
return Err(StorageError::TableNotActive(input.table_name.clone()));
}
Expand Down
Loading
Loading