From bed07ae2122b39e49c7fcdc61a33884aba681526 Mon Sep 17 00:00:00 2001 From: Artem Goncharov Date: Fri, 13 Nov 2026 19:18:52 +0100 Subject: [PATCH] feat(storage): Add replicated keyspace drop Add Mutation::DropKeyspace, committed through Raft and applied on every node, and StorageApi::list_keyspaces(prefix) for cleanup discovery (ADR 0044 step 3). DropKeyspace is accepted only for names matching single_use_{use_case}_{bucket} with a numeric bucket, and only in a transaction made of drops. Apply runs after the batch commits and removes the keyspace, its per-record metadata in the meta keyspace and any ephemeral entry. A missing keyspace is a no-op. The real storage reported the first violation of any type as Conflict, so a rate limit, quarantine or refused drop looked like a replay. Violations other than CONFLICT now surface as the new StoreError::Rejected, which keeps the violation type. Move require_no_conflict from the scim, service-grant and oauth2-client drivers into storage-api. It maps only a CONFLICT violation or error to the caller's conflict and returns Rejected for any other violation. The oauth2-key JTI retry loop uses it and no longer retries a rejected write. Signed-off-by: Artem Goncharov --- crates/oauth2-client-driver-raft/src/lib.rs | 21 +- crates/oauth2-key-driver-raft/src/lib.rs | 35 ++- crates/scim-driver-raft/src/lib.rs | 29 +-- crates/service-grant-driver-raft/src/lib.rs | 17 +- crates/storage-api/src/lib.rs | 168 +++++++++++++ crates/storage-api/src/wire_format_tests.rs | 7 + crates/storage/src/app/admin.rs | 10 +- crates/storage/src/app/storage_api.rs | 4 + crates/storage/src/error.rs | 49 +++- crates/storage/src/grpc/authz.rs | 17 +- crates/storage/src/lib.rs | 1 + crates/storage/src/mock.rs | 43 ++++ crates/storage/src/store/state_machine.rs | 3 + .../src/store/state_machine/keyspace_drop.rs | 229 ++++++++++++++++++ .../src/store/state_machine/raft_sm.rs | 46 ++++ crates/storage/src/store_command.rs | 11 + crates/storage/src/wire_format_tests.rs | 7 + crates/storage/tests/test_cluster/main.rs | 1 + .../storage/tests/test_cluster/single_use.rs | 146 +++++++++++ crates/webauthn/src/driver/raft.rs | 4 + 20 files changed, 758 insertions(+), 90 deletions(-) create mode 100644 crates/storage/src/store/state_machine/keyspace_drop.rs create mode 100644 crates/storage/tests/test_cluster/single_use.rs diff --git a/crates/oauth2-client-driver-raft/src/lib.rs b/crates/oauth2-client-driver-raft/src/lib.rs index 3d242bc02..c1f3a4424 100644 --- a/crates/oauth2-client-driver-raft/src/lib.rs +++ b/crates/oauth2-client-driver-raft/src/lib.rs @@ -25,29 +25,10 @@ use openstack_keystone_core::oauth2_client::backend::Oauth2ClientBackend; use openstack_keystone_core::plugin_manager::BackendRegistration; use openstack_keystone_core_types::oauth2_client::*; use openstack_keystone_distributed_storage::{ - ApiStoreError, Metadata, StorageApi, StoreDataEnvelope, StoreError, StoreResponse, + Metadata, StorageApi, StoreDataEnvelope, StoreError, require_no_conflict, store_command::Mutation, }; -/// Bridges two different backend behaviors for the same semantic event: the -/// real Raft-backed [`StorageApi`] converts the first `Violation` on a -/// `transaction` call into an immediate `Err(ApiStoreError::Conflict)`; -/// `MockStorage`, used by this crate's own unit tests, instead returns every -/// violation as data on an `Ok` response's `violations` field. Both call -/// sites need conflict detection to behave identically against either -/// backend (mirrors `scim-driver-raft`'s helper of the same name). -fn require_no_conflict( - result: Result, - conflict: impl FnOnce() -> StoreError, -) -> Result { - match result { - Ok(response) if response.violations.is_empty() => Ok(response), - Ok(_) => Err(conflict()), - Err(ApiStoreError::Conflict { .. }) => Err(conflict()), - Err(e) => Err(e.into()), - } -} - /// Global `client_id` index value payload: resolves to the owning /// `(domain_id, provider_id)` coordinate. Stored as a /// `Mutation::create_if_absent` data key (not `set_index`, which carries no diff --git a/crates/oauth2-key-driver-raft/src/lib.rs b/crates/oauth2-key-driver-raft/src/lib.rs index 5f4105f2d..c6bddd657 100644 --- a/crates/oauth2-key-driver-raft/src/lib.rs +++ b/crates/oauth2-key-driver-raft/src/lib.rs @@ -29,7 +29,8 @@ use openstack_keystone_core_types::oauth2_key::{ PendingRotationInfo, }; use openstack_keystone_distributed_storage::{ - ApiStoreError as StoreError, Metadata, StorageApi, StoreDataEnvelope, store_command::Mutation, + ApiStoreError as StoreError, Metadata, StorageApi, StoreDataEnvelope, require_no_conflict, + store_command::Mutation, }; use openstack_keystone_key_repository::asymmetric::{ ActiveKeys, AsymmetricKeyRepository, AsymmetricKeySource, KeyMaterial, KeyRole, @@ -786,24 +787,16 @@ impl RaftOauth2KeyBackend { storage.transaction(vec![mutation]).await } }; - match result { - // The real storage surfaces a CAS miss as `Err(Conflict)`; - // the mock returns it as `violations` on an `Ok` response. - Ok(response) if response.violations.is_empty() => return Ok(()), - // Only a CAS/create-if-absent miss is retried; any other - // violation is a real failure, not contention. - Ok(response) => { - if let Some(v) = response - .violations - .iter() - .find(|v| !v.r#type.eq_ignore_ascii_case("CONFLICT")) - { - return Err(persist(StoreError::Conflict { - subject: v.subject.clone(), - description: v.description.clone(), - })); - } - } + // Only a CAS/create-if-absent miss is retried; any other + // violation or error is a real failure, not contention. The + // helper maps both the real store's `Err(Conflict)` and the + // mock's violations-as-data to the same conflict. + let contended = || StoreError::Conflict { + subject: key.clone(), + description: "JTI revocation list contended".to_string(), + }; + match require_no_conflict(result, contended) { + Ok(_) => return Ok(()), Err(StoreError::Conflict { .. }) => {} Err(e) => return Err(persist(e)), } @@ -2320,6 +2313,10 @@ mod tests { self.inner.drop_keyspace(keyspace).await } + async fn list_keyspaces(&self, prefix: &str) -> Result, StoreError> { + self.inner.list_keyspaces(prefix).await + } + async fn node_id(&self) -> u64 { self.inner.node_id().await } diff --git a/crates/scim-driver-raft/src/lib.rs b/crates/scim-driver-raft/src/lib.rs index cadb02e9e..d259edffe 100644 --- a/crates/scim-driver-raft/src/lib.rs +++ b/crates/scim-driver-raft/src/lib.rs @@ -25,33 +25,10 @@ use openstack_keystone_core::scim_resource::backend::ScimResourceBackend; use openstack_keystone_core::scim_resource::error::ScimResourceProviderError; use openstack_keystone_core_types::scim::*; use openstack_keystone_distributed_storage::{ - ApiStoreError, Metadata, StorageApi, StoreDataEnvelope, StoreError, StoreResponse, + ApiStoreError, Metadata, StorageApi, StoreDataEnvelope, StoreError, require_no_conflict, store_command::Mutation, }; -/// Bridges two different backend behaviors for the same semantic event. -/// The real Raft-backed [`StorageApi`] impl (`Storage::write_command_to_ -/// storage`) converts the *first* -/// [`Violation`](openstack_keystone_distributed_storage::Violation) -/// on a `transaction`/`set_value` call into an immediate `Err(ApiStoreError:: -/// Conflict)`; `MockStorage`, used by this crate's own unit tests, instead -/// returns every violation as data on an `Ok` response's `violations` field. -/// Both call sites here need conflict detection to work identically against -/// either backend, so this normalizes them into a single `Err(conflict())` -/// on either shape, `Ok(response)` otherwise. Any other error passes through -/// via the ordinary `StoreError::StorageApi` conversion. -fn require_no_conflict( - result: Result, - conflict: impl FnOnce() -> StoreError, -) -> Result { - match result { - Ok(response) if response.violations.is_empty() => Ok(response), - Ok(_) => Err(conflict()), - Err(ApiStoreError::Conflict { .. }) => Err(conflict()), - Err(e) => Err(e.into()), - } -} - /// Raft Database SCIM realm backend. /// /// Realms are indexed directly by their `(domain_id, provider_id)` @@ -1130,6 +1107,10 @@ mod tests { self.inner.drop_keyspace(keyspace).await } + async fn list_keyspaces(&self, prefix: &str) -> Result, ApiStoreError> { + self.inner.list_keyspaces(prefix).await + } + async fn node_id(&self) -> u64 { self.inner.node_id().await } diff --git a/crates/service-grant-driver-raft/src/lib.rs b/crates/service-grant-driver-raft/src/lib.rs index 833c9be2f..b22105f00 100644 --- a/crates/service-grant-driver-raft/src/lib.rs +++ b/crates/service-grant-driver-raft/src/lib.rs @@ -36,7 +36,7 @@ use openstack_keystone_core::service_grant::ServiceGrantProviderError; use openstack_keystone_core::service_grant::backend::ServiceGrantBackend; use openstack_keystone_core_types::service_grant::*; use openstack_keystone_distributed_storage::{ - ApiStoreError, Metadata, StorageApi, StoreDataEnvelope, StoreError, StoreResponse, + Metadata, StorageApi, StoreDataEnvelope, StoreError, require_no_conflict, store_command::Mutation, }; @@ -46,21 +46,6 @@ const GRANTOR_IDX: &str = "service_grant:grantor_idx:v1:"; const GRANTEE_IDX: &str = "service_grant:grantee_idx:v1:"; const PROJECT_IDX: &str = "service_grant:project_idx:v1:"; -/// Bridges the real storage (first violation is an `Err(Conflict)`) and -/// `MockStorage` (violations returned as data); mirrors the helper of the -/// other Raft drivers. -fn require_no_conflict( - result: Result, - conflict: impl FnOnce() -> StoreError, -) -> Result { - match result { - Ok(response) if response.violations.is_empty() => Ok(response), - Ok(_) => Err(conflict()), - Err(ApiStoreError::Conflict { .. }) => Err(conflict()), - Err(e) => Err(e.into()), - } -} - /// A `(domain_id, id)` coordinate recovered from an index key. type Coordinate = (String, String); diff --git a/crates/storage-api/src/lib.rs b/crates/storage-api/src/lib.rs index 738645248..0fb5d38d5 100644 --- a/crates/storage-api/src/lib.rs +++ b/crates/storage-api/src/lib.rs @@ -133,6 +133,21 @@ pub enum StoreError { #[error("deserialization error")] Deserialize(#[from] rmp_serde::decode::Error), + /// The write was refused for a reason other than a conflict with + /// existing state (`WRITE_RATE_EXCEEDED`, `QUARANTINED`, + /// `RESERVED_KEYSPACE`, `KEYSPACE_DROP_REFUSED`, ...). Not a + /// [`Self::Conflict`]: a caller that reads a conflict as "already + /// exists" must not read this as one. + #[error("write rejected ({violation_type}): {subject} — {description}")] + Rejected { + /// The violation type reported by the state machine. + violation_type: String, + /// The resource or key the refusal refers to. + subject: String, + /// Human-readable description. + description: String, + }, + /// Key is already present in the store while the call expects it to be /// unset. #[error("key is already set")] @@ -170,6 +185,49 @@ impl StoreError { } } +/// Returns `true` if a violation type means "the key or revision already +/// conflicts" (ADR 0044 §3). Only this type is a conflict; every other type is +/// a refusal that must not be read as one. +pub fn is_conflict_violation(violation: &Violation) -> bool { + violation.r#type.eq_ignore_ascii_case("CONFLICT") +} + +/// Normalizes the two shapes in which a storage reports a failed write into +/// one: the Raft-backed store returns the first violation as an immediate +/// `Err`, while a mock returns violations as data on an `Ok` response. +/// +/// - no violation: `Ok(response)`; +/// - a conflict (`CONFLICT` violation or `Err(Conflict)`): `Err(conflict())`; +/// - any other violation: `Err(`[`StoreError::Rejected`]`)` converted into +/// `E`, keeping its type, never a conflict; +/// - any other error: passed through converted into `E`. +pub fn require_no_conflict( + result: Result, + conflict: impl FnOnce() -> E, +) -> Result +where + E: From, +{ + match result { + Ok(response) => match response + .violations + .iter() + .find(|v| !is_conflict_violation(v)) + .or(response.violations.first()) + { + None => Ok(response), + Some(v) if is_conflict_violation(v) => Err(conflict()), + Some(v) => Err(E::from(StoreError::Rejected { + violation_type: v.r#type.clone(), + subject: v.subject.clone(), + description: v.description.clone(), + })), + }, + Err(StoreError::Conflict { .. }) => Err(conflict()), + Err(e) => Err(E::from(e)), + } +} + /// Raft cluster node descriptor. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Node { @@ -400,6 +458,18 @@ pub enum Mutation { /// The index key. key: Vec, }, + + /// Drop a whole keyspace, with its data, on every node. + /// + /// Replicated through Raft, unlike [`StorageApi::drop_keyspace`]. Only a + /// keyspace whose name matches an allow-listed prefix registered in the + /// storage crate (`single_use_{use_case}_{bucket}`) may be dropped; any + /// other name is refused with a `KEYSPACE_DROP_REFUSED` violation. A + /// missing keyspace is a no-op, so the mutation is idempotent. + DropKeyspace { + /// The keyspace to drop. + keyspace: String, + }, } impl Mutation { @@ -425,6 +495,13 @@ impl Mutation { }) } + /// Create a mutation dropping the whole keyspace on every node. + pub fn drop_keyspace>(keyspace: S) -> Self { + Self::DropKeyspace { + keyspace: keyspace.into(), + } + } + /// Create a remove mutation for the given key. pub fn remove(key: K, keyspace: Option, expected_revision: Option) -> Self where @@ -558,6 +635,15 @@ pub trait StorageApi: Send + Sync { /// of empty partitions. async fn keyspace_exists(&self, keyspace: &str) -> Result; + /// List the names of the existing keyspaces that start with `prefix`, + /// sorted. + /// + /// Reads this node's local state and never creates a keyspace. Callers + /// that sweep time-bucketed keyspaces (ADR 0044 cleanup) use it to + /// discover buckets instead of probing computed names, so a bucket missed + /// during an outage or a leader change is found on the next sweep. + async fn list_keyspaces(&self, prefix: &str) -> Result, StoreError>; + /// Returns this node's own Raft node id. /// /// Combine with [`Self::current_leader`] (`current_leader().await == @@ -652,3 +738,85 @@ pub trait StorageApi: Send + Sync { /// Mutation transaction. async fn transaction(&self, mutations: Vec) -> Result; } + +#[cfg(test)] +mod require_no_conflict_tests { + use super::*; + + fn violation(kind: &str) -> Violation { + Violation { + r#type: kind.to_string(), + subject: "k".to_string(), + description: "d".to_string(), + } + } + + fn run(result: Result) -> Result { + require_no_conflict(result, || StoreError::KeyPresent) + } + + #[test] + fn clean_response_passes() { + assert!(run(Ok(StoreResponse::default())).is_ok()); + } + + #[test] + fn conflict_violation_as_data_is_a_conflict() { + for kind in ["CONFLICT", "conflict"] { + let res = StoreResponse { + violations: vec![violation(kind)], + ..Default::default() + }; + assert!( + matches!(run(Ok(res)), Err(StoreError::KeyPresent)), + "{kind}" + ); + } + } + + #[test] + fn conflict_error_is_a_conflict() { + let err = StoreError::Conflict { + subject: "k".into(), + description: "d".into(), + }; + assert!(matches!(run(Err(err)), Err(StoreError::KeyPresent))); + } + + #[test] + fn other_violation_is_rejected_not_a_conflict() { + for kind in [ + "WRITE_RATE_EXCEEDED", + "QUARANTINED", + "KEYSPACE_DROP_REFUSED", + ] { + let res = StoreResponse { + violations: vec![violation(kind)], + ..Default::default() + }; + match run(Ok(res)) { + Err(StoreError::Rejected { violation_type, .. }) => { + assert_eq!(kind, violation_type) + } + other => panic!("{kind}: {other:?}"), + } + } + } + + #[test] + fn rejected_and_other_errors_pass_through() { + let rejected = StoreError::Rejected { + violation_type: "QUARANTINED".into(), + subject: "k".into(), + description: "d".into(), + }; + assert!(matches!( + run(Err(rejected)), + Err(StoreError::Rejected { .. }) + )); + assert!(matches!( + run(Err(StoreError::ConfigMissing)), + Err(StoreError::ConfigMissing) + )); + } +} diff --git a/crates/storage-api/src/wire_format_tests.rs b/crates/storage-api/src/wire_format_tests.rs index ef1fe32fa..d6c9bad44 100644 --- a/crates/storage-api/src/wire_format_tests.rs +++ b/crates/storage-api/src/wire_format_tests.rs @@ -153,4 +153,11 @@ fn mutation_layouts() { &Mutation::SetIndex { key: vec![1] }, "81a8536574496e646578919101", ); + assert_wire( + "Mutation::DropKeyspace", + &Mutation::DropKeyspace { + keyspace: "ks".into(), + }, + "81ac44726f704b6579737061636591a26b73", + ); } diff --git a/crates/storage/src/app/admin.rs b/crates/storage/src/app/admin.rs index 3e5924a14..7ebd8b358 100644 --- a/crates/storage/src/app/admin.rs +++ b/crates/storage/src/app/admin.rs @@ -55,10 +55,7 @@ impl Storage { let resp = resp.into_inner(); // Check for violations if let Some(v) = resp.violations.first() { - return Err(StoreError::Conflict { - subject: v.subject.clone(), - description: v.description.clone(), - }); + return Err(StoreError::from_violation(v)); } // Wrap the deserialized wire response in the zeroizing // type so it's scrubbed on drop for the rest of its @@ -522,10 +519,7 @@ impl Storage { let rsp = rsp.data; // Check for violations (e.g., CAS conflicts) if let Some(v) = rsp.violations.first() { - return Err(StoreError::Conflict { - subject: v.subject.clone(), - description: v.description.clone(), - }); + return Err(StoreError::from_violation(v)); } Ok(rsp) } diff --git a/crates/storage/src/app/storage_api.rs b/crates/storage/src/app/storage_api.rs index 8a8c6f4e5..d1e294187 100644 --- a/crates/storage/src/app/storage_api.rs +++ b/crates/storage/src/app/storage_api.rs @@ -347,6 +347,10 @@ impl StorageApi for Storage { async fn keyspace_exists(&self, keyspace: &str) -> Result { Ok(self.state_machine_store.keyspace_exists(keyspace)) } + async fn list_keyspaces(&self, prefix: &str) -> Result, ApiStoreError> { + Ok(self.state_machine_store.list_keyspaces(prefix)) + } + async fn node_id(&self) -> u64 { self.node_id } diff --git a/crates/storage/src/error.rs b/crates/storage/src/error.rs index e5221afcb..31d96e7b4 100644 --- a/crates/storage/src/error.rs +++ b/crates/storage/src/error.rs @@ -43,6 +43,18 @@ pub enum StoreError { description: String, }, + /// The state machine refused the write for a reason other than a + /// conflict with existing state (rate limit, quarantine, reserved or + /// non-droppable keyspace, ...). Kept apart from [`Self::Conflict`] so a + /// caller that treats a conflict as "already exists" (ADR 0044) is not + /// told so by an unrelated refusal. + #[error("write rejected ({violation_type}): {subject} — {description}")] + Rejected { + violation_type: String, + subject: String, + description: String, + }, + /// Cryptographic error (AES-GCM, nonce management, DEK). #[error(transparent)] Crypto { @@ -257,12 +269,23 @@ impl From for crate::ApiStoreError { subject, description, }, + StoreError::Rejected { + violation_type, + subject, + description, + } => Self::Rejected { + violation_type, + subject, + description, + }, StoreError::KeyPresent => Self::KeyPresent, - StoreError::Quarantined(partition) => Self::Conflict { + StoreError::Quarantined(partition) => Self::Rejected { + violation_type: "QUARANTINED".to_string(), subject: partition, description: "partition quarantined due to repeated GCM tag failures".to_string(), }, - StoreError::WriteRateExceeded(key, version) => Self::Conflict { + StoreError::WriteRateExceeded(key, version) => Self::Rejected { + violation_type: "WRITE_RATE_EXCEEDED".to_string(), subject: key, description: format!( "write rate exceeded at version {version}; DEK rotation required" @@ -273,3 +296,25 @@ impl From for crate::ApiStoreError { } } } + +impl StoreError { + /// Maps the first violation of a committed write to an error. + /// + /// Only the `CONFLICT` type (revision mismatch, key already exists) is a + /// [`Self::Conflict`]; every other type is [`Self::Rejected`] and keeps + /// its type. + pub(crate) fn from_violation(v: &crate::protobuf::api::response::Violation) -> Self { + if v.r#type.eq_ignore_ascii_case("CONFLICT") { + Self::Conflict { + subject: v.subject.clone(), + description: v.description.clone(), + } + } else { + Self::Rejected { + violation_type: v.r#type.clone(), + subject: v.subject.clone(), + description: v.description.clone(), + } + } + } +} diff --git a/crates/storage/src/grpc/authz.rs b/crates/storage/src/grpc/authz.rs index 071cc5ba0..d7779b239 100644 --- a/crates/storage/src/grpc/authz.rs +++ b/crates/storage/src/grpc/authz.rs @@ -287,7 +287,11 @@ pub fn ensure_data_command(cmd: &StoreCommand) -> Result<(), Status> { | MutationInner::Remove { .. } | MutationInner::RemoveIndex { .. } | MutationInner::CreateIfAbsent { .. } - | MutationInner::SetIndex { .. } => {} + | MutationInner::SetIndex { .. } + // Proposed by the leader's cleanup task; a request + // forwarded after a leader change arrives here. The + // state machine enforces the prefix allow-list. + | MutationInner::DropKeyspace { .. } => {} _ => { return Err(Status::permission_denied( "mutation is not permitted on the data-plane command RPC", @@ -612,6 +616,17 @@ mod tests { assert!(ensure_data_command(&ok).is_ok()); } + #[test] + fn command_filter_allows_keyspace_drop() { + // The state machine enforces the prefix allow-list; the data-plane + // filter must let a forwarded cleanup proposal through after a + // leader change. + let ok = StoreCommand::Transaction(vec![MutationInner::DropKeyspace { + keyspace: "single_use_totp_1".into(), + }]); + assert!(ensure_data_command(&ok).is_ok()); + } + #[test] fn command_filter_accepts_empty_transaction() { // An empty transaction is a no-op; accepted by design. diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index 9ff838a05..bdbc0becd 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -79,6 +79,7 @@ mod wire_format_tests; pub use openstack_keystone_storage_api::{ DataTier, Metadata, Mutation, Node, ReadConsistency, StorageApi, StorageReadiness, StoreDataEnvelope, StoreError as ApiStoreError, StoreResponse, Violation, + is_conflict_violation, require_no_conflict, }; pub use error::StoreError; diff --git a/crates/storage/src/mock.rs b/crates/storage/src/mock.rs index 27631b48f..d2b77090c 100644 --- a/crates/storage/src/mock.rs +++ b/crates/storage/src/mock.rs @@ -216,6 +216,16 @@ impl StorageApi for MockStorage { Ok(lock(&self.data)?.contains_key(keyspace)) } + async fn list_keyspaces(&self, prefix: &str) -> Result, ApiStoreError> { + let mut names: Vec = lock(&self.data)? + .keys() + .filter(|k| k.starts_with(prefix)) + .cloned() + .collect(); + names.sort(); + Ok(names) + } + async fn node_id(&self) -> u64 { *self.node_id.lock().unwrap_or_else(|p| p.into_inner()) } @@ -333,6 +343,9 @@ impl StorageApi for MockStorage { async fn transaction(&self, mutations: Vec) -> Result { let mut violations: Vec = Vec::new(); + let has_non_drop = mutations + .iter() + .any(|m| !matches!(m, Mutation::DropKeyspace { .. })); let mut data = lock(&self.data)?; let mut metadata_map = lock(&self.metadata)?; @@ -430,6 +443,36 @@ impl StorageApi for MockStorage { Mutation::SetIndex { key } => { indexes.insert(key, ()); } + + Mutation::DropKeyspace { keyspace } => { + let mixed = has_non_drop; + if mixed { + violations.push(Violation { + r#type: "KEYSPACE_DROP_REFUSED".to_string(), + subject: keyspace, + description: "DropKeyspace must not share a transaction with \ + other mutations" + .to_string(), + }); + } else if crate::store::state_machine::keyspace_drop_allowed(&keyspace) { + if let Some(removed) = data.remove(&keyspace) { + // Mock metadata is flat: keep entries whose key + // also lives in another keyspace. + for key in removed.keys() { + if !data.values().any(|ks| ks.contains_key(key)) { + metadata_map.remove(key); + } + } + } + } else { + violations.push(Violation { + r#type: "KEYSPACE_DROP_REFUSED".to_string(), + subject: keyspace, + description: "keyspace is not on the replicated-drop allow-list" + .to_string(), + }); + } + } } } diff --git a/crates/storage/src/store/state_machine.rs b/crates/storage/src/store/state_machine.rs index 8a82f4def..da875428e 100644 --- a/crates/storage/src/store/state_machine.rs +++ b/crates/storage/src/store/state_machine.rs @@ -56,6 +56,7 @@ use crate::store_command::*; use crate::types::Metadata; mod dek; +mod keyspace_drop; mod quarantine; mod raft_sm; mod restore; @@ -64,6 +65,7 @@ mod snapshot; mod snapshot_file; mod status; +pub(crate) use self::keyspace_drop::keyspace_drop_allowed; pub(crate) use self::restore::MAX_LIVE_RESTORE_SIZE; pub(crate) use self::rotation::{ DEK_REVOKED_PENDING_PREFIX, DEK_REVOKED_PREFIX, META_DEK_CURRENT, unix_now, @@ -71,6 +73,7 @@ pub(crate) use self::rotation::{ pub use self::rotation::{PENDING_ROTATION_TTL_SECS, ReencryptReport, load_pending_rotations}; pub use self::status::EncryptionStatus; +use self::keyspace_drop::KEYSPACE_DROP_REFUSED; use self::quarantine::*; use self::restore::*; use self::rotation::*; diff --git a/crates/storage/src/store/state_machine/keyspace_drop.rs b/crates/storage/src/store/state_machine/keyspace_drop.rs new file mode 100644 index 000000000..841787a4a --- /dev/null +++ b/crates/storage/src/store/state_machine/keyspace_drop.rs @@ -0,0 +1,229 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//! Replicated keyspace drop (ADR 0044 §3). +//! +//! `MutationInner::DropKeyspace` removes a whole keyspace on every node. To +//! keep a bug in a caller's name computation from reaching unrelated state, +//! only keyspaces whose name matches a registered prefix pattern may be +//! dropped. + +use super::*; + +/// Violation type emitted when a drop names a keyspace outside the allow-list. +pub(crate) const KEYSPACE_DROP_REFUSED: &str = "KEYSPACE_DROP_REFUSED"; + +/// Prefix of the single-use credential registry keyspaces +/// (`single_use_{use_case}_{bucket}`, ADR 0044 §3). +pub(crate) const SINGLE_USE_KEYSPACE_PREFIX: &str = "single_use_"; + +/// Longest keyspace name accepted for a replicated drop. +const MAX_DROPPABLE_NAME_LEN: usize = 128; + +/// Returns `true` if `name` may be dropped through `DropKeyspace`. +/// +/// The name must have the form `single_use_{use_case}_{bucket}` where +/// `use_case` is a non-empty run of `[a-z0-9_]` and `bucket` is a non-empty +/// run of ASCII digits. Core, reserved and other feature keyspaces never +/// match. A feature that adopts the mutation later registers its own pattern +/// here. +pub(crate) fn keyspace_drop_allowed(name: &str) -> bool { + if name.len() > MAX_DROPPABLE_NAME_LEN { + return false; + } + let Some(rest) = name.strip_prefix(SINGLE_USE_KEYSPACE_PREFIX) else { + return false; + }; + let Some((use_case, bucket)) = rest.rsplit_once('_') else { + return false; + }; + !use_case.is_empty() + && use_case + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_') + && !bucket.is_empty() + && bucket.bytes().all(|b| b.is_ascii_digit()) +} + +impl FjallStateMachine { + /// Drops `name` with all its data and the per-record metadata. + /// + /// Called by `apply()` after the batch of the entry committed, with the + /// lifecycle read guard released. + /// + /// Unlike [`Self::drop_keyspace`] the keyspace need not be empty. A + /// missing keyspace is a no-op. The caller has checked + /// [`keyspace_drop_allowed`]. + pub(super) fn drop_keyspace_replicated(&self, name: &str) -> Result<(), StoreError> { + // Keyspaces are either ephemeral or Fjall-backed, but do not rely on + // it: clear both sides. + self.ephemeral.remove(name); + // Excludes `apply()` and the snapshot walk, as in `drop_keyspace`. + let _lifecycle_guard = self + .keyspace_lifecycle + .write() + .unwrap_or_else(|p| p.into_inner()); + // The per-record metadata lives in the `meta` keyspace under + // `\0` (see `meta_key`). Remove it first, or a record + // created again later would find the stale entry and conflict. A + // crash between the two steps replays this entry on restart, and both + // steps are idempotent. + let mut prefix = name.as_bytes().to_vec(); + prefix.push(0); + let mut batch = self.db.batch(); + for item in self.meta.prefix(&prefix) { + batch.remove(&self.meta, item.key()?); + } + batch.commit()?; + if !self.db.keyspace_exists(name) { + return Ok(()); + } + let ks = self.db.keyspace(name, KeyspaceCreateOptions::default)?; + self.db.delete_keyspace(ks)?; + Ok(()) + } + + /// Names of the existing keyspaces that start with `prefix`, sorted. + /// + /// Read-only: never creates a keyspace. Includes ephemeral keyspaces. + pub fn list_keyspaces(&self, prefix: &str) -> Vec { + let mut names: Vec = self + .db + .list_keyspace_names() + .into_iter() + .map(|n| n.to_string()) + .chain(self.ephemeral.iter().map(|e| e.key().clone())) + .filter(|n| n.starts_with(prefix)) + .collect(); + names.sort(); + names.dedup(); + names + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn allow_list_accepts_single_use_buckets() { + for name in [ + "single_use_assertion_5000", + "single_use_totp_0", + "single_use_dpop_token_123456", + "single_use_a_b_c_9", + ] { + assert!(keyspace_drop_allowed(name), "{name}"); + } + } + + #[test] + fn allow_list_refuses_everything_else() { + for name in [ + "data", + "meta", + "index", + "oauth2_keys", + "webauthn_state_1", + "single_use_", + "single_use_assertion", + "single_use__5", + "single_use_assertion_", + "single_use_assertion_5x", + "single_use_Assertion_5", + "single_use_assertion_-5", + "single_use_assertion_5/..", + "xsingle_use_assertion_5", + "SINGLE_USE_assertion_5", + "", + ] { + assert!(!keyspace_drop_allowed(name), "{name}"); + } + let long = format!("single_use_{}_5", "a".repeat(200)); + assert!(!keyspace_drop_allowed(&long)); + } + + fn put(sm: &FjallStateMachine, keyspace: &str, key: &[u8]) { + let ks = sm.keyspace(keyspace).expect("keyspace"); + let (cipher, dek_version) = sm + .encrypt_and_store(&ks, key, keyspace.as_bytes(), DataTier::Internal as u8, b"") + .expect("encrypt"); + ks.insert(key, cipher).expect("insert"); + let mut metadata = Metadata::new(); + metadata.dek_version = Some(dek_version); + sm.meta() + .insert(meta_key(keyspace, key), metadata.pack().expect("pack")) + .expect("insert meta"); + } + + #[test] + fn replicated_drop_removes_data_meta_and_snapshot_entry() { + let (sm, _td) = make_seeded_sm(0x31); + put(&sm, "single_use_totp_5", b"a"); + put(&sm, "single_use_totp_5", b"b"); + put(&sm, "single_use_totp_6", b"a"); + put(&sm, "data", b"a"); + + assert_eq!( + vec!["single_use_totp_5", "single_use_totp_6"], + sm.list_keyspaces("single_use_") + ); + + sm.drop_keyspace_replicated("single_use_totp_5") + .expect("drop non-empty keyspace"); + + assert!(!sm.keyspace_exists("single_use_totp_5")); + assert_eq!(vec!["single_use_totp_6"], sm.list_keyspaces("single_use_")); + // The record metadata went with it, so the key can be created again. + assert!( + sm.meta() + .get(meta_key("single_use_totp_5", b"a")) + .expect("get") + .is_none() + ); + assert!( + sm.meta() + .get(meta_key("single_use_totp_5", b"b")) + .expect("get") + .is_none() + ); + // Neighbours are untouched, including the same key elsewhere. + assert!( + sm.meta() + .get(meta_key("single_use_totp_6", b"a")) + .expect("get") + .is_some() + ); + assert!( + sm.meta() + .get(meta_key("data", b"a")) + .expect("get") + .is_some() + ); + assert!(sm.data().get(b"a").expect("get").is_some()); + + // A snapshot taken after the drop does not carry the keyspace, so a + // follower that installs it ends in the same state. + let payload = sm.snapshot_payload().expect("payload"); + let names: Vec<&str> = payload.keyspaces.iter().map(|(n, _)| n.as_str()).collect(); + assert!(!names.contains(&"single_use_totp_5"), "{names:?}"); + assert!(names.contains(&"single_use_totp_6")); + + // Idempotent. + sm.drop_keyspace_replicated("single_use_totp_5") + .expect("second drop is a no-op"); + sm.drop_keyspace_replicated("single_use_never_created_9") + .expect("missing keyspace is a no-op"); + assert!(!sm.keyspace_exists("single_use_never_created_9")); + } +} diff --git a/crates/storage/src/store/state_machine/raft_sm.rs b/crates/storage/src/store/state_machine/raft_sm.rs index a3ff5f0e3..5d4300603 100644 --- a/crates/storage/src/store/state_machine/raft_sm.rs +++ b/crates/storage/src/store/state_machine/raft_sm.rs @@ -77,6 +77,9 @@ impl RaftStateMachine for Arc { // See `PendingRotationMutation`: deferred the same way as // `pending_dek_swap` above. let mut pending_rotation_mutation: Option = None; + // Keyspaces named by `DropKeyspace`; dropped after the batch + // commits, once the lifecycle read guard is released. + let mut pending_keyspace_drops: Vec = Vec::new(); let response = if let Some(store_req) = entry.app_data { match StoreCommand::unpack(&store_req)? { @@ -131,6 +134,17 @@ impl RaftStateMachine for Arc { } StoreCommand::Transaction(mutations) => { let mut violations: Vec = Vec::new(); + // The drop runs after the batch commits, so it cannot + // keep its position among other mutations. It is + // accepted only in a transaction of drops, which also + // makes a replay of the entry after a crash + // idempotent. + let mixed_drop = mutations + .iter() + .any(|m| matches!(m, MutationInner::DropKeyspace { .. })) + && mutations + .iter() + .any(|m| !matches!(m, MutationInner::DropKeyspace { .. })); for mutation in mutations { match mutation { MutationInner::Remove { @@ -848,6 +862,28 @@ impl RaftStateMachine for Arc { } } } + MutationInner::DropKeyspace { keyspace } => { + if mixed_drop { + violations.push(Violation { + r#type: KEYSPACE_DROP_REFUSED.to_string(), + subject: keyspace, + description: "DropKeyspace must not share a \ + transaction with other mutations" + .to_string(), + }); + } else if keyspace_drop_allowed(&keyspace) { + pending_keyspace_drops.push(keyspace); + } else { + violations.push(Violation { + r#type: KEYSPACE_DROP_REFUSED.to_string(), + subject: keyspace.clone(), + description: format!( + "keyspace '{keyspace}' is not on the \ + replicated-drop allow-list" + ), + }); + } + } MutationInner::AbortPendingRotation { rotation_id } => { let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -924,6 +960,16 @@ impl RaftStateMachine for Arc { None => {} } + if !pending_keyspace_drops.is_empty() { + // The write side of the lifecycle lock is needed. + drop(lifecycle_guard.take()); + for name in &pending_keyspace_drops { + self.drop_keyspace_replicated(name) + .map_err(|e| io::Error::other(e.to_string()))?; + tracing::info!(keyspace = %name, "keyspace dropped via Raft"); + } + } + // Swap the active DEK epoch after a successful InstallDek // commit. if let Some((new_epoch, is_emergency_rotation)) = pending_dek_swap { diff --git a/crates/storage/src/store_command.rs b/crates/storage/src/store_command.rs index 80f787bbd..3c90eb5c5 100644 --- a/crates/storage/src/store_command.rs +++ b/crates/storage/src/store_command.rs @@ -269,6 +269,16 @@ pub enum MutationInner { /// UUID v4 identifier from `CreatePendingRotation`. rotation_id: String, }, + + /// Drop a whole allow-listed keyspace and its data on every node. + /// + /// See [`Mutation::DropKeyspace`]. Applied after the batch of the entry + /// commits; refused with a violation unless the name passes + /// `keyspace_drop_allowed`. + DropKeyspace { + /// The keyspace to drop. + keyspace: String, + }, } impl MutationInner { @@ -319,6 +329,7 @@ impl MutationInner { cipher: value, }, Mutation::SetIndex { key } => MutationInner::SetIndex { key }, + Mutation::DropKeyspace { keyspace } => MutationInner::DropKeyspace { keyspace }, }) } } diff --git a/crates/storage/src/wire_format_tests.rs b/crates/storage/src/wire_format_tests.rs index e511add87..a08441086 100644 --- a/crates/storage/src/wire_format_tests.rs +++ b/crates/storage/src/wire_format_tests.rs @@ -231,6 +231,13 @@ fn mutation_inner_layouts() { &MutationInner::SetIndex { key: vec![1] }, "81a8536574496e646578919101", ); + assert_wire( + "MutationInner::DropKeyspace", + &MutationInner::DropKeyspace { + keyspace: "ks".into(), + }, + "81ac44726f704b6579737061636591a26b73", + ); } /// The signed JSON of an audit record is a serialization of the struct, so diff --git a/crates/storage/tests/test_cluster/main.rs b/crates/storage/tests/test_cluster/main.rs index 4b7c0d210..6326c44c4 100644 --- a/crates/storage/tests/test_cluster/main.rs +++ b/crates/storage/tests/test_cluster/main.rs @@ -62,3 +62,4 @@ mod quarantine; mod races; mod restore; mod rotation; +mod single_use; diff --git a/crates/storage/tests/test_cluster/single_use.rs b/crates/storage/tests/test_cluster/single_use.rs new file mode 100644 index 000000000..7109c103e --- /dev/null +++ b/crates/storage/tests/test_cluster/single_use.rs @@ -0,0 +1,146 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +//! Replicated keyspace drop and keyspace listing (ADR 0044 §3). +//! +//! `DropKeyspace` is committed through Raft, so a drop proposed on the leader +//! removes the allow-listed keyspace, with data, on every node, while +//! unrelated keyspaces are refused and left untouched. + +use super::harness::*; +use super::*; + +const SINGLE_USE_PORT_BASE: u16 = 2400; + +/// Waits until every node lists exactly `expected` single-use keyspaces. +async fn expect_listing(nodes: &[RestartableNode], expected: &[&str]) -> bool { + for _ in 0..100 { + let mut all = true; + for node in nodes { + let got = node.storage().list_keyspaces("single_use_").await; + all &= got.is_ok_and(|g| g == expected); + } + if all { + return true; + } + TypeConfig::sleep(Duration::from_millis(100)).await; + } + false +} + +fn create(keyspace: &str, key: &str) -> Result { + Ok(Mutation::create_if_absent( + key.as_bytes().to_vec(), + Vec::::new(), + Metadata::new(), + Some(keyspace), + )?) +} + +#[serial_test::serial] +#[test] +fn test_drop_keyspace_is_replicated_and_allow_listed() { + TypeConfig::run(test_drop_keyspace_is_replicated_and_allow_listed_inner()).unwrap(); +} + +async fn test_drop_keyspace_is_replicated_and_allow_listed_inner() -> Result<()> { + let nodes = start_three_node_cluster_with(SINGLE_USE_PORT_BASE, |_| {}).await?; + let leader = nodes[0].storage().clone(); + + // A create, and its replay, in a bucket keyspace. + let first = leader + .transaction(vec![create("single_use_assertion_100", "k1")?]) + .await?; + assert!(first.violations.is_empty()); + let replay = leader + .transaction(vec![create("single_use_assertion_100", "k1")?]) + .await; + assert!( + matches!(replay, Err(ApiStoreError::Conflict { .. })), + "{replay:?}" + ); + leader + .transaction(vec![create("single_use_totp_100", "k2")?]) + .await?; + assert!( + expect_listing(&nodes, &["single_use_assertion_100", "single_use_totp_100"]).await, + "every node lists the bucket keyspaces" + ); + + // Unrelated state, which a drop must never reach. + set_value_retrying(&leader, "keep", "me").await?; + + // Refused names leave everything in place. The refusal is a `Rejected` + // error, not a `Conflict`: a replay check must not read it as one. + for name in ["data", "meta", "index", "webauthn_state_1", "single_use_x"] { + let res = leader + .transaction(vec![Mutation::drop_keyspace(name)]) + .await; + assert!( + matches!( + &res, + Err(ApiStoreError::Rejected { violation_type, .. }) + if violation_type == "KEYSPACE_DROP_REFUSED" + ), + "{name}: {res:?}" + ); + } + assert!(leader.get_by_key(b"keep", None).await?.is_some()); + + // A drop sharing a transaction with other mutations is refused and rolls the + // whole batch back. + let mixed = leader + .transaction(vec![ + create("single_use_assertion_100", "k3")?, + Mutation::drop_keyspace("single_use_assertion_100"), + ]) + .await; + assert!(matches!(mixed, Err(ApiStoreError::Rejected { .. }))); + assert!( + leader + .get_by_key(b"k3", Some("single_use_assertion_100")) + .await? + .is_none() + ); + + // The allowed drop removes the keyspace and its data everywhere and + // leaves the other bucket alone. + let dropped = leader + .transaction(vec![Mutation::drop_keyspace("single_use_assertion_100")]) + .await?; + assert!(dropped.violations.is_empty()); + assert!( + expect_listing(&nodes, &["single_use_totp_100"]).await, + "the drop reached every node" + ); + assert!(!leader.keyspace_exists("single_use_assertion_100").await?); + assert!( + leader + .get_by_key(b"k1", Some("single_use_assertion_100")) + .await? + .is_none() + ); + + // Idempotent: dropping a missing keyspace is a no-op. + let again = leader + .transaction(vec![Mutation::drop_keyspace("single_use_assertion_100")]) + .await?; + assert!(again.violations.is_empty()); + + // A dropped credential can be created again (new bucket life). + let recreated = leader + .transaction(vec![create("single_use_assertion_100", "k1")?]) + .await?; + assert!(recreated.violations.is_empty()); + Ok(()) +} diff --git a/crates/webauthn/src/driver/raft.rs b/crates/webauthn/src/driver/raft.rs index 16ca35264..b007b678d 100644 --- a/crates/webauthn/src/driver/raft.rs +++ b/crates/webauthn/src/driver/raft.rs @@ -731,6 +731,10 @@ mod tests { self.inner.drop_keyspace(keyspace).await } + async fn list_keyspaces(&self, prefix: &str) -> Result, ApiStoreError> { + self.inner.list_keyspaces(prefix).await + } + async fn node_id(&self) -> u64 { self.inner.node_id().await }