Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 1 addition & 20 deletions crates/oauth2-client-driver-raft/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,29 +25,10 @@ use openstack_keystone_core::oauth2_client::backend::Oauth2ClientBackend;
use openstack_keystone_core::plugin_manager::BackendRegistration;
use openstack_keystone_core_types::oauth2_client::*;
use openstack_keystone_distributed_storage::{
ApiStoreError, Metadata, StorageApi, StoreDataEnvelope, StoreError, StoreResponse,
Metadata, StorageApi, StoreDataEnvelope, StoreError, require_no_conflict,
store_command::Mutation,
};

/// Bridges two different backend behaviors for the same semantic event: the
/// real Raft-backed [`StorageApi`] converts the first `Violation` on a
/// `transaction` call into an immediate `Err(ApiStoreError::Conflict)`;
/// `MockStorage`, used by this crate's own unit tests, instead returns every
/// violation as data on an `Ok` response's `violations` field. Both call
/// sites need conflict detection to behave identically against either
/// backend (mirrors `scim-driver-raft`'s helper of the same name).
fn require_no_conflict(
result: Result<StoreResponse, ApiStoreError>,
conflict: impl FnOnce() -> StoreError,
) -> Result<StoreResponse, StoreError> {
match result {
Ok(response) if response.violations.is_empty() => Ok(response),
Ok(_) => Err(conflict()),
Err(ApiStoreError::Conflict { .. }) => Err(conflict()),
Err(e) => Err(e.into()),
}
}

/// Global `client_id` index value payload: resolves to the owning
/// `(domain_id, provider_id)` coordinate. Stored as a
/// `Mutation::create_if_absent` data key (not `set_index`, which carries no
Expand Down
35 changes: 16 additions & 19 deletions crates/oauth2-key-driver-raft/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ use openstack_keystone_core_types::oauth2_key::{
PendingRotationInfo,
};
use openstack_keystone_distributed_storage::{
ApiStoreError as StoreError, Metadata, StorageApi, StoreDataEnvelope, store_command::Mutation,
ApiStoreError as StoreError, Metadata, StorageApi, StoreDataEnvelope, require_no_conflict,
store_command::Mutation,
};
use openstack_keystone_key_repository::asymmetric::{
ActiveKeys, AsymmetricKeyRepository, AsymmetricKeySource, KeyMaterial, KeyRole,
Expand Down Expand Up @@ -786,24 +787,16 @@ impl RaftOauth2KeyBackend {
storage.transaction(vec![mutation]).await
}
};
match result {
// The real storage surfaces a CAS miss as `Err(Conflict)`;
// the mock returns it as `violations` on an `Ok` response.
Ok(response) if response.violations.is_empty() => return Ok(()),
// Only a CAS/create-if-absent miss is retried; any other
// violation is a real failure, not contention.
Ok(response) => {
if let Some(v) = response
.violations
.iter()
.find(|v| !v.r#type.eq_ignore_ascii_case("CONFLICT"))
{
return Err(persist(StoreError::Conflict {
subject: v.subject.clone(),
description: v.description.clone(),
}));
}
}
// Only a CAS/create-if-absent miss is retried; any other
// violation or error is a real failure, not contention. The
// helper maps both the real store's `Err(Conflict)` and the
// mock's violations-as-data to the same conflict.
let contended = || StoreError::Conflict {
subject: key.clone(),
description: "JTI revocation list contended".to_string(),
};
match require_no_conflict(result, contended) {
Ok(_) => return Ok(()),
Err(StoreError::Conflict { .. }) => {}
Err(e) => return Err(persist(e)),
}
Expand Down Expand Up @@ -2320,6 +2313,10 @@ mod tests {
self.inner.drop_keyspace(keyspace).await
}

async fn list_keyspaces(&self, prefix: &str) -> Result<Vec<String>, StoreError> {
self.inner.list_keyspaces(prefix).await
}

async fn node_id(&self) -> u64 {
self.inner.node_id().await
}
Expand Down
29 changes: 5 additions & 24 deletions crates/scim-driver-raft/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,33 +25,10 @@ use openstack_keystone_core::scim_resource::backend::ScimResourceBackend;
use openstack_keystone_core::scim_resource::error::ScimResourceProviderError;
use openstack_keystone_core_types::scim::*;
use openstack_keystone_distributed_storage::{
ApiStoreError, Metadata, StorageApi, StoreDataEnvelope, StoreError, StoreResponse,
ApiStoreError, Metadata, StorageApi, StoreDataEnvelope, StoreError, require_no_conflict,
store_command::Mutation,
};

/// Bridges two different backend behaviors for the same semantic event.
/// The real Raft-backed [`StorageApi`] impl (`Storage::write_command_to_
/// storage`) converts the *first*
/// [`Violation`](openstack_keystone_distributed_storage::Violation)
/// on a `transaction`/`set_value` call into an immediate `Err(ApiStoreError::
/// Conflict)`; `MockStorage`, used by this crate's own unit tests, instead
/// returns every violation as data on an `Ok` response's `violations` field.
/// Both call sites here need conflict detection to work identically against
/// either backend, so this normalizes them into a single `Err(conflict())`
/// on either shape, `Ok(response)` otherwise. Any other error passes through
/// via the ordinary `StoreError::StorageApi` conversion.
fn require_no_conflict(
result: Result<StoreResponse, ApiStoreError>,
conflict: impl FnOnce() -> StoreError,
) -> Result<StoreResponse, StoreError> {
match result {
Ok(response) if response.violations.is_empty() => Ok(response),
Ok(_) => Err(conflict()),
Err(ApiStoreError::Conflict { .. }) => Err(conflict()),
Err(e) => Err(e.into()),
}
}

/// Raft Database SCIM realm backend.
///
/// Realms are indexed directly by their `(domain_id, provider_id)`
Expand Down Expand Up @@ -1130,6 +1107,10 @@ mod tests {
self.inner.drop_keyspace(keyspace).await
}

async fn list_keyspaces(&self, prefix: &str) -> Result<Vec<String>, ApiStoreError> {
self.inner.list_keyspaces(prefix).await
}

async fn node_id(&self) -> u64 {
self.inner.node_id().await
}
Expand Down
17 changes: 1 addition & 16 deletions crates/service-grant-driver-raft/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ use openstack_keystone_core::service_grant::ServiceGrantProviderError;
use openstack_keystone_core::service_grant::backend::ServiceGrantBackend;
use openstack_keystone_core_types::service_grant::*;
use openstack_keystone_distributed_storage::{
ApiStoreError, Metadata, StorageApi, StoreDataEnvelope, StoreError, StoreResponse,
Metadata, StorageApi, StoreDataEnvelope, StoreError, require_no_conflict,
store_command::Mutation,
};

Expand All @@ -46,21 +46,6 @@ const GRANTOR_IDX: &str = "service_grant:grantor_idx:v1:";
const GRANTEE_IDX: &str = "service_grant:grantee_idx:v1:";
const PROJECT_IDX: &str = "service_grant:project_idx:v1:";

/// Bridges the real storage (first violation is an `Err(Conflict)`) and
/// `MockStorage` (violations returned as data); mirrors the helper of the
/// other Raft drivers.
fn require_no_conflict(
result: Result<StoreResponse, ApiStoreError>,
conflict: impl FnOnce() -> StoreError,
) -> Result<StoreResponse, StoreError> {
match result {
Ok(response) if response.violations.is_empty() => Ok(response),
Ok(_) => Err(conflict()),
Err(ApiStoreError::Conflict { .. }) => Err(conflict()),
Err(e) => Err(e.into()),
}
}

/// A `(domain_id, id)` coordinate recovered from an index key.
type Coordinate = (String, String);

Expand Down
168 changes: 168 additions & 0 deletions crates/storage-api/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,21 @@ pub enum StoreError {
#[error("deserialization error")]
Deserialize(#[from] rmp_serde::decode::Error),

/// The write was refused for a reason other than a conflict with
/// existing state (`WRITE_RATE_EXCEEDED`, `QUARANTINED`,
/// `RESERVED_KEYSPACE`, `KEYSPACE_DROP_REFUSED`, ...). Not a
/// [`Self::Conflict`]: a caller that reads a conflict as "already
/// exists" must not read this as one.
#[error("write rejected ({violation_type}): {subject} — {description}")]
Rejected {
/// The violation type reported by the state machine.
violation_type: String,
/// The resource or key the refusal refers to.
subject: String,
/// Human-readable description.
description: String,
},

/// Key is already present in the store while the call expects it to be
/// unset.
#[error("key is already set")]
Expand Down Expand Up @@ -170,6 +185,49 @@ impl StoreError {
}
}

/// Returns `true` if a violation type means "the key or revision already
/// conflicts" (ADR 0044 §3). Only this type is a conflict; every other type is
/// a refusal that must not be read as one.
pub fn is_conflict_violation(violation: &Violation) -> bool {
violation.r#type.eq_ignore_ascii_case("CONFLICT")
}

/// Normalizes the two shapes in which a storage reports a failed write into
/// one: the Raft-backed store returns the first violation as an immediate
/// `Err`, while a mock returns violations as data on an `Ok` response.
///
/// - no violation: `Ok(response)`;
/// - a conflict (`CONFLICT` violation or `Err(Conflict)`): `Err(conflict())`;
/// - any other violation: `Err(`[`StoreError::Rejected`]`)` converted into
/// `E`, keeping its type, never a conflict;
/// - any other error: passed through converted into `E`.
pub fn require_no_conflict<E>(
result: Result<StoreResponse, StoreError>,
conflict: impl FnOnce() -> E,
) -> Result<StoreResponse, E>
where
E: From<StoreError>,
{
match result {
Ok(response) => match response
.violations
.iter()
.find(|v| !is_conflict_violation(v))
.or(response.violations.first())
{
None => Ok(response),
Some(v) if is_conflict_violation(v) => Err(conflict()),
Some(v) => Err(E::from(StoreError::Rejected {
violation_type: v.r#type.clone(),
subject: v.subject.clone(),
description: v.description.clone(),
})),
},
Err(StoreError::Conflict { .. }) => Err(conflict()),
Err(e) => Err(E::from(e)),
}
}

/// Raft cluster node descriptor.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Node {
Expand Down Expand Up @@ -400,6 +458,18 @@ pub enum Mutation {
/// The index key.
key: Vec<u8>,
},

/// Drop a whole keyspace, with its data, on every node.
///
/// Replicated through Raft, unlike [`StorageApi::drop_keyspace`]. Only a
/// keyspace whose name matches an allow-listed prefix registered in the
/// storage crate (`single_use_{use_case}_{bucket}`) may be dropped; any
/// other name is refused with a `KEYSPACE_DROP_REFUSED` violation. A
/// missing keyspace is a no-op, so the mutation is idempotent.
DropKeyspace {
/// The keyspace to drop.
keyspace: String,
},
}

impl Mutation {
Expand All @@ -425,6 +495,13 @@ impl Mutation {
})
}

/// Create a mutation dropping the whole keyspace on every node.
pub fn drop_keyspace<S: Into<String>>(keyspace: S) -> Self {
Self::DropKeyspace {
keyspace: keyspace.into(),
}
}

/// Create a remove mutation for the given key.
pub fn remove<K, S>(key: K, keyspace: Option<S>, expected_revision: Option<u64>) -> Self
where
Expand Down Expand Up @@ -558,6 +635,15 @@ pub trait StorageApi: Send + Sync {
/// of empty partitions.
async fn keyspace_exists(&self, keyspace: &str) -> Result<bool, StoreError>;

/// List the names of the existing keyspaces that start with `prefix`,
/// sorted.
///
/// Reads this node's local state and never creates a keyspace. Callers
/// that sweep time-bucketed keyspaces (ADR 0044 cleanup) use it to
/// discover buckets instead of probing computed names, so a bucket missed
/// during an outage or a leader change is found on the next sweep.
async fn list_keyspaces(&self, prefix: &str) -> Result<Vec<String>, StoreError>;

/// Returns this node's own Raft node id.
///
/// Combine with [`Self::current_leader`] (`current_leader().await ==
Expand Down Expand Up @@ -652,3 +738,85 @@ pub trait StorageApi: Send + Sync {
/// Mutation transaction.
async fn transaction(&self, mutations: Vec<Mutation>) -> Result<StoreResponse, StoreError>;
}

#[cfg(test)]
mod require_no_conflict_tests {
use super::*;

fn violation(kind: &str) -> Violation {
Violation {
r#type: kind.to_string(),
subject: "k".to_string(),
description: "d".to_string(),
}
}

fn run(result: Result<StoreResponse, StoreError>) -> Result<StoreResponse, StoreError> {
require_no_conflict(result, || StoreError::KeyPresent)
}

#[test]
fn clean_response_passes() {
assert!(run(Ok(StoreResponse::default())).is_ok());
}

#[test]
fn conflict_violation_as_data_is_a_conflict() {
for kind in ["CONFLICT", "conflict"] {
let res = StoreResponse {
violations: vec![violation(kind)],
..Default::default()
};
assert!(
matches!(run(Ok(res)), Err(StoreError::KeyPresent)),
"{kind}"
);
}
}

#[test]
fn conflict_error_is_a_conflict() {
let err = StoreError::Conflict {
subject: "k".into(),
description: "d".into(),
};
assert!(matches!(run(Err(err)), Err(StoreError::KeyPresent)));
}

#[test]
fn other_violation_is_rejected_not_a_conflict() {
for kind in [
"WRITE_RATE_EXCEEDED",
"QUARANTINED",
"KEYSPACE_DROP_REFUSED",
] {
let res = StoreResponse {
violations: vec![violation(kind)],
..Default::default()
};
match run(Ok(res)) {
Err(StoreError::Rejected { violation_type, .. }) => {
assert_eq!(kind, violation_type)
}
other => panic!("{kind}: {other:?}"),
}
}
}

#[test]
fn rejected_and_other_errors_pass_through() {
let rejected = StoreError::Rejected {
violation_type: "QUARANTINED".into(),
subject: "k".into(),
description: "d".into(),
};
assert!(matches!(
run(Err(rejected)),
Err(StoreError::Rejected { .. })
));
assert!(matches!(
run(Err(StoreError::ConfigMissing)),
Err(StoreError::ConfigMissing)
));
}
}
7 changes: 7 additions & 0 deletions crates/storage-api/src/wire_format_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -153,4 +153,11 @@ fn mutation_layouts() {
&Mutation::SetIndex { key: vec![1] },
"81a8536574496e646578919101",
);
assert_wire(
"Mutation::DropKeyspace",
&Mutation::DropKeyspace {
keyspace: "ks".into(),
},
"81ac44726f704b6579737061636591a26b73",
);
}
Loading
Loading