Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 10 additions & 8 deletions crates/tinybox-drivers/src/repo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,9 @@ impl<T: Serialize + DeserializeOwned + Send + Sync> Repo<T> {
where
F: FnMut(Option<T>) -> Result<Option<T>> + Send,
{
for attempt in 0..self.attempts.max(1) {
let attempts = self.attempts.max(1);
let mut attempt = 0;
loop {
let (precondition, current) = match self.docs.get(&self.collection, id).await? {
Some(found) => {
let found = decode(found)?;
Expand All @@ -122,10 +124,8 @@ impl<T: Serialize + DeserializeOwned + Send + Sync> Repo<T> {
};
match self.write(id, &next, precondition).await {
Ok(version) => return Ok(Some((next, version))),
Err(error) if error.kind() == ErrorKind::Conflict => {
if attempt + 1 == self.attempts.max(1) {
return Err(error);
}
// The last attempt's conflict is returned as it is.
Err(error) if error.kind() == ErrorKind::Conflict && attempt + 1 < attempts => {
let shift = attempt.min(10);
let cap = self.backoff.saturating_mul(1_u32 << shift);
let nanos = u64::try_from(cap.as_nanos()).unwrap_or(u64::MAX);
Expand All @@ -137,13 +137,11 @@ impl<T: Serialize + DeserializeOwned + Send + Sync> Repo<T> {
))
.await;
}
attempt += 1;
}
Err(error) => return Err(error),
}
}
Err(StorageError::conflict(
"repository update retry budget exhausted",
))
}

async fn write(&self, id: &str, value: &T, precondition: Precondition) -> Result<Version> {
Expand All @@ -165,3 +163,7 @@ fn decode<T: DeserializeOwned>(found: Versioned<serde_json::Value>) -> Result<Ve
meta: found.meta,
})
}

#[cfg(test)]
#[path = "repo_tests.rs"]
mod tests;
86 changes: 86 additions & 0 deletions crates/tinybox-drivers/src/repo_tests.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
//! Tests for the typed repository: its debug form and how `update` ends.

use super::*;
use crate::faulty::{Faults, Faulty};
use serde_json::{Value, json};
use tinystoragedrivers_testkit::memory_storage;
use tinystoragedrivers_types::{CollectionSpec, Scope};

type TestResult = std::result::Result<(), Box<dyn std::error::Error>>;

const COLLECTION: &str = tinybox_bus::storage::BOX_COLLECTION;

async fn seeded() -> std::result::Result<Arc<dyn DocumentStore>, Box<dyn std::error::Error>> {
let docs = memory_storage(&Scope::new("tinybox-repo-tests")?)
.documents()
.clone();
docs.ensure_collection(&CollectionSpec::new(COLLECTION))
.await?;
Repo::<Value>::new(&docs, COLLECTION)?
.put("a", &json!({ "n": 1 }))
.await?;
Ok(docs)
}

fn bump(current: Option<Value>) -> Value {
let n = current.and_then(|doc| doc["n"].as_u64()).unwrap_or(0);
json!({ "n": n + 1 })
}

#[tokio::test]
async fn the_debug_form_names_the_collection_and_not_the_backend() -> TestResult {
let repo = Repo::<Value>::new(&seeded().await?, COLLECTION)?;
let debug = format!("{repo:?}");
assert!(debug.starts_with("Repo"), "{debug}");
assert!(debug.contains(COLLECTION), "{debug}");
assert!(!debug.contains("docs"), "{debug}");
Ok(())
}

#[tokio::test]
async fn an_update_whose_write_fails_returns_that_error_without_retrying() -> TestResult {
let inner = seeded().await?;
let faults = Faults {
put: true,
..Faults::default()
};
let docs: Arc<dyn DocumentStore> = Faulty::new(Arc::clone(&inner), faults);
let mut calls = 0;
let result = Repo::<Value>::new(&docs, COLLECTION)?
.update("a", |current| {
calls += 1;
Ok(Some(bump(current)))
})
.await;
let error = result
.err()
.ok_or("a failed write was reported as written")?;
assert_ne!(error.kind(), ErrorKind::Conflict, "{error}");
assert_eq!(calls, 1, "only a conflict is retried");
let stored = Repo::<Value>::new(&inner, COLLECTION)?.get("a").await?;
assert_eq!(stored.map(|found| found.doc), Some(json!({ "n": 1 })));
Ok(())
}

#[tokio::test]
async fn an_update_that_always_conflicts_returns_the_last_conflict_after_its_budget() -> TestResult
{
let inner = seeded().await?;
let docs: Arc<dyn DocumentStore> = Faulty::with_conflicts(Arc::clone(&inner), usize::MAX);
let mut repo = Repo::<Value>::new(&docs, COLLECTION)?;
repo.attempts = 3;
let mut calls = 0;
let result = repo
.update("a", |current| {
calls += 1;
Ok(Some(bump(current)))
})
.await;
let error = result
.err()
.ok_or("an update that never landed succeeded")?;
assert_eq!(error.kind(), ErrorKind::Conflict);
assert!(error.to_string().contains("injected conflict"), "{error}");
assert_eq!(calls, 3, "one try per attempt");
Ok(())
}
50 changes: 50 additions & 0 deletions crates/tinybox-drivers/src/template_store_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -216,3 +216,53 @@ async fn template_port_failures_keep_the_operation_context() -> TestResult {
));
Ok(())
}

#[tokio::test(flavor = "multi_thread")]
async fn an_import_reports_a_failed_insert_and_a_conflict_with_no_stored_row() -> TestResult {
let base = storage("template-import-failures")?;
let faulty = Faulty::new(Arc::clone(base.documents()), Faults::default());
let wrapped = Storage::new(
faulty.clone(),
Arc::clone(base.streams()),
Arc::clone(base.kv()),
Arc::clone(base.blobs()),
Arc::clone(base.queues()),
Arc::clone(base.secrets()),
);
let store = DocumentTemplateStore::open(wrapped).await?;
let name = TemplateName::new("legacy")?;
let snapshot = SnapshotId::new("snapshot")?;

// Anything but a conflict is passed through as it is.
faulty
.failing
.faults
.fail_operation_nth("put", 1, ErrorKind::Backend);
assert!(matches!(
store.import_if_missing(&name, &snapshot),
Err(Error::Store {
operation: "import",
..
})
));

// A conflict with nothing behind it: the row vanished between the two
// calls, which the import reports rather than retrying blindly.
faulty
.failing
.faults
.fail_operation_nth("put", 1, ErrorKind::Conflict);
let vanished = store.import_if_missing(&name, &snapshot);
assert!(
matches!(
&vanished,
Err(Error::Store { operation: "import", message }) if message.contains("disappeared")
),
"{vanished:?}"
);
assert!(matches!(
store.get(&name),
Err(Error::UnknownTemplate { .. })
));
Ok(())
}
80 changes: 80 additions & 0 deletions crates/tinybox-host/src/files_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -321,3 +321,83 @@ async fn concurrent_symlink_rename_cannot_redirect_a_staged_write_outside() -> R
);
Ok(())
}

/// A staged file whose flush or sync fails, as a full disk would.
#[derive(Debug)]
struct UnsyncableFile {
inner: cap_std::fs::File,
fail_flush: bool,
}

impl Write for UnsyncableFile {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
self.inner.write(bytes)
}

fn flush(&mut self) -> std::io::Result<()> {
if self.fail_flush {
return Err(std::io::Error::other("injected flush failure"));
}
self.inner.flush()
}
}

impl super::StagedFile for UnsyncableFile {
fn sync_all(&self) -> std::io::Result<()> {
Err(std::io::Error::other("injected sync failure"))
}
}

fn staged_writer<W>(dir: cap_std::fs::Dir, file: W, next_offset: u64) -> LocalFileWriter<W> {
LocalFileWriter {
dir,
temporary: PathBuf::from(".target.bin.tinybox-transfer.tmp"),
destination: PathBuf::from("target.bin"),
file: Some(file),
next_offset,
finished: false,
poisoned: false,
}
}

#[tokio::test]
async fn a_write_past_the_largest_offset_is_refused_before_it_is_written() -> Result<()> {
let root = tempfile::tempdir().map_err(|error| Error::io("create fixture", &error))?;
let dir = open_root(root.path())?;
let mut options = cap_std::fs::OpenOptions::new();
options.write(true).create_new(true);
let file = dir
.open_with(".target.bin.tinybox-transfer.tmp", &options)
.map_err(|error| Error::io("create staged fixture", &error))?;
let last = u64::MAX - 1;
let mut writer = staged_writer(dir, file, last);
let refused = writer.write_chunk(last, b"data").await;
assert!(matches!(refused, Err(Error::Io { .. })), "{refused:?}");
let staged = std::fs::metadata(root.path().join(".target.bin.tinybox-transfer.tmp"))
.map_err(|error| Error::io("inspect staged fixture", &error))?;
assert_eq!(staged.len(), 0, "nothing was written");
Ok(())
}

#[tokio::test]
async fn a_failed_flush_or_sync_never_publishes_the_staged_file() -> Result<()> {
for fail_flush in [true, false] {
let root = tempfile::tempdir().map_err(|error| Error::io("create fixture", &error))?;
let dir = open_root(root.path())?;
let mut options = cap_std::fs::OpenOptions::new();
options.write(true).create_new(true);
let inner = dir
.open_with(".target.bin.tinybox-transfer.tmp", &options)
.map_err(|error| Error::io("create staged fixture", &error))?;
let mut writer = staged_writer(dir, UnsyncableFile { inner, fail_flush }, 0);
writer.write_chunk(0, b"payload").await?;
let failed = writer.finish().await;
let expected = if fail_flush { "flush" } else { "sync" };
assert!(
matches!(&failed, Err(error) if error.to_string().contains(expected)),
"{expected}: {failed:?}"
);
assert!(!root.path().join("target.bin").exists(), "{expected}");
}
Ok(())
}
4 changes: 4 additions & 0 deletions crates/tinybox-module/src/tinybus_module/resources.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1856,3 +1856,7 @@ fn validate_id(id: &ResourceId) -> Result<()> {
#[cfg(test)]
#[path = "resources_tests.rs"]
mod tests;

#[cfg(test)]
#[path = "resources_mapping_tests.rs"]
mod mapping_tests;
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
//! Tests for the pure mappings between bus requests and the core model.

use super::*;

#[test]
fn an_allowlist_policy_reaches_the_box_spec_host_for_host_and_port_for_port() -> Result<()> {
let request = CreateRequest {
resource: ResourceId("allowlist-test".into()),
backend: "docker".into(),
workspace: Workspace::Directory("/work".into()),
network: tinybox_bus::NetworkPolicy::Allowlist(tinybox_bus::EgressAllowlist {
hosts: vec!["example.com".into(), "192.0.2.7".into()],
ports: vec![443, 8443],
}),
..Default::default()
};
let spec = make_spec(&request, "local")?;
assert_eq!(
spec.network,
tinybox_core::NetworkPolicy::Allowlist(tinybox_core::EgressAllowlist {
hosts: vec!["example.com".into(), "192.0.2.7".into()],
ports: vec![443, 8443],
})
);
Ok(())
}

#[test]
fn file_errors_keep_their_stable_names() {
let name = |error: &tinybox_core::Error| match file_error(error) {
tinybus::Error::MethodFailed { name, .. } => Some(name),
_ => None,
};
for (error, expected) in [
(
tinybox_core::Error::UnsupportedHostFileTransfer { host: "ssh".into() },
tinybox_bus::FILE_UNSUPPORTED,
),
(
tinybox_core::Error::InvalidWorkspacePath,
tinybox_bus::INVALID_FILE_PATH,
),
(
tinybox_core::Error::InvalidFileTransfer { reason: "offset" },
tinybox_bus::INVALID_FILE_CHUNK,
),
(
tinybox_core::Error::io("read", &std::io::Error::other("disk")),
tinybox_bus::FILE_ERROR,
),
] {
assert_eq!(name(&error).as_deref(), Some(expected), "{error}");
}
}

#[test]
fn agent37_executes_on_every_platform_and_other_backends_only_on_supported_ones() {
for platform in [
super::super::Platform::Linux,
super::super::Platform::Unix,
super::super::Platform::Windows,
super::super::Platform::Other,
] {
assert!(supports_execution(platform, "agent37"));
}
assert!(!supports_execution(super::super::Platform::Other, "docker"));
assert!(supports_execution(super::super::Platform::Linux, "docker"));
assert!(!supports_execution(
super::super::Platform::Linux,
"namespace"
));
}
Loading
Loading