diff --git a/crates/tinybox-drivers/src/repo.rs b/crates/tinybox-drivers/src/repo.rs index 523455de..6564364a 100644 --- a/crates/tinybox-drivers/src/repo.rs +++ b/crates/tinybox-drivers/src/repo.rs @@ -109,7 +109,9 @@ impl Repo { where F: FnMut(Option) -> Result> + Send, { - for attempt in 0..self.attempts.max(1) { + let attempts = self.attempts.max(1); + let mut attempt = 0; + loop { let (precondition, current) = match self.docs.get(&self.collection, id).await? { Some(found) => { let found = decode(found)?; @@ -122,10 +124,8 @@ impl Repo { }; match self.write(id, &next, precondition).await { Ok(version) => return Ok(Some((next, version))), - Err(error) if error.kind() == ErrorKind::Conflict => { - if attempt + 1 == self.attempts.max(1) { - return Err(error); - } + // The last attempt's conflict is returned as it is. + Err(error) if error.kind() == ErrorKind::Conflict && attempt + 1 < attempts => { let shift = attempt.min(10); let cap = self.backoff.saturating_mul(1_u32 << shift); let nanos = u64::try_from(cap.as_nanos()).unwrap_or(u64::MAX); @@ -137,13 +137,11 @@ impl Repo { )) .await; } + attempt += 1; } Err(error) => return Err(error), } } - Err(StorageError::conflict( - "repository update retry budget exhausted", - )) } async fn write(&self, id: &str, value: &T, precondition: Precondition) -> Result { @@ -165,3 +163,7 @@ fn decode(found: Versioned) -> Result>; + +const COLLECTION: &str = tinybox_bus::storage::BOX_COLLECTION; + +async fn seeded() -> std::result::Result, Box> { + let docs = memory_storage(&Scope::new("tinybox-repo-tests")?) + .documents() + .clone(); + docs.ensure_collection(&CollectionSpec::new(COLLECTION)) + .await?; + Repo::::new(&docs, COLLECTION)? + .put("a", &json!({ "n": 1 })) + .await?; + Ok(docs) +} + +fn bump(current: Option) -> Value { + let n = current.and_then(|doc| doc["n"].as_u64()).unwrap_or(0); + json!({ "n": n + 1 }) +} + +#[tokio::test] +async fn the_debug_form_names_the_collection_and_not_the_backend() -> TestResult { + let repo = Repo::::new(&seeded().await?, COLLECTION)?; + let debug = format!("{repo:?}"); + assert!(debug.starts_with("Repo"), "{debug}"); + assert!(debug.contains(COLLECTION), "{debug}"); + assert!(!debug.contains("docs"), "{debug}"); + Ok(()) +} + +#[tokio::test] +async fn an_update_whose_write_fails_returns_that_error_without_retrying() -> TestResult { + let inner = seeded().await?; + let faults = Faults { + put: true, + ..Faults::default() + }; + let docs: Arc = Faulty::new(Arc::clone(&inner), faults); + let mut calls = 0; + let result = Repo::::new(&docs, COLLECTION)? + .update("a", |current| { + calls += 1; + Ok(Some(bump(current))) + }) + .await; + let error = result + .err() + .ok_or("a failed write was reported as written")?; + assert_ne!(error.kind(), ErrorKind::Conflict, "{error}"); + assert_eq!(calls, 1, "only a conflict is retried"); + let stored = Repo::::new(&inner, COLLECTION)?.get("a").await?; + assert_eq!(stored.map(|found| found.doc), Some(json!({ "n": 1 }))); + Ok(()) +} + +#[tokio::test] +async fn an_update_that_always_conflicts_returns_the_last_conflict_after_its_budget() -> TestResult +{ + let inner = seeded().await?; + let docs: Arc = Faulty::with_conflicts(Arc::clone(&inner), usize::MAX); + let mut repo = Repo::::new(&docs, COLLECTION)?; + repo.attempts = 3; + let mut calls = 0; + let result = repo + .update("a", |current| { + calls += 1; + Ok(Some(bump(current))) + }) + .await; + let error = result + .err() + .ok_or("an update that never landed succeeded")?; + assert_eq!(error.kind(), ErrorKind::Conflict); + assert!(error.to_string().contains("injected conflict"), "{error}"); + assert_eq!(calls, 3, "one try per attempt"); + Ok(()) +} diff --git a/crates/tinybox-drivers/src/template_store_tests.rs b/crates/tinybox-drivers/src/template_store_tests.rs index 5914f538..01b36b31 100644 --- a/crates/tinybox-drivers/src/template_store_tests.rs +++ b/crates/tinybox-drivers/src/template_store_tests.rs @@ -216,3 +216,53 @@ async fn template_port_failures_keep_the_operation_context() -> TestResult { )); Ok(()) } + +#[tokio::test(flavor = "multi_thread")] +async fn an_import_reports_a_failed_insert_and_a_conflict_with_no_stored_row() -> TestResult { + let base = storage("template-import-failures")?; + let faulty = Faulty::new(Arc::clone(base.documents()), Faults::default()); + let wrapped = Storage::new( + faulty.clone(), + Arc::clone(base.streams()), + Arc::clone(base.kv()), + Arc::clone(base.blobs()), + Arc::clone(base.queues()), + Arc::clone(base.secrets()), + ); + let store = DocumentTemplateStore::open(wrapped).await?; + let name = TemplateName::new("legacy")?; + let snapshot = SnapshotId::new("snapshot")?; + + // Anything but a conflict is passed through as it is. + faulty + .failing + .faults + .fail_operation_nth("put", 1, ErrorKind::Backend); + assert!(matches!( + store.import_if_missing(&name, &snapshot), + Err(Error::Store { + operation: "import", + .. + }) + )); + + // A conflict with nothing behind it: the row vanished between the two + // calls, which the import reports rather than retrying blindly. + faulty + .failing + .faults + .fail_operation_nth("put", 1, ErrorKind::Conflict); + let vanished = store.import_if_missing(&name, &snapshot); + assert!( + matches!( + &vanished, + Err(Error::Store { operation: "import", message }) if message.contains("disappeared") + ), + "{vanished:?}" + ); + assert!(matches!( + store.get(&name), + Err(Error::UnknownTemplate { .. }) + )); + Ok(()) +} diff --git a/crates/tinybox-host/src/files_tests.rs b/crates/tinybox-host/src/files_tests.rs index 6deed474..73140d95 100644 --- a/crates/tinybox-host/src/files_tests.rs +++ b/crates/tinybox-host/src/files_tests.rs @@ -321,3 +321,83 @@ async fn concurrent_symlink_rename_cannot_redirect_a_staged_write_outside() -> R ); Ok(()) } + +/// A staged file whose flush or sync fails, as a full disk would. +#[derive(Debug)] +struct UnsyncableFile { + inner: cap_std::fs::File, + fail_flush: bool, +} + +impl Write for UnsyncableFile { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.inner.write(bytes) + } + + fn flush(&mut self) -> std::io::Result<()> { + if self.fail_flush { + return Err(std::io::Error::other("injected flush failure")); + } + self.inner.flush() + } +} + +impl super::StagedFile for UnsyncableFile { + fn sync_all(&self) -> std::io::Result<()> { + Err(std::io::Error::other("injected sync failure")) + } +} + +fn staged_writer(dir: cap_std::fs::Dir, file: W, next_offset: u64) -> LocalFileWriter { + LocalFileWriter { + dir, + temporary: PathBuf::from(".target.bin.tinybox-transfer.tmp"), + destination: PathBuf::from("target.bin"), + file: Some(file), + next_offset, + finished: false, + poisoned: false, + } +} + +#[tokio::test] +async fn a_write_past_the_largest_offset_is_refused_before_it_is_written() -> Result<()> { + let root = tempfile::tempdir().map_err(|error| Error::io("create fixture", &error))?; + let dir = open_root(root.path())?; + let mut options = cap_std::fs::OpenOptions::new(); + options.write(true).create_new(true); + let file = dir + .open_with(".target.bin.tinybox-transfer.tmp", &options) + .map_err(|error| Error::io("create staged fixture", &error))?; + let last = u64::MAX - 1; + let mut writer = staged_writer(dir, file, last); + let refused = writer.write_chunk(last, b"data").await; + assert!(matches!(refused, Err(Error::Io { .. })), "{refused:?}"); + let staged = std::fs::metadata(root.path().join(".target.bin.tinybox-transfer.tmp")) + .map_err(|error| Error::io("inspect staged fixture", &error))?; + assert_eq!(staged.len(), 0, "nothing was written"); + Ok(()) +} + +#[tokio::test] +async fn a_failed_flush_or_sync_never_publishes_the_staged_file() -> Result<()> { + for fail_flush in [true, false] { + let root = tempfile::tempdir().map_err(|error| Error::io("create fixture", &error))?; + let dir = open_root(root.path())?; + let mut options = cap_std::fs::OpenOptions::new(); + options.write(true).create_new(true); + let inner = dir + .open_with(".target.bin.tinybox-transfer.tmp", &options) + .map_err(|error| Error::io("create staged fixture", &error))?; + let mut writer = staged_writer(dir, UnsyncableFile { inner, fail_flush }, 0); + writer.write_chunk(0, b"payload").await?; + let failed = writer.finish().await; + let expected = if fail_flush { "flush" } else { "sync" }; + assert!( + matches!(&failed, Err(error) if error.to_string().contains(expected)), + "{expected}: {failed:?}" + ); + assert!(!root.path().join("target.bin").exists(), "{expected}"); + } + Ok(()) +} diff --git a/crates/tinybox-module/src/tinybus_module/resources.rs b/crates/tinybox-module/src/tinybus_module/resources.rs index b3e3802b..24414646 100644 --- a/crates/tinybox-module/src/tinybus_module/resources.rs +++ b/crates/tinybox-module/src/tinybus_module/resources.rs @@ -1856,3 +1856,7 @@ fn validate_id(id: &ResourceId) -> Result<()> { #[cfg(test)] #[path = "resources_tests.rs"] mod tests; + +#[cfg(test)] +#[path = "resources_mapping_tests.rs"] +mod mapping_tests; diff --git a/crates/tinybox-module/src/tinybus_module/resources_mapping_tests.rs b/crates/tinybox-module/src/tinybus_module/resources_mapping_tests.rs new file mode 100644 index 00000000..fa84a9e2 --- /dev/null +++ b/crates/tinybox-module/src/tinybus_module/resources_mapping_tests.rs @@ -0,0 +1,72 @@ +//! Tests for the pure mappings between bus requests and the core model. + +use super::*; + +#[test] +fn an_allowlist_policy_reaches_the_box_spec_host_for_host_and_port_for_port() -> Result<()> { + let request = CreateRequest { + resource: ResourceId("allowlist-test".into()), + backend: "docker".into(), + workspace: Workspace::Directory("/work".into()), + network: tinybox_bus::NetworkPolicy::Allowlist(tinybox_bus::EgressAllowlist { + hosts: vec!["example.com".into(), "192.0.2.7".into()], + ports: vec![443, 8443], + }), + ..Default::default() + }; + let spec = make_spec(&request, "local")?; + assert_eq!( + spec.network, + tinybox_core::NetworkPolicy::Allowlist(tinybox_core::EgressAllowlist { + hosts: vec!["example.com".into(), "192.0.2.7".into()], + ports: vec![443, 8443], + }) + ); + Ok(()) +} + +#[test] +fn file_errors_keep_their_stable_names() { + let name = |error: &tinybox_core::Error| match file_error(error) { + tinybus::Error::MethodFailed { name, .. } => Some(name), + _ => None, + }; + for (error, expected) in [ + ( + tinybox_core::Error::UnsupportedHostFileTransfer { host: "ssh".into() }, + tinybox_bus::FILE_UNSUPPORTED, + ), + ( + tinybox_core::Error::InvalidWorkspacePath, + tinybox_bus::INVALID_FILE_PATH, + ), + ( + tinybox_core::Error::InvalidFileTransfer { reason: "offset" }, + tinybox_bus::INVALID_FILE_CHUNK, + ), + ( + tinybox_core::Error::io("read", &std::io::Error::other("disk")), + tinybox_bus::FILE_ERROR, + ), + ] { + assert_eq!(name(&error).as_deref(), Some(expected), "{error}"); + } +} + +#[test] +fn agent37_executes_on_every_platform_and_other_backends_only_on_supported_ones() { + for platform in [ + super::super::Platform::Linux, + super::super::Platform::Unix, + super::super::Platform::Windows, + super::super::Platform::Other, + ] { + assert!(supports_execution(platform, "agent37")); + } + assert!(!supports_execution(super::super::Platform::Other, "docker")); + assert!(supports_execution(super::super::Platform::Linux, "docker")); + assert!(!supports_execution( + super::super::Platform::Linux, + "namespace" + )); +} diff --git a/crates/tinybox-module/src/tinybus_module/resources_tests.rs b/crates/tinybox-module/src/tinybus_module/resources_tests.rs index ec45990e..e5caf4cb 100644 --- a/crates/tinybox-module/src/tinybus_module/resources_tests.rs +++ b/crates/tinybox-module/src/tinybus_module/resources_tests.rs @@ -3478,3 +3478,195 @@ async fn agent37_exec_is_admitted_through_the_module_guard() -> Result<()> { resources.close(&id).await?; Ok(()) } + +/// A remote host whose file providers break their contract: the reader returns +/// more than it was asked for and the writer acknowledges a wrong offset. +#[derive(Debug, Default)] +struct OverreachingFileHost { + remote: RemoteProcessHost, +} + +#[derive(Debug)] +struct OverreadingReader; + +#[async_trait::async_trait] +impl tinybox_core::WorkspaceFileReader for OverreadingReader { + fn size(&self) -> u64 { + 16 + } + + async fn read_chunk( + &mut self, + _offset: u64, + max_bytes: usize, + ) -> tinybox_core::Result> { + Ok(vec![b'x'; max_bytes + 1]) + } +} + +#[derive(Debug)] +struct MisacknowledgingWriter; + +#[async_trait::async_trait] +impl tinybox_core::WorkspaceFileWriter for MisacknowledgingWriter { + async fn write_chunk(&mut self, offset: u64, bytes: &[u8]) -> tinybox_core::Result { + Ok(offset + bytes.len() as u64 + 1) + } + + async fn finish(&mut self) -> tinybox_core::Result { + Ok(0) + } + + async fn abort(&mut self) -> tinybox_core::Result<()> { + Ok(()) + } +} + +#[async_trait::async_trait] +impl Host for OverreachingFileHost { + fn name(&self) -> &'static str { + tinybox_ssh::NAME + } + + async fn run(&self, request: &NativeRequest) -> tinybox_core::Result { + self.remote.run(request).await + } + + async fn open_workspace_file( + &self, + _root: &std::path::Path, + _relative: &std::path::Path, + ) -> tinybox_core::Result> { + Ok(Box::new(OverreadingReader)) + } + + async fn begin_workspace_file_write( + &self, + _root: &std::path::Path, + _relative: &std::path::Path, + _transfer_id: &str, + ) -> tinybox_core::Result> { + Ok(Box::new(MisacknowledgingWriter)) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_provider_that_overreads_or_misacknowledges_is_a_backend_error() -> Result<()> { + let resources = Resources::test_resources(); + let resource = resources.reserve(ReserveRequest::Resource).await?; + resources + .create_on( + CreateRequest { + resource: resource.clone(), + backend: "passthrough".into(), + workspace: Workspace::Directory(".".into()), + ..Default::default() + }, + Arc::new(OverreachingFileHost::default()), + ) + .await?; + + let transfer = resources + .reserve(ReserveRequest::FileRead(resource.clone())) + .await?; + resources + .begin_file_read(BeginFileReadRequest { + resource: resource.clone(), + transfer: transfer.clone(), + path: "data.bin".into(), + }) + .await?; + let overread = resources + .read_file_chunk(ReadFileChunkRequest { + resource: resource.clone(), + transfer, + offset: 0, + max_bytes: 4, + }) + .await; + assert_eq!( + error_name(overread.err()).as_deref(), + Some(tinybox_bus::BACKEND_ERROR) + ); + + let transfer = resources + .reserve(ReserveRequest::FileWrite(resource.clone())) + .await?; + resources + .begin_file_write(BeginFileWriteRequest { + resource: resource.clone(), + transfer: transfer.clone(), + path: "data.bin".into(), + }) + .await?; + let chunk = WriteFileChunkRequest { + resource: resource.clone(), + transfer, + offset: 0, + bytes: b"data".to_vec(), + }; + let misacknowledged = resources.write_file_chunk(chunk.clone()).await; + assert_eq!( + error_name(misacknowledged.err()).as_deref(), + Some(tinybox_bus::BACKEND_ERROR) + ); + // The bad acknowledgement was not taken as progress. + let retried = resources.write_file_chunk(chunk).await; + assert_eq!( + error_name(retried.err()).as_deref(), + Some(tinybox_bus::BACKEND_ERROR) + ); + resources.close(&resource).await?; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_finished_write_replays_for_its_own_path_and_refuses_another() -> Result<()> { + let workspace = tempfile::tempdir() + .map_err(|error| Error::failed(format!("create workspace fixture: {error}")))?; + let resources = Resources::test_resources(); + let resource = resources.reserve(ReserveRequest::Resource).await?; + resources + .create(CreateRequest { + resource: resource.clone(), + backend: "passthrough".into(), + workspace: Workspace::Directory(workspace.path().to_string_lossy().into_owned()), + ..Default::default() + }) + .await?; + let transfer = resources + .reserve(ReserveRequest::FileWrite(resource.clone())) + .await?; + let begin = |path: &str| BeginFileWriteRequest { + resource: resource.clone(), + transfer: transfer.clone(), + path: path.into(), + }; + resources.begin_file_write(begin("out.bin")).await?; + resources + .write_file_chunk(WriteFileChunkRequest { + resource: resource.clone(), + transfer: transfer.clone(), + offset: 0, + bytes: b"data".to_vec(), + }) + .await?; + resources + .finish_file_write(FinishFileWriteRequest { + resource: resource.clone(), + transfer: transfer.clone(), + }) + .await?; + + // A retried begin after the finish reports where the write ended. + let replayed = resources.begin_file_write(begin("out.bin")).await?; + assert_eq!(replayed.next_offset, 4); + let elsewhere = resources.begin_file_write(begin("other.bin")).await; + assert_eq!( + error_name(elsewhere.err()).as_deref(), + Some(tinybox_bus::INVALID_ID) + ); + assert!(!workspace.path().join("other.bin").exists()); + resources.close(&resource).await?; + Ok(()) +} diff --git a/scripts/check-coverage.sh b/scripts/check-coverage.sh index 3ff71a96..0478ce4f 100755 --- a/scripts/check-coverage.sh +++ b/scripts/check-coverage.sh @@ -2,7 +2,14 @@ set -euo pipefail root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" cd "$root" +# The gate's line accounting depends on the compiler: the same tests over the +# same code count different lines as coverable, and as covered, under another +# rustc. So the gate runs on one pinned toolchain, and the pin is bumped on +# purpose, together with whatever the new numbers need, never by a stable +# release landing on the runner. +export RUSTUP_TOOLCHAIN=1.99.1 if [[ "${GITHUB_ACTIONS:-}" == true && "${RUNNER_OS:-}" == Linux ]]; then + rustup toolchain install "$RUSTUP_TOOLCHAIN" --profile minimal --component llvm-tools source scripts/ci/prepare-rootless-native-resources.sh export TINYBOX_REQUIRE_NATIVE_RESOURCE_TESTS=1 fi