diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml new file mode 100644 index 000000000..d7777def1 --- /dev/null +++ b/.github/codeql/codeql-config.yml @@ -0,0 +1,4 @@ +name: "Gitlawb CodeQL config" +paths-ignore: + - "**/tests/**" + - "**/test_support/**" diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 752c55baa..6c1632440 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -195,6 +195,16 @@ jobs: # shellcheck disable=SC2086 # PACKAGES is a word list by design cargo test --locked $PACKAGES + # The real-node deny harness is behind the `test-harness` feature (so its + # spawn surface never compiles into the production binary), which means + # `--workspace` above skips it. Run it explicitly so the trust-boundary + # regression cases (INV-1/INV-2/INV-8) execute on every PR. Uses the same + # Postgres service; `#[sqlx::test]` provisions an isolated DB per test. + - name: cargo test (real-node deny harness) + env: + DATABASE_URL: postgres://postgres:postgres@localhost:5432/gitlawb_test + run: cargo test --locked -p gitlawb-node --features test-harness --lib --test deny_harness + build-release: name: build --release needs: changes diff --git a/Dockerfile b/Dockerfile index f030de508..14bee67f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,11 +26,15 @@ COPY crates/icaptcha-client/Cargo.toml crates/icaptcha-client/ # is what let it sit dead from the moment gl gained a path dependency on # icaptcha-client. A failure here should stop the build, not silently turn every # image build into a cold one. +# gitlawb-node declares BOTH a [[bin]] and a [lib], so it needs both stubs: +# cargo resolves every declared target before it fetches anything, and a +# missing src/lib.rs aborts the layer with a target resolution error. RUN mkdir -p crates/gitlawb-core/src crates/gitlawb-node/src crates/gl/src \ crates/git-remote-gitlawb/src crates/gitlawb-attest/src crates/icaptcha-client/src && \ echo 'fn main() {}' > crates/gitlawb-node/src/main.rs && \ echo 'fn main() {}' > crates/gl/src/main.rs && \ echo 'fn main() {}' > crates/git-remote-gitlawb/src/main.rs && \ + echo '' > crates/gitlawb-node/src/lib.rs && \ echo '' > crates/gitlawb-core/src/lib.rs && \ echo '' > crates/gitlawb-attest/src/lib.rs && \ echo '' > crates/icaptcha-client/src/lib.rs && \ diff --git a/crates/gitlawb-node/Cargo.toml b/crates/gitlawb-node/Cargo.toml index f9647e70b..c06780f6c 100644 --- a/crates/gitlawb-node/Cargo.toml +++ b/crates/gitlawb-node/Cargo.toml @@ -6,10 +6,28 @@ edition.workspace = true rust-version.workspace = true license.workspace = true +[lib] +name = "gitlawb_node" +path = "src/lib.rs" + [[bin]] name = "gitlawb-node" path = "src/main.rs" +# Exposes the `test_harness` spawn surface (src/test_harness.rs) to the +# real-node deny-harness integration crate. Off by default so the production +# binary never compiles test-only boot code. Enable with +# `cargo test -p gitlawb-node --features test-harness`. +[features] +test-harness = ["dep:tempfile"] + +# Only builds when `test-harness` is enabled; otherwise skipped (not an error), +# so `cargo test --workspace` without the feature stays green. +[[test]] +name = "deny_harness" +path = "tests/deny_harness.rs" +required-features = ["test-harness"] + [dependencies] gitlawb-core = { path = "../gitlawb-core" } ed25519-dalek = { workspace = true } @@ -78,10 +96,19 @@ alloy = { version = "1", default-features = false, features = [ ] } rand = { workspace = true } libp2p-dns = { version = "0.45.0", features = ["tokio"] } +# Optional: only needed by the test-harness feature's lib modules +# (test_git_shim, test_harness); dev-deps are invisible to lib code. +tempfile = { version = "3", optional = true } [dev-dependencies] mockito = "1" tempfile = "3" -# `test-util` for `#[tokio::test(start_paused = true)]`: lets the deadline tests assert -# on which timer fired instead of on wall clock. +# test-util enables tokio::test(start_paused): the delayed claim re-sweep +# test drives the real CLAIM_SWEEP_MIN_AGE-plus-margin sleep on paused time. tokio = { workspace = true, features = ["test-util"] } +# Used by the deny-harness integration crate (tests/deny_harness.rs): the +# #[sqlx::test] macro for an ephemeral per-test pool, and reqwest as the real +# HTTP client that drives deny paths over the socket. +sqlx = { version = "0.8", features = ["postgres", "runtime-tokio-rustls", "macros", "migrate"] } +reqwest = { workspace = true } +hex = "0.4" diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index d219a3efe..0e3d4930c 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -5352,7 +5352,7 @@ const MAX_REF_UPDATES: usize = 8192; /// replication tail at the durability boundary while the certificate and webhook /// loops below still iterate their own copy (#174 U5). #[derive(Clone, Debug)] -pub(crate) struct RefUpdate { +pub struct RefUpdate { pub(crate) old_sha: String, pub(crate) new_sha: String, pub(crate) ref_name: String, diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index cbb8296c1..01cebf2bc 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -278,9 +278,9 @@ pub struct Config { #[arg(long, env = "GITLAWB_METRICS_ADDR", default_value = "")] pub metrics_addr: String, - /// Maximum time to wait for in-flight requests to drain on shutdown, in - /// seconds. After this elapses, the server returns 503 to anything still - /// in flight and exits. Default: 30s. + /// Shutdown grace setting, in seconds. Currently not enforced: the value + /// is logged at startup and axum drains in-flight requests on the shutdown + /// signal with no upper bound. Default: 30s. #[arg(long, env = "GITLAWB_SHUTDOWN_GRACE_SECS", default_value_t = 30)] pub shutdown_grace_secs: u64, diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 9dd89e448..d1fa22e25 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -786,27 +786,27 @@ pub struct Db { } impl Db { - #[cfg(test)] - pub fn for_testing(pool: PgPool) -> Self { - Self { pool } - } - /// Test-only: access the underlying Postgres connection pool. /// /// The production accessor was removed on main (ed76adf) once every /// production caller had migrated to typed `Db` methods; this restores /// it for tests only, for the raw-SQL fixtures and closed-pool cases in /// this change that have no typed equivalent. - #[cfg(test)] + #[cfg(any(test, feature = "test-harness"))] pub fn pool(&self) -> &PgPool { &self.pool } + #[cfg(any(test, feature = "test-harness"))] + pub fn for_testing(pool: PgPool) -> Self { + Self { pool } + } + /// Test-only: apply the full schema to a fresh pool. `#[sqlx::test]` /// provisions an empty per-test database, so DB-backed tests must run this /// before seeding. Reuses the production `migrate()` path (the advisory lock /// is harmless on an isolated test DB and migrations are idempotent). - #[cfg(test)] + #[cfg(any(test, feature = "test-harness"))] pub(crate) async fn run_migrations(&self) -> Result<()> { self.migrate().await } @@ -15398,8 +15398,8 @@ mod peer_authority_tests { /// | `prune_non_public_peers` (db/mod.rs) | a delete keyed on a computed bad-DID array; cannot repoint; boot-only caller in main.rs | /// | `seed_local_peer` (sync.rs) | excluded by test-module location: a deliberate `upsert_peer` bypass for `file://` fixtures, which the public-URL gate rejects | /// | `a_legacy_row_can_still_refresh_its_liveness` (db/mod.rs) | test-only. Seeds a PRE-GATE row by raw SQL on purpose: `upsert_peer` cannot create one, since the gate it is testing refuses exactly that DID. The fixture models what a deployed table already holds | -/// | `gossip_ping_round_requires_two_failures_before_persisting_unreachable` (main.rs) | test-only fixture seed. Raw SQL because the test drives the readiness HYSTERESIS, which needs a row already at `last_ping_ok = TRUE` before the round runs; it never exercises the announce gate | /// | `manual_ping_uses_readiness_without_mutating_federation_gate` (api/peers.rs) | test-only fixture seed, same shape and same reason: the row under test must pre-exist so the assertion is about what the ping does NOT rewrite | +/// | `gossip_ping_round_requires_two_failures_before_persisting_unreachable` (lib.rs) | test-only fixture seed for the gossip hysteresis round; the row must pre-exist so the assertion is about when `mark_peer_ping` fires | /// | `seed_peer` (p2p/mod.rs) | test-only fixture seed. Raw SQL so the gossip ingest tests can drive the known-peer gate directly, including the did:web case `upsert_peer` would refuse | /// | `federated_peer_query_is_bounded` (api/repos.rs) | test-only fixture seed for the bounded federation query; inserts reachable rows without exercising peer admission | /// | `federated_route_annotates_peer_repos_on_real_200_body` (api/repos.rs) | test-only fixture seed; inserts the mock peer row the bounded federation query reads, so the response-envelope assertions run against a real HTTP 200 body | @@ -15498,11 +15498,11 @@ mod peers_table_writer_guard { 1, ), ( - "gossip_ping_round_requires_two_failures_before_persisting_unreachable", + "manual_ping_uses_readiness_without_mutating_federation_gate", 1, ), ( - "manual_ping_uses_readiness_without_mutating_federation_gate", + "gossip_ping_round_requires_two_failures_before_persisting_unreachable", 1, ), ("mark_peer_ping", 1), diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 57a9a87f6..c3cfe264d 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -82,18 +82,9 @@ pub struct RepoStore { } impl RepoStore { - #[cfg(test)] + #[cfg(any(test, feature = "test-harness"))] pub fn for_testing(repos_dir: PathBuf, lock_pool: PgPool) -> Self { - Self { - repos_dir, - tigris: None, - lock_pool, - lock_held_transfer_timeout: Duration::from_secs(300), - lock_acquire_deadline: LOCK_ACQUIRE_DEADLINE, - migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), - pre_unlock_gate: None, - upload_site_reached: Arc::new(std::sync::atomic::AtomicUsize::new(0)), - } + Self::new(repos_dir, None, lock_pool, Duration::from_secs(300)) } /// Same as [`RepoStore::for_testing`] but with Tigris enabled, so the paths @@ -1237,7 +1228,7 @@ async fn retry_fork_archive_delete( /// `remove_dir_all` / `rename` can take it and static analysers can treat it as /// sanitised input (CodeQL `rust/path-injection`). #[derive(Debug, Clone)] -pub(crate) struct ValidatedRepoDiskPath(PathBuf); +pub struct ValidatedRepoDiskPath(PathBuf); impl ValidatedRepoDiskPath { pub(crate) fn as_path(&self) -> &Path { @@ -2408,7 +2399,7 @@ impl RepoWriteGuard { /// Only the UNLOCK has to wait for the reaper. This token carries everything /// the settlement itself needs, so the request future can hold it and settle /// the tree at the disconnect instant while the reaper still holds the lock. -pub(crate) struct TreeSettlement { +pub struct TreeSettlement { local_path: ValidatedRepoDiskPath, repo_name: String, /// Shared with the guard, so the stage read here is the stage the publish diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index 4d2ca23f4..f3a82e901 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -663,7 +663,7 @@ pub(crate) fn publish_lock(local_path: &Path) -> Arc> { /// Owns a snapshot temp dir from extraction through handoff to [`RepoSnapshot`]. /// Dropped when the `spawn_blocking` join result is abandoned, so cancellation /// before the outer future resumes still removes the directory. -pub(crate) struct TempSnapshotDir { +pub struct TempSnapshotDir { path: PathBuf, } diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs new file mode 100644 index 000000000..f1c4c1170 --- /dev/null +++ b/crates/gitlawb-node/src/lib.rs @@ -0,0 +1,3991 @@ +// Crate builds as both a library (`gitlawb_node`) and the `gitlawb-node` binary. +// The library exposes the boot surface (build_router, AppState, Config, +// migrations) so the out-of-crate deny-harness integration crate can spawn a +// real node; `src/main.rs` is a thin shim over `run()`. Modules the harness +// reaches are `pub`; the rest stay crate-private. +mod ans104; +pub mod api; +mod arweave; +mod arweave_v2; +pub mod auth; +mod bootstrap; +mod cert; +pub mod config; +pub mod db; +mod durable_outbox; +mod encrypted_pin; +pub mod error; +pub mod git; +mod graphql; +mod icaptcha; +mod ipfs_pin; +mod metrics; +mod operator; +mod p2p; +mod pinata; +mod rate_limit; +mod reconciliation; +pub mod server; +pub mod state; +mod sync; +#[cfg(feature = "test-harness")] +pub mod test_harness; +#[cfg(any(test, feature = "test-harness"))] +pub mod test_support; +mod upstream_mirror; +mod visibility; +mod webhook_secret; +mod webhooks; + +use anyhow::{anyhow, Context, Result}; +use axum::extract::State; +use axum::http::StatusCode; +use axum::response::IntoResponse; +use axum::routing::get; +use axum::{Json, Router}; +use clap::Parser; +use std::collections::HashSet; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use tokio::net::TcpListener; +use tokio::sync::watch; +use tracing::{error, info, warn}; + +use gitlawb_core::http_sig::sign_request; +use gitlawb_core::identity::Keypair; + +use config::Config; +use db::Db; +use state::AppState; + +#[derive(Clone)] +struct DegradedState { + node_did: String, + db_startup: Arc, + web_url: Option, +} + +/// Two independent counters with no cross-field invariant — atomics, not a +/// lock, so the retry loop and the degraded handlers never contend. +#[derive(Default)] +struct DbStartupStatus { + attempts: AtomicU64, + next_retry_secs: AtomicU64, +} + +pub async fn run() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::from_default_env() + .add_directive("gitlawb_node=debug".parse().unwrap()) + .add_directive("tower_http=info".parse().unwrap()), + ) + .init(); + + let mut config = Config::parse(); + + // Merge the embedded seed list of public network nodes into the runtime + // bootstrap peers. Operators can opt out via GITLAWB_BOOTSTRAP_DISABLE_SEEDS. + bootstrap::merge_seeds(&mut config); + + // Fail fast on config combinations that are individually in-range but jointly + // unsafe — notably a DB pool too small for the concurrent-write cap, which + // would let a push burst starve every other DB path (#174 F1). + config + .validate() + .map_err(|e| anyhow::anyhow!("invalid configuration: {e}"))?; + + if !config.public_read { + warn!( + "GITLAWB_PUBLIC_READ=false has no effect; reads are gated per repository by is_public and path-scoped visibility rules, not this flag" + ); + } + + // Load or generate the node's identity keypair + let keypair = Arc::new(load_or_create_keypair(&config)?); + // Sealing key for the legacy-scan continuation tokens, DERIVED from the identity + // just loaded so it is the same key after a restart (see `derive_scan_token_key`). + let scan_token_key = AppState::derive_scan_token_key(&keypair); + let webhook_secret_key = AppState::derive_webhook_secret_key(&keypair); + let node_did = keypair.did(); + + // One-time metrics init. Must run before any handler that calls into + // `metrics::record_*` so the registry exists when the first event fires. + // Safe to call even when GITLAWB_METRICS_ADDR is unset — those helpers + // are simply no-ops until something reads from the registry. + metrics::init(env!("CARGO_PKG_VERSION"), &node_did.to_string()); + + info!("╔══════════════════════════════════════════╗"); + info!( + "║ Twigpine node v{} ║", + env!("CARGO_PKG_VERSION") + ); + info!("╚══════════════════════════════════════════╝"); + // Process-wide shutdown signal. One sender lives in AppState (cloned + // into every handler); main() keeps a clone and flips it on SIGINT + // or SIGTERM. Tasks that hold a watch::Receiver get notified at + // their next await point. + let (shutdown_tx, _shutdown_rx_for_main) = watch::channel(false); + spawn_shutdown_signal(shutdown_tx.clone()); + + info!(did = %node_did, "node identity"); + info!(addr = %config.bind_addr(), "binding HTTP listener"); + + // Bind HTTP once, before dependency initialization, and keep this socket + // for the life of the process. The degraded server accepts on a dup of the + // same socket, so the degraded→full handoff never closes the port: while + // the full server initializes, connections queue in the shared backlog + // instead of being refused. + let listener = TcpListener::bind(config.bind_addr()) + .await + .with_context(|| format!("failed to bind to {}", config.bind_addr()))?; + let full_std_listener = listener.into_std()?; + let degraded_listener = TcpListener::from_std( + full_std_listener + .try_clone() + .context("failed to clone HTTP listener for degraded server")?, + )?; + + // Metrics must stay observable during a database outage — the degraded + // window is exactly when dashboards need data — so this listener starts + // before the DB connects. + let metrics_handle = if !config.metrics_addr.is_empty() { + match spawn_metrics_server(&config.metrics_addr, shutdown_tx.subscribe()).await { + Ok(handle) => { + info!(addr = %config.metrics_addr, "metrics endpoint listening"); + Some(handle) + } + Err(e) => { + warn!(err = %e, addr = %config.metrics_addr, "failed to start metrics endpoint — continuing without"); + None + } + } + } else { + info!("metrics endpoint disabled (GITLAWB_METRICS_ADDR not set)"); + None + }; + + let db_startup = Arc::new(DbStartupStatus::default()); + let (db_ready_tx, db_ready_rx) = watch::channel(false); + let mut degraded_handle = tokio::spawn(run_degraded_server( + degraded_listener, + node_did.to_string(), + Arc::clone(&db_startup), + config.web_url.clone(), + db_ready_rx, + shutdown_tx.subscribe(), + )); + + // Resolve the p2p identity BEFORE the database connect, both arms of the + // port gate together. + // + // The key load is pure filesystem work with no database dependency, and + // its failure is deliberately non-fatal. Leaving it behind the DB connect + // meant the "HTTP up, p2p off" outcome and the port-zero no-IO guarantee + // were both unobservable whenever the database was unreachable, because + // `connect_db_with_retry` retries indefinitely and never falls through. + // Moving only the load would have left the disabled arm stranded, so the + // whole gate moves and just `p2p::start` stays behind the database. + let p2p_local_key = if config.p2p_port > 0 { + match p2p::load_or_create_p2p_keypair(&config.resolved_p2p_key_path()) { + Ok(local_key) => { + metrics::set_p2p_key_load_failed(false); + Some(local_key) + } + // Non-fatal by policy, and the cost is named rather than hidden: + // the node keeps serving HTTP with a green /health while it is off + // the p2p network entirely. Logged at error with a stable event + // name and mirrored into a metric, so the outage is alertable + // without reading startup logs by hand. + Err(e) => { + error!( + err = %format!("{e:#}"), + event = "p2p_identity_key_load_failed", + "failed to load p2p identity key, continuing without p2p" + ); + metrics::set_p2p_key_load_failed(true); + None + } + } + } else { + info!("p2p disabled (p2p_port = 0)"); + metrics::set_p2p_key_load_failed(false); + None + }; + + // Connect to PostgreSQL database. A transient outage or bad secret should + // not crash-loop the process and hammer the database provider; permanent + // misconfiguration surfaces through error-level logs and the /ready check. + let db = tokio::select! { + db = connect_db_with_retry(&config, Arc::clone(&db_startup), shutdown_tx.subscribe()) => { + match db { + Some(db) => db, + None => { + // Shutdown requested while waiting for the database. The + // degraded server only serves one-shot 503s — abort it + // rather than drain, so a slow client can't stall exit. + degraded_handle.abort(); + return Ok(()); + } + } + } + degraded = &mut degraded_handle => { + if *shutdown_tx.borrow() { + return Ok(()); + } + return match degraded { + Ok(Ok(())) => Err(anyhow!("degraded HTTP server stopped before database became ready")), + Ok(Err(err)) => Err(err.context("degraded HTTP server failed")), + Err(err) => Err(anyhow!("degraded HTTP server task failed: {err}")), + }; + } + }; + + // Flip the degraded server into graceful shutdown, but do NOT await the + // drain: one slow in-flight request must not delay the full server, and + // the shared socket means there is no port gap to cover. The drain + // finishes (and logs) in the background. + db_ready_tx.send(true).ok(); + tokio::spawn(async move { + match degraded_handle.await { + Ok(Ok(())) => {} + Ok(Err(err)) => warn!(err = %err, "degraded HTTP server exited with error"), + Err(err) => warn!(err = %err, "degraded HTTP server task failed"), + } + }); + info!(addr = %config.bind_addr(), "database ready; starting full HTTP server"); + + // Prune peer rows that point back at this node (stale self-loop entries) + if let Some(public_url) = config.public_url.as_deref() { + match db.prune_self_peers(public_url).await { + Ok(0) => {} + Ok(n) => info!(removed = n, public_url, "pruned self-loop peer rows"), + Err(e) => warn!(err = %e, "prune_self_peers failed (non-fatal)"), + } + } + + // Prune peer rows with non-public hosts (loopback/private/internal) that + // were injected via the unauthenticated announce route — they poison the + // sync-notify fan-out (SSRF + crowding out real peers). + match db.prune_non_public_peers().await { + Ok(0) => {} + Ok(n) => info!(removed = n, "pruned non-public (poisoned) peer rows"), + Err(e) => warn!(err = %e, "prune_non_public_peers failed (non-fatal)"), + } + + // Ensure repos directory exists + std::fs::create_dir_all(&config.repos_dir).context("failed to create repos directory")?; + + // The identity was resolved before the database connect; this is only the + // swarm start, which genuinely needs the database handle. + let p2p_handle = match p2p_local_key { + Some(local_key) => { + let bootstrap_addrs = config + .p2p_bootstrap + .iter() + .filter_map(|s| s.parse().ok()) + .collect(); + let shutdown_rx = shutdown_tx.subscribe(); + match p2p::start( + local_key, + config.p2p_port, + bootstrap_addrs, + Arc::clone(&db), + config.auto_sync, + shutdown_rx, + Arc::clone(&keypair), + config.require_signed_peer_writes, + ) + .await + { + Ok(handle) => { + info!(port = config.p2p_port, peer_id = %handle.local_peer_id, "libp2p swarm started"); + Some(Arc::new(handle)) + } + Err(e) => { + tracing::warn!(err = %e, "failed to start libp2p swarm — continuing without p2p"); + None + } + } + } + None => None, + }; + + // Shared no-redirect HTTP client. See build_http_client for the SSRF rationale. + let http_client = Arc::new(build_http_client()?); + + let (ref_update_tx, _) = tokio::sync::broadcast::channel::(256); + let (task_event_tx, _) = tokio::sync::broadcast::channel::(256); + + // Node-keyed, so continuation tokens this node mints are only accepted by + // this node and survive its restarts without any configured secret. + let task_cursor_key = api::task_cursor::TaskCursorKey::derive(&keypair.to_seed()); + + let graphql_schema = Arc::new(graphql::build_schema( + Arc::clone(&db), + ref_update_tx.clone(), + task_event_tx.clone(), + task_cursor_key.clone(), + )); + + let machine_id = std::env::var("FLY_MACHINE_ID").ok(); + if let Some(ref mid) = machine_id { + info!(" fly machine: {mid}"); + } + + // Initialize Tigris S3 client if bucket is configured + let tigris = if !config.tigris_bucket.is_empty() { + match git::tigris::TigrisClient::new(&config.tigris_bucket).await { + Ok(client) => { + info!(bucket = %config.tigris_bucket, "tigris storage enabled"); + Some(client) + } + Err(e) => { + tracing::warn!(err = %e, "failed to initialize Tigris client — using local-only storage"); + None + } + } + } else { + info!("tigris storage disabled (no bucket configured)"); + None + }; + + // Advisory-lock connections come from their own pool: a write guard pins one + // for its whole lifetime, and sharing the application pool would let a push + // burst starve ordinary request handlers. + let lock_pool = db::Db::lock_pool( + &config.database_url, + config.db_lock_pool_max_connections, + std::time::Duration::from_secs(config.db_acquire_timeout_secs), + )?; + let repo_store = git::repo_store::RepoStore::new( + config.repos_dir.clone(), + tigris, + lock_pool, + std::time::Duration::from_secs(config.lock_held_transfer_timeout_secs), + ); + + // Per-DID limiter for the creation endpoints. Keyed on the authenticated + // DID (attacker-varied), so bound its key set to cap memory. Max and window + // are operator-configurable via GITLAWB_RATE_LIMIT_MAX / + // GITLAWB_RATE_LIMIT_WINDOW_SECS; both reject 0 at parse time. + let rate_limiter = rate_limit::RateLimiter::new_bounded( + config.rate_limit_max_requests as usize, + std::time::Duration::from_secs(config.rate_limit_window_secs), + 200_000, + ); + + // Per-client-IP flood brake for the creation endpoints. The per-DID limiter + // above is bypassed by a DID farm (one throwaway did:key per repo), which is + // exactly how the recurring spam-repo floods get past both it and the + // iCaptcha gate. Keyed on the resolved client IP so a single-source flood is + // capped regardless of how many identities it mints. Sized well above any + // legitimate per-IP creation rate; GITLAWB_CREATE_RATE_LIMIT overrides, 0 + // disables. Bounded key set — the key is a client-influenced IP. + let create_limit = std::env::var("GITLAWB_CREATE_RATE_LIMIT") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .unwrap_or(120); + let create_ip_rate_limiter = rate_limit::RateLimiter::new_bounded( + create_limit, + std::time::Duration::from_secs(3600), + 200_000, + ); + if create_limit == 0 { + tracing::warn!("GITLAWB_CREATE_RATE_LIMIT=0 — per-IP creation rate limiting disabled"); + } + + // Push-path flood brake: max git-receive-pack requests per client IP per + // hour (counts both the info/refs advertisement and the push POST). Sized + // for heavy agent automation while still stopping flood traffic (the June + // 2026 attack pushed several times per second per IP). GITLAWB_PUSH_RATE_LIMIT + // overrides; 0 disables. Bounded key set — the key is a client-influenced IP. + let push_limit = std::env::var("GITLAWB_PUSH_RATE_LIMIT") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .unwrap_or(600); + let push_rate_limiter = rate_limit::RateLimiter::new_bounded( + push_limit, + std::time::Duration::from_secs(3600), + 200_000, + ); + if push_limit == 0 { + tracing::warn!("GITLAWB_PUSH_RATE_LIMIT=0 — per-IP push rate limiting disabled"); + } + + // close_issue drives a full archive download for non-owner author pre-checks. + // Keep it on its own bucket so it cannot drain the receive-pack limit. + let close_issue_limit = std::env::var("GITLAWB_CLOSE_ISSUE_RATE_LIMIT") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .unwrap_or(120); + let close_issue_rate_limiter = rate_limit::RateLimiter::new_bounded( + close_issue_limit, + std::time::Duration::from_secs(3600), + 200_000, + ); + if close_issue_limit == 0 { + tracing::warn!( + "GITLAWB_CLOSE_ISSUE_RATE_LIMIT=0 — per-IP close_issue rate limiting disabled" + ); + } + + // Which forwarded header the edge is trusted to set. Default None (trust + // nothing, key on the socket peer). Fly nodes set GITLAWB_TRUSTED_PROXY=fly; + // a node behind Caddy/NGINX sets it to x-forwarded-for. + let push_limiter_trust = rate_limit::TrustedProxy::from_env_value( + &std::env::var("GITLAWB_TRUSTED_PROXY").unwrap_or_default(), + ); + tracing::info!(trust = ?push_limiter_trust, push_limit, "push rate limiter configured"); + + // Peer-sync flood brakes, keyed on the resolved client IP (per-DID is useless + // here — a did:key farm self-registers). Two buckets so an unsigned notify + // flood can't drain the signed trigger caller's quota (#82). Bounded key sets + // (the key is a client-influenced IP); 0 disables each. + let sync_trigger_rate_limiter = rate_limit::RateLimiter::new_bounded( + config.sync_trigger_rate_limit, + std::time::Duration::from_secs(3600), + 200_000, + ); + let peer_write_rate_limiter = rate_limit::RateLimiter::new_bounded( + config.peer_write_rate_limit, + std::time::Duration::from_secs(3600), + 200_000, + ); + let federated_rate_limiter = + rate_limit::RateLimiter::new_bounded(12, std::time::Duration::from_secs(60), 10_000); + if config.sync_trigger_rate_limit == 0 { + tracing::warn!( + "GITLAWB_SYNC_TRIGGER_RATE_LIMIT=0 — /sync/trigger IP rate limiting disabled" + ); + } + if config.peer_write_rate_limit == 0 { + tracing::warn!("GITLAWB_PEER_WRITE_RATE_LIMIT=0 — peer-write IP rate limiting disabled"); + } + + // Initialize the iCaptcha proof gate (inert unless ICAPTCHA_MODE is set). + icaptcha::init().await; + + let state = AppState { + config: Arc::new(config.clone()), + db, + node_did: node_did.clone(), + node_keypair: keypair, + task_cursor_key, + p2p: p2p_handle, + http_client, + ref_update_tx, + task_event_tx, + graphql_schema, + machine_id, + repo_store, + rate_limiter, + create_ip_rate_limiter, + push_rate_limiter, + close_issue_rate_limiter, + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + ipfs_max_legacy_probes: AppState::ipfs_legacy_probe_budget(&config), + ipfs_legacy_scan_page_rows: crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, + ipfs_max_legacy_scan_rows: AppState::ipfs_legacy_scan_row_budget(&config), + ipfs_max_legacy_scan_rule_bytes: crate::api::ipfs::MAX_LEGACY_SCAN_RULE_BYTES_PER_REQUEST, + ipfs_scan_token_key: Arc::new(scan_token_key), + webhook_secret_key: Arc::new(webhook_secret_key), + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, + push_limiter_trust, + sync_trigger_rate_limiter, + peer_write_rate_limiter, + federated_rate_limiter, + shutdown_tx: shutdown_tx.clone(), + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), + // Anon receive-pack advertisements get their OWN pool, same size as the + // write pool but disjoint, so filling it (which takes many source IPs, each + // capped by git_push_advert_per_caller) never occupies a permit the + // authenticated POST needs (#174). + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), + // Bounds concurrent detached post-push encryption walks, sized from the push + // pool (no separate knob — Q1): completed pushes cannot outnumber active + // encryption walks past this (#174 P1-e). + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), + // Bounds how many post-push pin loops run concurrently across all repos (#174 F6), + // independent of the per-repo encrypt-task coalescing below. Not a bound on the + // MB-scale object-id lists themselves: parked tasks still hold theirs (see the + // field doc on AppState::pin_semaphore). + pin_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_pin_tasks)), + // Coalesces the DETACHED post-push encryption tasks per repo so a rapid pusher + // cannot grow the outstanding parked-waiter set past one task per repo (#174 + // P2-2). No knob: it is a natural cap (one entry per distinct repo), not a + // sized pool. + encrypt_inflight: crate::state::EncryptInflight::new(), + // Per-repo in-process write-lease serializer (#174 U2/F3): supplements the pg + // advisory lock so a disconnected push's still-reaping git group can't be raced + // by a second same-node push. The map is naturally capped (one entry per contended + // repo, freed when unreferenced); the sized knob is how many pushes may PARK on + // one repo, since each parked push holds a fully buffered pack. + repo_write_leases: crate::state::RepoWriteLeases::new(config.repo_lease_max_waiters), + git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + config.max_concurrent_reads_per_caller, + ), + // Per-source cap on the receive-pack advertisement, sized to an eighth of the + // write pool (min 1): one resolved client key (rate_limit::client_key) can hold + // at most this many slots in the DEDICATED advert pool (git_push_advert_semaphore, + // disjoint from the write pool), so saturating that pool takes ~8 distinct keys + // (#174). That bounds an IPv4 or single-address caller; a caller controlling many + // addresses (an IPv6 /64 is 2^64 keys) still gets one cap per address, since + // client_key uses the full IP with no prefix folding. Narrowing the keying is a + // deferred design call, not something these caps claim to solve. Sized off the + // write pool only because the advert pool is created at the same size; an advert + // flood cannot touch a write permit. + git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + rate_limit::per_source_push_cap(config.max_concurrent_git_pushes), + ), + // Per-source cap on the authenticated receive-pack POST, sized like the advert + // cap: one resolved client key can hold at most this many write-pool slots, so + // monopolizing the pool takes ~8 distinct keys (#174 P1-d). Same residual as + // above: keys are full IPs, so a caller with many addresses has many caps. + git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + rate_limit::per_source_push_cap(config.max_concurrent_git_pushes), + ), + // Bounds concurrent /ipfs visibility walks — a distinct public cost center, so + // its own pool + per-source sub-cap + per-IP rate limiter, never a git pool + // (#174 P1-3). The per-source map is bounded (reject-before-insert, INV-15). + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_ipfs_walks, + )), + git_ipfs_walk_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + config.ipfs_walk_per_source, + ), + ipfs_rate_limiter: rate_limit::RateLimiter::new_bounded( + config.ipfs_rate_limit, + std::time::Duration::from_secs(3600), + 200_000, + ), + ipfs_work_rate_limiter: rate_limit::RateLimiter::new_bounded( + AppState::ipfs_work_budget(&config), + std::time::Duration::from_secs(3600), + 200_000, + ), + task_read_rate_limiter: rate_limit::RateLimiter::new_bounded( + config.task_read_rate_limit, + std::time::Duration::from_secs(3600), + 200_000, + ), + git_bin: "git".to_string(), + }; + + // One-time re-encryption of any webhook HMAC secrets still resting as + // plaintext (written before at-rest encryption shipped). A SQL migration + // cannot do it — Postgres never sees the node-held key — so it runs here. + match state + .db + .reencrypt_legacy_webhook_secrets(state.webhook_secret_key.as_ref()) + .await + { + Ok(0) => {} + Ok(n) => info!( + count = n, + "re-encrypted legacy webhook HMAC secrets at rest" + ), + Err(e) => warn!(err = %e, "failed to re-encrypt legacy webhook secrets"), + } + + if config.ipfs_rate_limit == 0 { + tracing::warn!("GITLAWB_IPFS_RATE_LIMIT=0 — per-IP /ipfs rate limiting disabled"); + } + if config.task_read_rate_limit == 0 { + tracing::warn!("GITLAWB_TASK_READ_RATE_LIMIT=0 — per-IP task read rate limiting disabled"); + } + + // Periodic peer-count poll for the metrics gauge. If p2p is disabled + // we still set the gauge to 0 so dashboards don't show "no data". + { + let p2p_for_metrics = state.p2p.clone(); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(15)); + loop { + tokio::select! { + _ = interval.tick() => { + let count = match &p2p_for_metrics { + Some(h) => h.status().await.map(|s| s.connected_peers).unwrap_or(0), + None => 0, + }; + metrics::set_peers_connected(count as i64); + } + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + return; + } + } + } + } + }); + } + + // Periodic cleanup of expired rate limit entries + consumed-proof ledger + { + let sweep_state = state.clone(); + let db = state.db.clone(); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + loop { + tokio::select! { + _ = tokio::time::sleep(std::time::Duration::from_secs(300)) => { + sweep_state.sweep_rate_limiters().await; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0); + if let Err(e) = db.sweep_expired_proofs(now).await { + tracing::warn!(err = %e, "failed to sweep expired iCaptcha proofs"); + } + } + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + break; + } + } + } + } + }); + } + + let _legacy_cid_sweep = spawn_legacy_cid_sweep(&state, &config); + let _queue_lifecycle_sweep = spawn_queue_lifecycle_sweep(&state, &config); + // NOTE: the due-request worker is spawned AFTER the one-shot startup + // reconcile/drain below, not here: its 5s interval would otherwise tick + // during operator/PoS setup and interleave with boot recovery on backlog + // rows (state gates prevent data loss, but the interleave costs recovery + // latency and widens the reconcile/drain race window). + let _marker_cleanup_worker = spawn_marker_cleanup_worker(&state); + + let router = server::build_router(state.clone()); + // Re-register the socket bound at startup — same fd, so there was never a + // moment with the port closed between the degraded and full servers. + let listener = TcpListener::from_std(full_std_listener) + .context("failed to re-register HTTP listener with the runtime")?; + + info!("✓ node started — did:{}", node_did); + info!(" repos dir: {}", config.repos_dir.display()); + info!( + " database: PostgreSQL ({})", + &config.database_url.split('@').next_back().unwrap_or("?") + ); + + // Publish our DID record to the Kademlia DHT shortly after startup + if let Some(p2p) = &state.p2p { + let did_record = p2p::DidRecord { + did: node_did.to_string(), + http_url: config.public_url.clone().unwrap_or_default(), + peer_id: p2p.local_peer_id.to_string(), + p2p_port: config.p2p_port, + timestamp: chrono::Utc::now().to_rfc3339(), + }; + let p2p_clone = Arc::clone(p2p); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + // Small delay so Kademlia can find peers first + tokio::select! { + _ = tokio::time::sleep(std::time::Duration::from_secs(5)) => {} + _ = shutdown_rx.changed() => return, + } + p2p_clone.put_did(did_record).await; + info!("DID record published to Kademlia DHT"); + }); + } + + // Spawn background gossip: announce to bootstrap peers, then ping known peers periodically + { + let gossip_state = state.clone(); + let bootstrap_peers = config.bootstrap_peers.clone(); + let shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + gossip_task(gossip_state, bootstrap_peers, shutdown_rx).await; + }); + } + + // Start multi-node sync worker if auto_sync is enabled + if config.auto_sync { + sync::start( + Arc::clone(&state.db), + Arc::clone(&state.config), + Arc::clone(&state.node_keypair), + state.subscribe_shutdown(), + ); + info!("auto-sync worker started"); + } + + // External-forge mirroring is a separate, default-off worker. Startup + // validation already requires owner-only pushes when it is enabled; the + // worker additionally verifies the Git runtime can pin DNS answers before + // any URL reaches `git fetch`. + if config.upstream_mirror_enabled { + upstream_mirror::start( + Arc::clone(&state.db), + Arc::clone(&state.config), + state.repo_store.clone(), + state.repo_write_leases.clone(), + Arc::clone(&state.git_write_semaphore), + state.subscribe_shutdown(), + ) + .context("starting upstream mirror worker")?; + } + + // Periodic reconciliation sweep: re-derives pin/seal sets and fills gaps + // so a dropped replication job never means data loss. + { + let db = state.db.clone(); + let config = Arc::clone(&state.config); + let http_client = Arc::clone(&state.http_client); + let node_keypair = Arc::clone(&state.node_keypair); + let node_did = state.node_did.clone(); + let pin_sem = Arc::clone(&state.pin_semaphore); + let shutdown_rx = state.subscribe_shutdown(); + if reconciliation::spawn( + db, + config, + http_client, + node_keypair, + node_did, + pin_sem, + shutdown_rx, + Some(state.repo_store.clone()), + ) { + info!("reconciliation sweep worker started"); + } + } + + // On-chain operator setup: verify stake + spawn heartbeat loop + if !state.config.contract_node_staking.is_empty() + && !state.config.operator_private_key.is_empty() + { + match build_operator_client(&state.config, &state.node_did.to_string()) { + Ok(client) => match operator::startup_check(&client).await { + Ok(_) => { + let arc_client = Arc::new(client); + arc_client.spawn_heartbeat_loop(state.subscribe_shutdown()); + } + Err(e) => { + if state.config.operator_strict_mode { + return Err(e.context("strict-mode operator check failed")); + } + tracing::warn!(err = %e, "operator startup check failed — continuing without heartbeat loop"); + } + }, + Err(e) => { + if state.config.operator_strict_mode { + return Err(e.context("strict-mode: failed to build operator client")); + } + tracing::warn!(err = %e, "operator client could not be built — continuing without PoS"); + } + } + } else { + info!("on-chain PoS disabled (GITLAWB_CONTRACT_NODE_STAKING or GITLAWB_OPERATOR_PRIVATE_KEY unset)"); + } + + // axum's `with_graceful_shutdown` waits for in-flight requests to + // complete (up to the configured grace) once the future resolves. + let shutdown_signal_for_axum = state.subscribe_shutdown(); + let grace = std::time::Duration::from_secs(config.shutdown_grace_secs); + info!(grace_secs = config.shutdown_grace_secs, "axum server ready"); + + // #26 Split PR 1: drain any `applied` rows left by a previous + // process that crashed after Git applied a ref but before the + // bookkeeping landed. Runs once, BEFORE the server accepts new + // pushes, so a recovery re-derivation does not race a fresh push. + // Non-fatal: a transient drain failure is logged and the rows + // remain `applied` for the next startup to pick up. + // + // P1-A: the reconcile step runs FIRST and promotes any `prepared` + // or `uncertain` row whose target SHA actually landed on disk. + // This is the path that recovers a ref when the post-receive + // `mark_pending_ref_transitions_applied` call errored or was + // interrupted after `receive_pack` returned Ok, or when + // receive-pack returned Err but some refs may have landed + // (the `uncertain` state). Without this step, the drain (gated + // on `state = 'applied'`) would never see those rows. + // + // P2 (reviewer-1/2 round 3): use the multi-pass reconcile so + // prepared/uncertain rows beyond the first 1000-row page are + // processed in the same startup, rather than waiting for the + // next restart (where they might age out of MAX_RECONCILE_AGE). + match durable_outbox::reconcile_prepared_from_disk_all( + state.clone(), + durable_outbox::DRAIN_PER_PASS_LIMIT, + durable_outbox::DRAIN_MAX_PASSES, + ) + .await + { + Ok(0) => {} + Ok(n) => info!( + n, + "reconciled prepared/uncertain -> applied via on-disk ref match" + ), + Err(e) => warn!( + err = %e, + "pending ref transition reconcile failed at startup (non-fatal; will retry on next start)" + ), + } + match durable_outbox::drain_receive_pack_requests_all( + state.clone(), + durable_outbox::DRAIN_PER_PASS_LIMIT, + durable_outbox::DRAIN_MAX_PASSES, + ) + .await + { + Ok(0) => {} + Ok(n) => info!(n, "drained pending ref transitions from prior run"), + Err(e) => warn!( + err = %e, + "pending ref transition drain failed at startup (non-fatal; will retry on next start)" + ), + } + + // Start the due-request worker only now that boot recovery is done (see + // the note at the other spawns above): its first tick must not interleave + // with the one-shot reconcile/drain on backlog rows. + let _due_request_worker = spawn_due_request_worker(&state); + + // `into_make_service_with_connect_info` exposes the socket peer address as + // `ConnectInfo` so the push limiter can key on the real client + // when no trusted proxy header applies (see `rate_limit::client_key`). + let serve_result = axum::serve( + listener, + router.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(async move { + let mut rx = shutdown_signal_for_axum; + // Wait until the watcher flips to true, then return so axum + // can begin draining. + while !*rx.borrow_and_update() { + if rx.changed().await.is_err() { + // Sender dropped — treat as shutdown. + break; + } + } + }) + .await; + + // Server has stopped accepting new connections and drained in-flight + // requests. Tear the rest of the system down. + info!("HTTP server stopped, beginning process shutdown"); + if let Some(h) = metrics_handle { + h.abort(); + } + let _ = grace; // recorded for operators in the log above; not enforced + serve_result?; + info!("clean exit"); + Ok(()) +} + +/// U4 (#173): spawn the periodic legacy provider-CID repair sweep. Releases before this +/// version stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`, +/// and this version's `/ipfs/{cid}` resolver withholds any row whose stored key is not +/// the raw-content CID. The opportunistic repair on the pin path only fires when a push +/// re-carries the object, which normal git negotiation makes it not do, so those rows +/// need a walk. DETACHED, never on the boot path: the caller keeps serving while this +/// runs, and the sweep's own batch bound plus inter-batch delay keep it off the DB's +/// critical path. Its cursor is durable, so a restart mid-walk resumes instead of +/// rewinding. It re-arms after every run, including a failed one, so it never returns +/// and there is no awaited value to log here; the shutdown watcher below is what ends +/// it. +/// +/// A named function rather than an inline block in `main` so the WIRING has a seam a +/// test can call: that the task is spawned at all, that it reads its batch and delay +/// from the config knobs rather than some other field, that the caller is not blocked +/// on it, and that the shutdown watcher actually ends it mid-walk. The sweep's own +/// behavior is covered elsewhere; this is the boot-path half. +fn spawn_legacy_cid_sweep(state: &AppState, config: &Config) -> tokio::task::JoinHandle<()> { + let db = state.db.clone(); + let repos_dir = config.repos_dir.clone(); + let git_bin = state.git_bin.clone(); + let git_timeout = std::time::Duration::from_secs(config.git_service_timeout_secs); + let batch = config.pin_repair_sweep_batch; + let delay = std::time::Duration::from_secs(config.pin_repair_sweep_delay_secs); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + tokio::select! { + _ = ipfs_pin::run_sweep_rearmed( + &repos_dir, &git_bin, git_timeout, batch, delay, + ipfs_pin::SWEEP_REARM_DELAY, &db, + ) => {} + // Shutdown mid-walk simply drops the run; the persisted cursor means the + // next boot picks up where this one stopped. + _ = shutdown_rx.changed() => {} + } + }) +} + +/// #26 Split PR 1 step 4 — periodic queue-lifecycle purge. Runs on +/// the same detached task pattern as `spawn_legacy_cid_sweep`: +/// tokio::spawn with a shutdown watcher, never on the boot path. The +/// interval is fixed at 24 hours (the spec calls for "one per +/// cluster per day"); the inter-batch delay is implicit in the +/// batch size plus the wall-clock cost of each pass. The drain +/// (`drain_receive_pack_requests_all`) and the purge +/// (`purge_request_queue`) share the same `DRAIN_PER_PASS_LIMIT` +/// budget so a 1000-row purge pass takes roughly the same time as +/// a 1000-row drain pass. +fn spawn_queue_lifecycle_sweep(state: &AppState, config: &Config) -> tokio::task::JoinHandle<()> { + let app_state = state.clone(); + let retention_days = config.queue_retention_days; + let batch = config.queue_purge_batch; + let git_timeout = std::time::Duration::from_secs(config.git_service_timeout_secs.min(30)); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(24 * 3600)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + // Consume the immediate first tick: Tokio intervals fire + // immediately on creation, which would run the purge + // concurrently with startup reconcile/drain. The child purge + // is terminal-parent-gated in SQL, but delaying the first + // sweep a full period removes the race entirely. + interval.tick().await; + loop { + tokio::select! { + _ = interval.tick() => { + if let Err(e) = durable_outbox::purge_request_queue( + &app_state.db, + retention_days, + batch, + ).await { + tracing::warn!(err = %e, "queue lifecycle purge failed; will retry on next tick"); + } + // Drain marker tombstones with the bounded deleter; + // failures retain the tombstone for the next tick. + if let Err(e) = durable_outbox::drain_marker_cleanup_queue( + &app_state, + batch, + git_timeout, + ).await { + tracing::warn!(err = %e, "marker cleanup drain failed"); + } + } + _ = shutdown_rx.changed() => { + break; + } + } + } + }) +} + +/// Shutdown-aware background due-request loop. The startup drain runs +/// once; without this, `effects_pending` rows with a future +/// `next_attempt_at` would wait until the next process restart. +/// Polls the indexed due query with bounded batches and failure +/// isolation (one request's error never aborts the batch). +fn spawn_due_request_worker(state: &AppState) -> tokio::task::JoinHandle<()> { + let app_state = state.clone(); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(5)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + _ = interval.tick() => { + match durable_outbox::drain_receive_pack_requests( + app_state.clone(), + durable_outbox::DRAIN_PER_PASS_LIMIT, + ) + .await + { + Ok((0, _)) => {} + Ok((n, _)) => tracing::info!(n, "due request worker drained"), + Err(e) => tracing::warn!(err = %e, "due request worker drain failed"), + } + } + _ = shutdown_rx.changed() => break, + } + } + }) +} + +/// Dedicated marker-tombstone worker at a shorter cadence than the +/// daily purge, so external Git refs do not linger a full period +/// after their SQL owner is gone. +fn spawn_marker_cleanup_worker(state: &AppState) -> tokio::task::JoinHandle<()> { + let app_state = state.clone(); + let git_timeout = + std::time::Duration::from_secs(app_state.config.git_service_timeout_secs.min(30)); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(60)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tokio::select! { + _ = interval.tick() => { + if let Err(e) = durable_outbox::drain_marker_cleanup_queue( + &app_state, + durable_outbox::DRAIN_PER_PASS_LIMIT, + git_timeout, + ) + .await + { + tracing::warn!(err = %e, "marker cleanup worker failed"); + } + } + _ = shutdown_rx.changed() => break, + } + } + }) +} + +fn spawn_shutdown_signal(tx: watch::Sender) { + tokio::spawn(async move { + #[cfg(unix)] + { + use tokio::signal::unix::{signal as unix_signal, SignalKind}; + let mut sigterm = + unix_signal(SignalKind::terminate()).expect("install SIGTERM handler"); + let mut sigint = unix_signal(SignalKind::interrupt()).expect("install SIGINT handler"); + tokio::select! { + _ = sigterm.recv() => info!("SIGTERM received, shutting down"), + _ = sigint.recv() => info!("SIGINT received, shutting down"), + } + } + #[cfg(not(unix))] + { + use tokio::signal; + let _ = signal::ctrl_c().await; + info!("Ctrl-C received, shutting down"); + } + tx.send(true).ok(); + }); +} + +async fn connect_db_with_retry( + config: &Config, + db_startup: Arc, + mut shutdown_rx: watch::Receiver, +) -> Option> { + let initial_retry_secs = config.db_retry_initial_secs; + let max_retry_secs = config.db_retry_max_secs.max(initial_retry_secs); + let acquire_timeout = std::time::Duration::from_secs(config.db_acquire_timeout_secs); + let attempt_timeout = std::time::Duration::from_secs(config.db_connect_timeout_secs); + let mut attempts = 0_u64; + + loop { + if *shutdown_rx.borrow() { + return None; + } + + attempts = attempts.saturating_add(1); + db_startup.attempts.store(attempts, Ordering::Relaxed); + + // Bound the whole attempt, not just the pool connect: migrations + // block on a cross-instance advisory lock, and an unbounded wait + // there would wedge this loop — no retries, no logs, no recovery. + // Timing out and retrying is safe; migrations are idempotent. + let attempt = match tokio::time::timeout( + attempt_timeout, + Db::connect( + &config.database_url, + config.db_max_connections, + acquire_timeout, + ), + ) + .await + { + Ok(result) => result, + Err(_) => Err(anyhow!( + "connect + migrate attempt exceeded {}s (GITLAWB_DB_CONNECT_TIMEOUT_SECS); \ + is another instance holding the migration lock?", + attempt_timeout.as_secs() + )), + }; + + match attempt { + Ok(db) => { + info!(attempts, "database connection established"); + return Some(Arc::new(db)); + } + Err(err) => { + // A bad DATABASE_URL or rejected credentials won't heal on + // their own. Still retry (exiting would crash-loop and hammer + // the provider — and take liveness down with it), but log at + // error level and skip straight to the maximum backoff; the + // /ready health check is what surfaces this to deploys. + let permanent = is_likely_permanent_db_error(&err); + let retry_secs = if permanent { + max_retry_secs + } else { + database_retry_delay_secs(initial_retry_secs, max_retry_secs, attempts) + }; + db_startup + .next_retry_secs + .store(retry_secs, Ordering::Relaxed); + if permanent { + tracing::error!( + attempts, + retry_secs, + err = %err, + "database rejected our configuration (bad DATABASE_URL or credentials?) — retrying, but operator action is likely required" + ); + } else { + warn!( + attempts, + retry_secs, + err = %err, + "database unavailable during startup; retrying" + ); + } + + tokio::select! { + _ = tokio::time::sleep(std::time::Duration::from_secs(retry_secs)) => {} + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return None; + } + } + } + } + } + } +} + +/// Errors that indicate misconfiguration rather than a transient outage: a +/// malformed DATABASE_URL, or a server that answered and rejected us — +/// Postgres error class 28xxx (invalid authorization) or 3D000 (database +/// does not exist). Best-effort: an error that anyhow can't downcast back to +/// sqlx just counts as transient. +fn is_likely_permanent_db_error(err: &anyhow::Error) -> bool { + match err.downcast_ref::() { + Some(sqlx::Error::Configuration(_)) => true, + Some(sqlx::Error::Database(db)) => db + .code() + .map(|c| c.starts_with("28") || c.starts_with("3D")) + .unwrap_or(false), + _ => false, + } +} + +fn database_retry_delay_secs(initial_secs: u64, max_secs: u64, attempts: u64) -> u64 { + // The exponent bound only keeps the u32 cast safe — max_secs is the real + // (operator-configurable) cap, and saturating math handles overflow. + let exponent = attempts.saturating_sub(1).min(63) as u32; + initial_secs + .saturating_mul(2_u64.saturating_pow(exponent)) + .min(max_secs) +} + +async fn run_degraded_server( + listener: TcpListener, + node_did: String, + db_startup: Arc, + web_url: Option, + mut db_ready_rx: watch::Receiver, + mut shutdown_rx: watch::Receiver, +) -> Result<()> { + let addr = listener.local_addr().ok(); + let router = build_degraded_router(node_did, db_startup, web_url); + info!(?addr, "degraded HTTP server ready"); + + axum::serve(listener, router) + .with_graceful_shutdown(async move { + // wait_for resolves on predicate-true or sender-drop; either way + // this phase is over. + tokio::select! { + _ = db_ready_rx.wait_for(|ready| *ready) => {} + _ = shutdown_rx.wait_for(|stop| *stop) => {} + } + }) + .await?; + + Ok(()) +} + +fn build_degraded_router( + node_did: String, + db_startup: Arc, + web_url: Option, +) -> Router { + let state = DegradedState { + node_did, + db_startup, + web_url, + }; + // Everything answers 503 with the same body — including /health and + // /ready, so peer readiness probes and uptime monitors correctly see a + // node that cannot serve traffic. + // `/` additionally carries the node identity for probing peers. + Router::new() + .route("/", get(degraded_node_info)) + .fallback(degraded_unavailable) + .with_state(state) +} + +/// One source of truth for the degraded 503 body, sharing the error +/// vocabulary with error.rs so clients see the same code/message for +/// "database unavailable" regardless of which phase produced it. +fn degraded_body(db_startup: &DbStartupStatus) -> serde_json::Value { + serde_json::json!({ + "status": "degraded", + "database": "initializing", + "error": error::DB_UNAVAILABLE_CODE, + "message": error::DB_UNAVAILABLE_MESSAGE, + "db_attempts": db_startup.attempts.load(Ordering::Relaxed), + "db_next_retry_secs": db_startup.next_retry_secs.load(Ordering::Relaxed), + }) +} + +async fn degraded_node_info(State(state): State) -> impl IntoResponse { + let mut body = degraded_body(&state.db_startup); + if let Some(obj) = body.as_object_mut() { + obj.insert("name".into(), "gitlawb-node".into()); + obj.insert("version".into(), env!("CARGO_PKG_VERSION").into()); + obj.insert("did".into(), state.node_did.clone().into()); + if let Some(web_url) = &state.web_url { + obj.insert("web_url".into(), web_url.clone().into()); + } + } + (StatusCode::SERVICE_UNAVAILABLE, Json(body)) +} + +async fn degraded_unavailable(State(state): State) -> impl IntoResponse { + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(degraded_body(&state.db_startup)), + ) +} + +/// Spawn a small axum router that exposes only `GET /metrics` on its own +/// listener. Returns the JoinHandle so `main()` can abort it on shutdown. +/// This is deliberately separate from the main router so the metrics port +/// can be firewalled differently from the API port — bind to localhost +/// or a private interface only. +async fn spawn_metrics_server( + addr: &str, + mut shutdown_rx: watch::Receiver, +) -> Result> { + use axum::{response::IntoResponse, routing::get, Router}; + + async fn metrics_handler() -> impl IntoResponse { + match metrics::encode() { + Ok(body) => ( + axum::http::StatusCode::OK, + [( + axum::http::header::CONTENT_TYPE, + "text/plain; version=0.0.4; charset=utf-8", + )], + body, + ), + Err(e) => ( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + [( + axum::http::header::CONTENT_TYPE, + "text/plain; charset=utf-8", + )], + format!("metrics encode error: {e}"), + ), + } + } + + let listener = TcpListener::bind(addr) + .await + .with_context(|| format!("failed to bind metrics listener to {addr}"))?; + let app = Router::new().route("/metrics", get(metrics_handler)); + + let handle = tokio::spawn(async move { + if let Err(e) = axum::serve(listener, app) + .with_graceful_shutdown(async move { + while !*shutdown_rx.borrow_and_update() { + if shutdown_rx.changed().await.is_err() { + break; + } + } + }) + .await + { + warn!(err = %e, "metrics server exited with error"); + } + }); + Ok(handle) +} + +fn build_operator_client( + config: &config::Config, + node_did: &str, +) -> Result { + use alloy::primitives::Address; + use std::str::FromStr; + + let contract_address = Address::from_str(&config.contract_node_staking) + .with_context(|| format!("invalid contract address: {}", config.contract_node_staking))?; + + let cfg = operator::OperatorConfig { + rpc_url: config.chain_rpc_url.clone(), + private_key: config.operator_private_key.clone(), + contract_address, + node_did: node_did.to_string(), + heartbeat_interval: std::time::Duration::from_secs(config.heartbeat_interval_hours * 3600), + strict_mode: config.operator_strict_mode, + }; + Ok(operator::OperatorClient::new(cfg)) +} + +/// Announce to bootstrap peers on startup, then periodically ping all known peers. +async fn gossip_task( + state: AppState, + bootstrap_peers: Vec, + mut shutdown_rx: tokio::sync::watch::Receiver, +) { + // If shutdown arrives during the initial delay, exit before announcing. + tokio::select! { + _ = tokio::time::sleep(std::time::Duration::from_secs(2)) => {} + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + info!("gossip: shutdown during startup delay, exiting"); + return; + } + } + } + + // Reuse the shared no-redirect client for every gossip outbound call (the + // bootstrap announce POST and the periodic peer /ready ping). Peer URLs are + // attacker-influenceable, so a 3xx to a private address must not be followed. + // Do NOT fall back to reqwest::Client::new(): its default follows redirects + // and would reintroduce the SSRF closed here (#93). + let client = state.http_client.clone(); + let my_did = state.node_did.to_string(); + let my_url = state.config.public_url.clone().unwrap_or_default(); + + // Announce ourselves to each bootstrap peer + for peer_url in &bootstrap_peers { + // Cooperative shutdown between peers — a slow peer shouldn't + // block the node exiting. + if *shutdown_rx.borrow() { + info!("gossip: shutdown signalled during peer announce, exiting"); + return; + } + let path = "/api/v1/peers/announce"; + let announce_url = format!("{}{}", peer_url.trim_end_matches('/'), path); + let body = serde_json::json!({ + "did": my_did.clone(), + "http_url": my_url.clone(), + }); + let body_bytes = match serde_json::to_vec(&body) { + Ok(bytes) => bytes, + Err(e) => { + tracing::warn!(err = %e, "failed to serialize peer announce body"); + continue; + } + }; + let signed = sign_request(state.node_keypair.as_ref(), "POST", path, &body_bytes); + // Per-request timeout inside the loop; do not let one hung peer + // block others. The request itself is a normal tokio future so + // it's cancel-safe on shutdown. + match tokio::time::timeout( + std::time::Duration::from_secs(5), + client + .post(&announce_url) + .header("Content-Type", "application/json") + .header("Content-Digest", signed.content_digest) + .header("Signature-Input", signed.signature_input) + .header("Signature", signed.signature) + .body(body_bytes) + .send(), + ) + .await + { + Ok(Ok(resp)) => { + if resp.status().is_success() { + if let Ok(json) = resp.json::().await { + // Add them back to our peer list + if let (Some(their_did), Some(their_url)) = ( + json.get("node_did").and_then(|v| v.as_str()), + json.get("node_url").and_then(|v| v.as_str()), + ) { + if !their_url.is_empty() { + // Unproven, unconditionally: their_did and + // their_url come straight out of the contacted + // peer's JSON response body, with no signature + // and no proof the claimed DID belongs to the + // peer we reached. This path therefore seeds + // new peers and can never repoint an existing + // row. The upsert's result is matched rather + // than discarded, because "bootstrap peer + // added" printed over a refused write is a + // denial rendering as success. Non-fatal to the + // loop either way. + match state + .db + .upsert_peer( + their_did, + their_url, + db::PeerWriteAuthority::Unproven, + ) + .await + { + Ok(()) => { + tracing::info!(did = %their_did, url = %their_url, "bootstrap peer added") + } + Err(e) => { + tracing::warn!(did = %their_did, url = %their_url, err = %e, "bootstrap peer announce-back rejected") + } + } + } + } + } + } + } + Ok(Err(e)) => { + tracing::warn!(url = %announce_url, err = %e, "failed to announce to bootstrap peer") + } + Err(_) => tracing::warn!(url = %announce_url, "bootstrap peer announce timed out (5s)"), + } + } + + // Periodic ping every 5 minutes — exit on shutdown. + let mut interval = tokio::time::interval(std::time::Duration::from_secs(300)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut failed_once: HashSet = HashSet::new(); + loop { + tokio::select! { + _ = interval.tick() => { + let peers = match state.db.list_peers().await { + Ok(p) => p, + Err(_) => continue, + }; + gossip_ping_round( + state.db.as_ref(), + client.as_ref(), + &mut failed_once, + peers, + ) + .await; + } + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + info!("gossip task: shutdown signal received, exiting"); + return; + } + } + } + } +} + +#[cfg(test)] +mod rate_limiter_sweep_tests { + use crate::rate_limit::RateLimiter; + use std::time::Duration; + + // Every per-key limiter the router mounts must be swept by the periodic + // task, the `/ipfs` one included: a limiter left out keeps expired keys + // until its map fills and the inline capacity sweep fires. Fails on the + // pre-fix sweeper, which skipped `ipfs_rate_limiter`. + #[tokio::test] + async fn sweep_evicts_expired_keys_from_every_limiter() { + let window = Duration::from_millis(30); + let mut state = crate::test_support::test_state_lazy(); + state.rate_limiter = RateLimiter::new(10, window); + state.create_ip_rate_limiter = RateLimiter::new(10, window); + state.push_rate_limiter = RateLimiter::new(10, window); + state.close_issue_rate_limiter = RateLimiter::new(10, window); + state.sync_trigger_rate_limiter = RateLimiter::new(10, window); + state.peer_write_rate_limiter = RateLimiter::new(10, window); + state.federated_rate_limiter = RateLimiter::new(10, window); + state.ipfs_rate_limiter = RateLimiter::new(10, window); + state.ipfs_work_rate_limiter = RateLimiter::new(10, window); + state.task_read_rate_limiter = RateLimiter::new(10, window); + + let limiters = |s: &crate::state::AppState| { + [ + s.rate_limiter.clone(), + s.create_ip_rate_limiter.clone(), + s.push_rate_limiter.clone(), + s.close_issue_rate_limiter.clone(), + s.sync_trigger_rate_limiter.clone(), + s.peer_write_rate_limiter.clone(), + s.federated_rate_limiter.clone(), + s.ipfs_rate_limiter.clone(), + s.ipfs_work_rate_limiter.clone(), + s.task_read_rate_limiter.clone(), + ] + }; + for l in limiters(&state) { + assert!(l.check("1.2.3.4").await); + assert_eq!(l.tracked_keys().await, 1); + } + + tokio::time::sleep(window * 3).await; + state.sweep_rate_limiters().await; + + for (i, l) in limiters(&state).into_iter().enumerate() { + assert_eq!(l.tracked_keys().await, 0, "limiter {i} was not swept"); + } + } +} + +/// it. +fn peer_ping_db_update(failed_once: &mut HashSet, did: &str, ready: bool) -> Option { + if ready { + failed_once.remove(did); + Some(true) + } else if failed_once.insert(did.to_owned()) { + None + } else { + Some(false) + } +} + +/// Build the shared node HTTP client used for every outbound fan-out (sync +/// trigger, profile/repo fetches, gossip announce + peer pings). +/// +/// No redirects: peer URLs are attacker-influenceable, so a `3xx` to a private +/// address must not be followed (SSRF guard, #78/#93). Do NOT replace with +/// `reqwest::Client::new()` — its default follows redirects. Kept as a named +/// builder so tests bind the redirect guarantee to the real client the node +/// runs, not a hand-rolled equivalent. +fn build_http_client() -> reqwest::Result { + reqwest::Client::builder() + .timeout(OUTBOUND_HTTP_TIMEOUT) + .redirect(reqwest::redirect::Policy::none()) + .build() +} + +const OUTBOUND_HTTP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +/// Ping a peer's DB-aware `/ready` endpoint and report whether it answered 2xx. +/// +/// A 404 falls back to `/health` for compatibility with nodes released before +/// `/ready` existed. Other readiness failures, including 503 during a database +/// outage, fail closed and must not use the liveness-only endpoint. The complete +/// `/ready`-then-`/health` probe shares one timeout budget. +/// +/// Takes the client by reference so callers supply the shared, no-redirect +/// `state.http_client`. Peer URLs are attacker-influenceable, so a `3xx` to a +/// private address must not be followed. Do NOT call this with a bare +/// `reqwest::Client::new()`: its default follows redirects and would +/// reintroduce the SSRF this guards against (#93). +pub(crate) async fn ping_peer_readiness(client: &reqwest::Client, http_url: &str) -> bool { + ping_peer_readiness_with_timeout(client, http_url, OUTBOUND_HTTP_TIMEOUT).await +} + +async fn ping_peer_readiness_with_timeout( + client: &reqwest::Client, + http_url: &str, + timeout: std::time::Duration, +) -> bool { + let base_url = http_url.trim_end_matches('/'); + let result = tokio::time::timeout(timeout, async { + let readiness = client.get(format!("{base_url}/ready")).send().await; + + match readiness { + Ok(response) if response.status().is_success() => true, + Ok(response) if response.status() == reqwest::StatusCode::NOT_FOUND => { + info!( + peer_url = %http_url, + "peer has no readiness endpoint; falling back to legacy health probe" + ); + match client.get(format!("{base_url}/health")).send().await { + Ok(response) if response.status().is_success() => true, + Ok(response) => { + warn!( + peer_url = %http_url, + status = %response.status(), + "legacy peer health probe reported unhealthy" + ); + false + } + Err(error) => { + warn!( + peer_url = %http_url, + err = %error, + "legacy peer health probe failed" + ); + false + } + } + } + Ok(response) => { + warn!( + peer_url = %http_url, + status = %response.status(), + "peer readiness probe reported unready" + ); + false + } + Err(error) => { + warn!( + peer_url = %http_url, + err = %error, + "peer readiness probe failed" + ); + false + } + } + }) + .await; + + match result { + Ok(ready) => ready, + Err(_) => { + warn!( + peer_url = %http_url, + timeout_ms = timeout.as_millis(), + "peer readiness probe timed out" + ); + false + } + } +} + +fn load_or_create_keypair(config: &Config) -> Result { + load_or_create_keypair_at(&config.resolved_key_path()) +} + +/// The node identity key's load-or-create, taken by path so the storage +/// contract can be tested without building a whole `Config`. +fn load_or_create_keypair_at(key_path: &std::path::Path) -> Result { + if p2p::path_denotes_a_directory(key_path, None) + || key_path + .components() + .any(|c| matches!(c, std::path::Component::ParentDir)) + { + anyhow::bail!( + "GITLAWB_KEY ({}) must name a key file, not a directory; a trailing separator, \ + a final `.` or `..` component, and `..` traversal are refused so the path \ + cannot retarget a parent", + key_path.display() + ); + } + + #[cfg(unix)] + if let Some(pem) = p2p::load_identity_pem_if_present(key_path)? { + let kp = Keypair::from_pem(&pem).map_err(|e| anyhow::anyhow!("invalid PEM key: {e}"))?; + info!(path = %key_path.display(), "loaded existing identity"); + return Ok(kp); + } + + #[cfg(not(unix))] + if key_path.exists() { + let pem = std::fs::read_to_string(key_path) + .with_context(|| format!("failed to read key from {}", key_path.display()))?; + let kp = Keypair::from_pem(&pem).map_err(|e| anyhow::anyhow!("invalid PEM key: {e}"))?; + info!(path = %key_path.display(), "loaded existing identity"); + return Ok(kp); + } + + let kp = Keypair::generate(); + let pem = kp + .to_pem() + .map_err(|e| anyhow::anyhow!("failed to serialize key: {e}"))?; + + // The directory is created pinned to 0700 and the PEM is published + // through the same scratch-then-link path the p2p key uses, at a + // verified 0600. The previous flow (create_dir_all with no mode, then + // write, then set_permissions) is the sequence INV-23 prohibits: it + // left the directory world-writable under a permissive umask, and + // under a restrictive one it could not be opened at all, which failed + // the whole node here, before the listener binds. + #[cfg(unix)] + p2p::create_pinned_dir_and_publish(key_path, pem.as_bytes())?; + + #[cfg(not(unix))] + { + if let Some(parent) = key_path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(key_path, pem.as_bytes())?; + } + + info!(path = %key_path.display(), did = %kp.did(), "generated new node identity"); + Ok(kp) +} + +async fn gossip_ping_round( + db: &Db, + client: &reqwest::Client, + failed_once: &mut HashSet, + peers: Vec, +) { + { + let current_dids: HashSet<&str> = peers.iter().map(|peer| peer.did.as_str()).collect(); + failed_once.retain(|did| current_dids.contains(did.as_str())); + } + for peer in peers { + let ok = ping_peer_readiness(client, &peer.http_url).await; + match peer_ping_db_update(failed_once, &peer.did, ok) { + Some(reachable) => { + if let Err(error) = db.mark_peer_ping(&peer.did, reachable).await { + warn!(did = %peer.did, err = %error, "failed to persist peer readiness"); + } + } + None => warn!( + did = %peer.did, + "peer readiness probe failed once; preserving previous federation status" + ), + } + } +} + +#[cfg(test)] +mod gossip_ssrf_tests { + use super::{ + gossip_ping_round, peer_ping_db_update, ping_peer_readiness, + ping_peer_readiness_with_timeout, + }; + use axum::{http::StatusCode, routing::get, Router}; + use sqlx::PgPool; + use std::collections::HashSet; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + use std::time::Duration; + + // Build the client exactly as production does (super::build_http_client) so + // these tests bind the redirect guarantee to the real shared client the + // node runs. A regression that makes build_http_client follow redirects + // fails ping_peer_readiness_does_not_follow_redirect. + fn production_http_client() -> reqwest::Client { + super::build_http_client().expect("failed to build production http client") + } + + #[test] + fn peer_ping_requires_two_failures_before_marking_unreachable() { + let mut failed_once = HashSet::new(); + let did = "did:key:z6MkPeer"; + + assert_eq!( + peer_ping_db_update(&mut failed_once, did, false), + None, + "one transient failure must preserve the persisted federation gate" + ); + assert_eq!( + peer_ping_db_update(&mut failed_once, did, false), + Some(false), + "a sustained failure must mark the peer unreachable" + ); + assert_eq!( + peer_ping_db_update(&mut failed_once, did, true), + Some(true), + "a success must restore reachability immediately" + ); + assert_eq!( + peer_ping_db_update(&mut failed_once, did, false), + None, + "a success must reset the consecutive-failure state" + ); + } + + #[sqlx::test] + async fn gossip_ping_round_requires_two_failures_before_persisting_unreachable(pool: PgPool) { + let mut server = mockito::Server::new_async().await; + let ready = server + .mock("GET", "/ready") + .with_status(503) + .expect(2) + .create_async() + .await; + let state = crate::test_support::test_state(pool.clone()).await; + let did = "did:key:z6MkGossipHysteresis"; + let now = chrono::Utc::now().to_rfc3339(); + sqlx::query( + "INSERT INTO peers (did, http_url, last_seen, last_ping_ok, announced_at) + VALUES ($1, $2, $3, TRUE, $3)", + ) + .bind(did) + .bind(server.url()) + .bind(now) + .execute(&pool) + .await + .unwrap(); + + let mut failed_once = HashSet::from(["did:key:z6MkRemovedPeer".to_owned()]); + gossip_ping_round( + state.db.as_ref(), + &production_http_client(), + &mut failed_once, + state.db.list_peers().await.unwrap(), + ) + .await; + assert!( + !failed_once.contains("did:key:z6MkRemovedPeer"), + "failure tracking must prune peers outside the current snapshot" + ); + assert!( + failed_once.contains(did), + "the first failed sample must be retained for the next round" + ); + assert!( + state + .db + .list_peers() + .await + .unwrap() + .into_iter() + .find(|peer| peer.did == did) + .unwrap() + .last_ping_ok, + "one transient failure must preserve the persisted federation gate" + ); + + gossip_ping_round( + state.db.as_ref(), + &production_http_client(), + &mut failed_once, + state.db.list_peers().await.unwrap(), + ) + .await; + assert!( + !state + .db + .list_peers() + .await + .unwrap() + .into_iter() + .find(|peer| peer.did == did) + .unwrap() + .last_ping_ok, + "two consecutive failed rounds must persist the peer as unreachable" + ); + ready.assert_async().await; + } + + // A peer answering `/ready` with a 302 toward an internal address must not + // be followed: the redirect target must never be requested (#93). + #[tokio::test] + async fn ping_peer_readiness_does_not_follow_redirect() { + let mut server = mockito::Server::new_async().await; + let internal = server + .mock("GET", "/internal-metadata") + .with_status(200) + .expect(0) + .create_async() + .await; + let _ready = server + .mock("GET", "/ready") + .with_status(302) + .with_header("location", &format!("{}/internal-metadata", server.url())) + .create_async() + .await; + + let ok = ping_peer_readiness(&production_http_client(), &server.url()).await; + + assert!(!ok, "a 302 must not count as a healthy peer"); + // expect(0) is enforced only at assert time; this fails if the redirect + // was followed to the internal target. + internal.assert_async().await; + } + + #[tokio::test] + async fn ping_peer_readiness_reports_success_on_200() { + let mut server = mockito::Server::new_async().await; + let _ready = server + .mock("GET", "/ready") + .with_status(200) + .create_async() + .await; + + let ok = ping_peer_readiness(&production_http_client(), &server.url()).await; + + assert!(ok, "a 200 /ready must count as a ready peer"); + } + + #[tokio::test] + async fn ping_peer_readiness_ignores_liveness_only_health() { + let mut server = mockito::Server::new_async().await; + let health = server + .mock("GET", "/health") + .with_status(200) + .expect(0) + .create_async() + .await; + let ready = server + .mock("GET", "/ready") + .with_status(503) + .expect(1) + .create_async() + .await; + + let ok = ping_peer_readiness(&production_http_client(), &server.url()).await; + + assert!(!ok, "a peer with an unavailable database must not be ready"); + health.assert_async().await; + ready.assert_async().await; + } + + #[tokio::test] + async fn ping_peer_readiness_falls_back_for_legacy_peer() { + let mut server = mockito::Server::new_async().await; + let ready = server + .mock("GET", "/ready") + .with_status(404) + .expect(1) + .create_async() + .await; + let health = server + .mock("GET", "/health") + .with_status(200) + .expect(1) + .create_async() + .await; + + let ok = ping_peer_readiness(&production_http_client(), &server.url()).await; + + assert!( + ok, + "a legacy peer with a healthy /health must remain reachable" + ); + ready.assert_async().await; + health.assert_async().await; + } + + #[tokio::test] + async fn ping_peer_readiness_legacy_fallback_shares_deadline() { + let health_requests = Arc::new(AtomicUsize::new(0)); + let health_requests_for_route = Arc::clone(&health_requests); + let app = Router::new() + .route( + "/ready", + get(|| async { + tokio::time::sleep(Duration::from_millis(100)).await; + StatusCode::NOT_FOUND + }), + ) + .route( + "/health", + get(move || { + let health_requests = Arc::clone(&health_requests_for_route); + async move { + health_requests.fetch_add(1, Ordering::Relaxed); + tokio::time::sleep(Duration::from_millis(300)).await; + StatusCode::OK + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("failed to bind delayed legacy peer"); + let peer_url = format!( + "http://{}", + listener + .local_addr() + .expect("delayed legacy peer has no local address") + ); + let server = tokio::spawn(async move { + axum::serve(listener, app) + .await + .expect("delayed legacy peer failed"); + }); + + let ok = ping_peer_readiness_with_timeout( + &production_http_client(), + &peer_url, + // Each response arrives within 350 ms, but the two serial requests + // cannot both finish within one 350 ms probe budget. + Duration::from_millis(350), + ) + .await; + + assert!(!ok, "the fallback must not start a fresh timeout budget"); + assert_eq!( + health_requests.load(Ordering::Relaxed), + 1, + "the delayed 404 should reach the legacy fallback" + ); + server.abort(); + } + + #[tokio::test] + async fn ping_peer_readiness_legacy_fallback_does_not_follow_redirect() { + let mut server = mockito::Server::new_async().await; + let internal = server + .mock("GET", "/internal-metadata") + .with_status(200) + .expect(0) + .create_async() + .await; + let _ready = server + .mock("GET", "/ready") + .with_status(404) + .create_async() + .await; + let _health = server + .mock("GET", "/health") + .with_status(302) + .with_header("location", &format!("{}/internal-metadata", server.url())) + .create_async() + .await; + + let ok = ping_peer_readiness(&production_http_client(), &server.url()).await; + + assert!(!ok, "the legacy fallback must not follow redirects"); + internal.assert_async().await; + } + + // A transport error (nothing listening) must map to unhealthy, never a + // spurious healthy — the .unwrap_or(false) arm. + #[tokio::test] + async fn ping_peer_readiness_reports_unready_on_connection_error() { + let ok = ping_peer_readiness(&production_http_client(), "http://127.0.0.1:1").await; + assert!(!ok, "a connection error must count as an unready peer"); + } +} + +#[cfg(test)] +mod identity_key_storage_tests { + use super::*; + + /// Requested modes for the objects this invocation creates. Named once so + /// a failure prints achieved against requested rather than a bare boolean. + #[cfg(unix)] + const IDENTITY_WANT_DIR_MODE: u32 = 0o700; + #[cfg(unix)] + const IDENTITY_WANT_KEY_MODE: u32 = 0o600; + /// Mode a PRE-EXISTING ancestor is built at. Not group-writable, so the + /// write-authority check accepts it, and nothing this process does may + /// change it: only directories this invocation creates are its business. + #[cfg(unix)] + const IDENTITY_PREEXISTING_MODE: u32 = 0o755; + /// Mode the group-writable rows build. A parent carrying it must be + /// refused before anything is created inside it. + #[cfg(unix)] + const IDENTITY_LOOSE_MODE: u32 = 0o775; + + #[cfg(unix)] + fn identity_mode_of(path: &std::path::Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + std::fs::symlink_metadata(path) + .unwrap_or_else(|e| panic!("stat {}: {e}", path.display())) + .permissions() + .mode() + & 0o7777 + } + + /// Every directory on the key path with its achieved mode against what the + /// contract requested, so a RED is attributable to the mode rather than to + /// "something failed". + #[cfg(unix)] + fn identity_key_tree(base: &std::path::Path, key: &std::path::Path) -> String { + let mut dirs = Vec::new(); + let mut cur = key.parent(); + while let Some(d) = cur { + dirs.push(d.to_path_buf()); + if d == base { + break; + } + cur = d.parent(); + } + dirs.reverse(); + let mut out = String::new(); + for d in dirs { + match std::fs::symlink_metadata(&d) { + Ok(_) => out.push_str(&format!( + " dir {} achieved mode {:04o}, requested {:04o}\n", + d.display(), + identity_mode_of(&d), + IDENTITY_WANT_DIR_MODE + )), + Err(e) => out.push_str(&format!(" dir {} absent ({e})\n", d.display())), + } + } + match std::fs::symlink_metadata(key) { + Ok(_) => out.push_str(&format!( + " key {} achieved mode {:04o}, requested {:04o}\n", + key.display(), + identity_mode_of(key), + IDENTITY_WANT_KEY_MODE + )), + Err(e) => out.push_str(&format!(" key {} absent ({e})\n", key.display())), + } + out + } + + /// An object a refused boot must not have left behind. The message is the + /// fixture's own: the anyhow text never carries the phrase, so a `contains` + /// on the error would be a check that can never pass. + #[cfg(unix)] + fn assert_identity_absent(path: &std::path::Path) { + assert!( + std::fs::symlink_metadata(path).is_err(), + "{} left behind by a refused boot", + path.display() + ); + } + + /// A symlink refusal must name the symlink. An unrelated EACCES, ENOENT, + /// or write-authority failure satisfies "an error happened" while proving + /// nothing about whether the interior component was followed, so every + /// symlink row asserts the reason rather than the bare failure. + #[cfg(unix)] + fn assert_names_symlink_refusal(text: &str, what: &str) { + let lower = text.to_lowercase(); + assert!( + lower.contains("symlink") || lower.contains("symbolic link"), + "{what} must be refused for the symlink on the path, and the refusal must say so, \ + got: {text}" + ); + } + + /// Every regular file under `root`, following no symlink out of it. Used + /// to prove a refused boot published nothing where a followed interior + /// link would have put it, including a target outside the base. + #[cfg(unix)] + fn identity_collect_files(root: &std::path::Path, found: &mut Vec) { + let Ok(entries) = std::fs::read_dir(root) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + let Ok(md) = std::fs::symlink_metadata(&path) else { + continue; + }; + if md.is_dir() { + identity_collect_files(&path, found); + } else if md.is_file() { + found.push(path); + } + } + } + + /// No key anywhere beneath `root`. Stronger than naming one expected path, + /// because a followed link can land the key at a name the test did not + /// predict. + #[cfg(unix)] + fn assert_no_identity_key_under(root: &std::path::Path, what: &str) { + let mut found = Vec::new(); + identity_collect_files(root, &mut found); + let keys: Vec = found + .iter() + .filter(|p| p.file_name().is_some_and(|n| n == "identity.pem")) + .map(|p| p.display().to_string()) + .collect(); + assert!( + keys.is_empty(), + "{what}: a refused boot published an identity under {}: {keys:?}", + root.display() + ); + } + + /// The assertions every success row shares: exact 0700 on each directory + /// this invocation created, 0600 on the key, no scratch residue beside it, + /// and a base this process never touched. + #[cfg(unix)] + fn assert_identity_success( + base: &std::path::Path, + key: &std::path::Path, + created: &[std::path::PathBuf], + ) { + for d in created { + let mode = identity_mode_of(d); + assert_eq!( + mode, + IDENTITY_WANT_DIR_MODE, + "created directory {} achieved mode {:04o}, requested {:04o}\n{}", + d.display(), + mode, + IDENTITY_WANT_DIR_MODE, + identity_key_tree(base, key) + ); + } + let key_mode = identity_mode_of(key); + assert_eq!( + key_mode, + IDENTITY_WANT_KEY_MODE, + "identity key achieved mode {key_mode:04o}, requested 0600\n{}", + identity_key_tree(base, key) + ); + let leaf = key.parent().expect("the key names a parent"); + let entries: Vec = std::fs::read_dir(leaf) + .expect("read the key directory") + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + assert_eq!( + entries, + vec![key.file_name().unwrap().to_string_lossy().into_owned()], + "the key directory must hold only the published key, no scratch residue: {entries:?}" + ); + let base_mode = identity_mode_of(base); + assert_eq!( + base_mode, 0o700, + "the base directory must be left at 0700, achieved {base_mode:04o}" + ); + } + + /// Boot a success row and check its contract. `call_path` is what the code + /// is handed (relative for the relative row); `key` is the same key under + /// `base`, so the mode checks never depend on the child's cwd. + #[cfg(unix)] + fn run_identity_success_row( + base: &std::path::Path, + call_path: &std::path::Path, + key: &std::path::Path, + created: &[std::path::PathBuf], + phase: &str, + ) -> String { + if phase == "reload" { + assert!( + key.exists(), + "the reload phase requires the create phase to have published a key at {}", + key.display() + ); + } + let kp = load_or_create_keypair_at(call_path).unwrap_or_else(|e| { + panic!( + "{} the node identity, got: {e:#}\nkey storage at failure:\n{}", + if phase == "reload" { + "reload in a fresh process must load" + } else { + "first boot must create" + }, + identity_key_tree(base, key) + ) + }); + assert_identity_success(base, key, created); + kp.did().to_string() + } + + /// Fixture: one (layout, umask, phase) row of the identity lifecycle + /// matrix. + /// + /// `~/.gitlawb` holds BOTH keys, and `load_or_create_keypair` runs before + /// the listener binds, so a mask that strips owner bits here takes the + /// whole node down before any p2p code is reached. `umask` is + /// process-global, so every row is a child. Double-gated like the p2p + /// fixtures: `#[ignore]` keeps it out of a normal run and the env check + /// keeps it inert under a bare `--ignored` sweep, which would otherwise set + /// a process-global umask inside the shared test process. + #[cfg(unix)] + #[test] + #[ignore = "self-exec fixture: only runs under GITLAWB_TEST_FIXTURE=identity-key-umask"] + fn fixture_identity_key_under_hostile_umask() { + use std::os::unix::fs::PermissionsExt; + + if std::env::var("GITLAWB_TEST_FIXTURE").ok().as_deref() != Some("identity-key-umask") { + return; + } + let base = std::path::PathBuf::from( + std::env::var("GITLAWB_TEST_BASE").expect("GITLAWB_TEST_BASE"), + ); + let umask_val = + u32::from_str_radix(&std::env::var("GITLAWB_TEST_UMASK").unwrap(), 8).unwrap(); + let layout = std::env::var("GITLAWB_TEST_LAYOUT").expect("GITLAWB_TEST_LAYOUT"); + let phase = std::env::var("GITLAWB_TEST_PHASE").expect("GITLAWB_TEST_PHASE"); + + // SAFETY: `umask` only reads and replaces the process-wide value, and + // this process exists solely for this probe. No restore: the value dies + // with the child. Set FIRST, before any layout is built, so the + // explicit chmods below are what give a pre-existing directory its + // mode regardless of the mask. + unsafe { libc::umask(umask_val as libc::mode_t) }; + + match layout.as_str() { + "one-missing" => { + let dir = base.join(".gitlawb"); + let key = dir.join("identity.pem"); + let did = run_identity_success_row(&base, &key, &key, &[dir], &phase); + println!("identity-key-umask: asserted did={did}"); + } + "multi-missing" => { + let one = base.join("one"); + let two = one.join("two"); + let key = two.join("identity.pem"); + let did = run_identity_success_row(&base, &key, &key, &[one, two], &phase); + println!("identity-key-umask: asserted did={did}"); + } + "three-missing" => { + let one = base.join("one"); + let two = one.join("two"); + let three = two.join("three"); + let key = three.join("identity.pem"); + let did = run_identity_success_row(&base, &key, &key, &[one, two, three], &phase); + println!("identity-key-umask: asserted did={did}"); + } + "under-preexisting" => { + let pre = base.join("pre"); + if phase == "create" { + std::fs::create_dir(&pre).expect("create the pre-existing ancestor"); + std::fs::set_permissions( + &pre, + std::fs::Permissions::from_mode(IDENTITY_PREEXISTING_MODE), + ) + .expect("chmod the pre-existing ancestor"); + } + let one = pre.join("one"); + let two = one.join("two"); + let key = two.join("identity.pem"); + let did = run_identity_success_row(&base, &key, &key, &[one, two], &phase); + let pre_mode = identity_mode_of(&pre); + assert_eq!( + pre_mode, + IDENTITY_PREEXISTING_MODE, + "a pre-existing ancestor is neither chmodded nor judged: {} achieved \ + {pre_mode:04o}, expected {:04o}", + pre.display(), + IDENTITY_PREEXISTING_MODE + ); + println!("identity-key-umask: asserted did={did}"); + } + "relative-multi-missing" => { + std::env::set_current_dir(&base).expect("chdir into the row's base"); + let one = base.join("one"); + let two = one.join("two"); + let key = two.join("identity.pem"); + let call_path = std::path::PathBuf::from("one/two/identity.pem"); + let did = run_identity_success_row(&base, &call_path, &key, &[one, two], &phase); + println!("identity-key-umask: asserted did={did}"); + } + "loose-ancestor" => { + let loose = base.join("loose"); + std::fs::create_dir(&loose).expect("create the group-writable ancestor"); + std::fs::set_permissions( + &loose, + std::fs::Permissions::from_mode(IDENTITY_LOOSE_MODE), + ) + .expect("chmod the group-writable ancestor"); + let one = loose.join("one"); + let two = one.join("two"); + let key = two.join("identity.pem"); + let Err(err) = load_or_create_keypair_at(&key) else { + panic!( + "a group-writable deepest ancestor must be refused\nkey storage:\n{}", + identity_key_tree(&base, &key) + ) + }; + let text = format!("{err:#}"); + // The reason is checked before the leftovers so a refusal for + // the wrong reason is reported as that, not as a stray file. + assert!( + text.contains("writable beyond its owner"), + "a group-writable deepest ancestor must be refused for the write-authority \ + reason, got: {text}" + ); + assert_identity_absent(&one); + assert_identity_absent(&two); + assert_identity_absent(&key); + let loose_mode = identity_mode_of(&loose); + assert_eq!( + loose_mode, + IDENTITY_LOOSE_MODE, + "a refusal must not chmod {}: achieved {loose_mode:04o}", + loose.display() + ); + println!("identity-key-umask: refused"); + } + "write-failure" => { + let one = base.join("one"); + let two = one.join("two"); + let three = two.join("three"); + let key = three.join("identity.pem"); + // Armed on this thread only, and disarmed before the + // assertions so a later boot in this process is unaffected. + p2p::FAIL_KEY_WRITE.with(|f| f.set(true)); + let result = load_or_create_keypair_at(&key); + p2p::FAIL_KEY_WRITE.with(|f| f.set(false)); + let Err(err) = result else { + panic!("an injected key-write failure must not report a created identity") + }; + let text = format!("{err:#}"); + assert!( + text.contains("injected key-write failure"), + "the failure must name the injected key write, got: {text}" + ); + assert_identity_absent(&one); + assert_identity_absent(&two); + assert_identity_absent(&three); + assert_identity_absent(&key); + let residue: Vec = std::fs::read_dir(&base) + .expect("read the base directory") + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + assert!( + residue.is_empty(), + "a failed first boot must leave nothing under the base: {residue:?}" + ); + println!("identity-key-umask: refused"); + } + "race-loose-intermediate" => { + let one = base.join("one"); + let two = one.join("two"); + let key = two.join("identity.pem"); + // A concurrent process wins the mkdir with a group-writable + // mode. The loser adopts nothing it did not create, so its + // rollback must leave the winner's directory alone. + p2p::pin::RACE_CREATE_MODE.with(|c| c.set(Some(IDENTITY_LOOSE_MODE))); + let result = load_or_create_keypair_at(&key); + let Err(err) = result else { + panic!("a race-won group-writable intermediate must be refused") + }; + let text = format!("{err:#}"); + assert!( + text.contains("writable beyond its owner"), + "the refusal must name the write-authority problem, got: {text}" + ); + assert!( + one.exists(), + "a directory this invocation did not create must not be removed: {} is gone", + one.display() + ); + let one_mode = identity_mode_of(&one); + assert_eq!( + one_mode, + IDENTITY_LOOSE_MODE, + "a directory this invocation did not create must not be removed or chmodded: \ + {} achieved {one_mode:04o}", + one.display() + ); + assert_identity_absent(&two); + assert_identity_absent(&key); + println!("identity-key-umask: refused"); + } + "symlinked-deepest-ancestor" => { + let real = base.join("real"); + std::fs::create_dir(&real).expect("create the real ancestor"); + std::fs::set_permissions(&real, std::fs::Permissions::from_mode(0o700)) + .expect("chmod the real ancestor"); + let link = base.join("link"); + std::os::unix::fs::symlink("real", &link).expect("plant the symlinked ancestor"); + let key = link.join("one").join("two").join("identity.pem"); + let Err(err) = load_or_create_keypair_at(&key) else { + panic!("a symlinked deepest existing ancestor must be refused, not followed") + }; + let text = format!("{err:#}"); + assert!( + text.contains("a symlink here is refused rather than followed"), + "a symlinked deepest existing ancestor must be refused, not followed, and the \ + refusal must say so, got: {text}" + ); + for p in [ + link.join("one"), + link.join("one").join("two"), + key.clone(), + real.join("one"), + real.join("one").join("two"), + real.join("one").join("two").join("identity.pem"), + ] { + assert_identity_absent(&p); + } + println!("identity-key-umask: refused"); + } + "populated-leaf-refuses-rollback" => { + let one = base.join("one"); + let two = one.join("two"); + let key = two.join("identity.pem"); + // The key is published and only the scratch removal fails, so + // the leaf this invocation created is no longer empty when the + // rollback reaches it. `AT_REMOVEDIR` must refuse it. + p2p::FAIL_SCRATCH_UNLINK.with(|f| f.set(true)); + let result = load_or_create_keypair_at(&key); + p2p::FAIL_SCRATCH_UNLINK.with(|f| f.set(false)); + let Err(err) = result else { + panic!("an injected scratch-unlink failure must not report success") + }; + let text = format!("{err:#}"); + assert!( + text.contains("failed to remove the scratch name"), + "the failure must name the scratch it could not remove, got: {text}" + ); + assert!( + text.contains("also failed to remove") + && text.contains("which this process created"), + "the rollback must report the populated leaf it was refused, got: {text}" + ); + let key_mode = identity_mode_of(&key); + assert_eq!( + key_mode, IDENTITY_WANT_KEY_MODE, + "a published key must survive its own boot's rollback: achieved \ + {key_mode:04o}, requested 0600" + ); + for d in [&one, &two] { + let mode = identity_mode_of(d); + assert_eq!( + mode, + IDENTITY_WANT_DIR_MODE, + "created directory {} achieved mode {mode:04o}, requested {:04o}", + d.display(), + IDENTITY_WANT_DIR_MODE + ); + } + let scratch: Vec = std::fs::read_dir(&two) + .expect("read the key directory") + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|n| n.starts_with(".p2p.key.") && n.ends_with(".tmp")) + .collect(); + assert_eq!( + scratch.len(), + 1, + "the refused unlink must leave exactly one scratch beside the key: {scratch:?}" + ); + let kp = load_or_create_keypair_at(&key) + .expect("a boot after the injected failure must load the published key"); + println!("identity-key-umask: refused did={}", kp.did()); + } + // One process of a real concurrent first boot. Every child of a + // race runs this arm; the driver owns the tree assertions, because + // only it knows the race has finished. + "concurrent-first-boot" => { + let barrier = std::path::PathBuf::from( + std::env::var("GITLAWB_TEST_BARRIER").expect("GITLAWB_TEST_BARRIER"), + ); + let id = std::env::var("GITLAWB_TEST_RACE_ID").expect("GITLAWB_TEST_RACE_ID"); + let role = std::env::var("GITLAWB_TEST_RACE_ROLE").expect("GITLAWB_TEST_RACE_ROLE"); + let key = base.join("one").join("two").join("identity.pem"); + + // Two-phase rendezvous: each child announces itself with + // `ready.` and then spins on `go`, which the driver creates + // only once every ready file is present. Chosen over a shared + // wake-up timestamp because a deadline bounds only the skew it + // cannot observe: a child that is slow to exec still arrives + // late and the race never overlaps. Here every child is already + // inside the spin loop before `go` can appear, so the spread + // across the racing call is one loop iteration rather than the + // tens of milliseconds a process takes to start. The deadline + // exists so a child whose sibling died never hangs the suite. + std::fs::write(barrier.join(format!("ready.{id}")), b"") + .expect("announce readiness at the starting barrier"); + let go = barrier.join("go"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(120); + while !go.exists() { + assert!( + std::time::Instant::now() < deadline, + "the starting barrier never opened" + ); + std::hint::spin_loop(); + } + + // The `fail-write` role is the only injected part of the race, + // and it is what puts a REAL second process into the window + // section 7 of the plan describes: this child creates a + // component, the other adopts it, this one fails before either + // has written, and its rollback reaches the still-empty + // directory the other is holding open. + if role == "fail-write" { + p2p::FAIL_KEY_WRITE.with(|f| f.set(true)); + } + let result = load_or_create_keypair_at(&key); + p2p::FAIL_KEY_WRITE.with(|f| f.set(false)); + + match result { + Ok(kp) => { + // A boot that reports success must have the identity it + // returned on disk, at 0600, and readable as a keypair. + // Checked here rather than in the driver so a success + // that raced with a rollback is attributed to the + // process that claimed it. + let mode = identity_mode_of(&key); + assert_eq!( + mode, + IDENTITY_WANT_KEY_MODE, + "a successful concurrent boot published {} at {mode:04o}, requested \ + 0600", + key.display() + ); + let pem = std::fs::read_to_string(&key) + .unwrap_or_else(|e| panic!("read {}: {e}", key.display())); + let on_disk = Keypair::from_pem(&pem) + .expect("a successful concurrent boot must leave a loadable keypair"); + assert_eq!( + on_disk.did().to_string(), + kp.did().to_string(), + "a successful boot must return the identity that is on disk" + ); + println!("identity-key-race: ok did={}", kp.did()); + } + Err(err) => { + // One line so the driver can classify it; the anyhow + // chain is multi-line under `{:#}` only when a context + // carries a newline, but flattening is free insurance. + let text = format!("{err:#}").replace('\n', " "); + println!("identity-key-race: err {text}"); + } + } + } + other => panic!("unknown layout {other}"), + } + } + + /// GITLAWB_KEY names a PEM file, not a dedicated key directory. An existing + /// 0755 parent is usable (no group/world write) and must not be chmodded. + #[cfg(unix)] + #[test] + fn existing_identity_parent_is_not_chmodded() { + use std::os::unix::fs::PermissionsExt; + + for parent_name in [".gitlawb", "shared"] { + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let dir = base.path().join(parent_name); + std::fs::create_dir(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)).unwrap(); + + let key = dir.join("identity.pem"); + load_or_create_keypair_at(&key).expect("first boot into an existing 0755 parent"); + + let mode = std::fs::symlink_metadata(&dir) + .unwrap() + .permissions() + .mode() + & 0o7777; + assert_eq!( + mode, 0o755, + "{parent_name}: an existing 0755 identity parent must stay 0755, found {mode:04o}" + ); + assert_eq!( + std::fs::symlink_metadata(&key) + .unwrap() + .permissions() + .mode() + & 0o7777, + 0o600, + "the identity key must be owner-only" + ); + } + } + + /// A parent this process creates is still pinned to 0700. That is the + /// missing-directory path, not an adopt of an operator-owned tree. + #[cfg(unix)] + #[test] + fn missing_identity_parent_is_created_0700() { + use std::os::unix::fs::PermissionsExt; + + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let dir = base.path().join(".gitlawb"); + let key = dir.join("identity.pem"); + load_or_create_keypair_at(&key).expect("first boot creates the identity parent"); + let mode = std::fs::symlink_metadata(&dir) + .unwrap() + .permissions() + .mode() + & 0o7777; + assert_eq!(mode, 0o700, "a parent this process created must be 0700"); + } + + /// Group/world write on the parent is replacement authority over a 0600 + /// key. Refuse, and leave the directory untouched. + #[cfg(unix)] + #[test] + fn writable_identity_parent_is_refused_unchanged() { + use std::os::unix::fs::PermissionsExt; + + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let dir = base.path().join("tmp"); + std::fs::create_dir(&dir).unwrap(); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o777)).unwrap(); + let key = dir.join("identity.pem"); + let Err(err) = load_or_create_keypair_at(&key) else { + panic!("a world-writable identity parent must be refused"); + }; + let text = format!("{err:#}"); + assert!( + text.contains("writable") || text.contains("replace"), + "the refusal must name the write-authority problem, got: {text}" + ); + let mode = std::fs::symlink_metadata(&dir) + .unwrap() + .permissions() + .mode() + & 0o7777; + assert_eq!(mode, 0o777, "a refusal must not chmod the parent"); + assert!(!key.exists(), "a refusal must not publish the identity key"); + } + + /// An existing key is loaded, never rewritten and never chmodded: the + /// creation path is the only thing this change touches. + #[cfg(unix)] + #[test] + fn existing_identity_key_is_loaded_unchanged() { + use std::os::unix::fs::PermissionsExt; + + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let key = base.path().join(".gitlawb").join("identity.pem"); + + let created = load_or_create_keypair_at(&key).expect("first boot"); + let before = std::fs::read(&key).unwrap(); + + // A deliberately odd but readable mode must survive: an existing key is + // the operator's, and this path does not repair it. + std::fs::set_permissions(&key, std::fs::Permissions::from_mode(0o400)).unwrap(); + let reloaded = load_or_create_keypair_at(&key).expect("reload"); + + assert_eq!(created.did(), reloaded.did(), "the identity must be stable"); + assert_eq!( + std::fs::read(&key).unwrap(), + before, + "the key must not be rewritten" + ); + assert_eq!( + std::fs::symlink_metadata(&key) + .unwrap() + .permissions() + .mode() + & 0o7777, + 0o400, + "an existing key's mode must not be changed" + ); + } + + /// Parent for the identity lifecycle matrix: drives every row as a child + /// process and requires create and reload to agree on the DID. + #[cfg(unix)] + #[test] + fn identity_key_storage_is_umask_independent() { + use std::os::unix::fs::PermissionsExt; + + const SUCCESS_SENTINEL: &str = "identity-key-umask: asserted did="; + const REFUSED_SENTINEL: &str = "identity-key-umask: refused"; + // Fixed order, and part of the contract: 0000 and 0002 are the masks + // that leave a group/world-writable ancestor behind, 0022 is the one + // that boots with the wrong mode, and 0777 strips owner bits. Rows + // abort at the first failing mask, so several load-bearing proofs are + // pinned to which mask that is. + const ALL_UMASKS: &[&str] = &["0000", "0002", "0022", "0777"]; + + // (layout, umasks, succeeds). A success row runs create then reload + // against one base and must return the same DID; a reject row runs + // create alone and must print the refusal sentinel. + let rows: &[(&str, &[&str], bool)] = &[ + ("one-missing", ALL_UMASKS, true), + ("multi-missing", ALL_UMASKS, true), + ("three-missing", ALL_UMASKS, true), + ("under-preexisting", ALL_UMASKS, true), + ("relative-multi-missing", ALL_UMASKS, true), + ("loose-ancestor", ALL_UMASKS, false), + ("write-failure", ALL_UMASKS, false), + ("race-loose-intermediate", &["0002"], false), + ("symlinked-deepest-ancestor", &["0022"], false), + ("populated-leaf-refuses-rollback", &["0022"], false), + ]; + + // Both filters are optional and both panic on a value that names + // nothing: a typo that quietly ran no child would be an empty green, + // which is the failure this matrix exists to rule out. + let layout_only = std::env::var("GITLAWB_TEST_LAYOUT_ONLY").ok(); + let umask_only = std::env::var("GITLAWB_TEST_UMASK_ONLY").ok(); + if let Some(l) = layout_only.as_deref() { + assert!( + rows.iter().any(|r| r.0 == l), + "GITLAWB_TEST_LAYOUT_ONLY={l} names no row in the matrix" + ); + } + if let Some(u) = umask_only.as_deref() { + assert!( + ALL_UMASKS.contains(&u), + "GITLAWB_TEST_UMASK_ONLY={u} names no umask in the matrix" + ); + } + + // Run one row in a child and return whatever follows its sentinel. + fn run_row( + base: &std::path::Path, + umask: &str, + layout: &str, + phase: &str, + sentinel: &str, + ) -> String { + let mut cmd = std::process::Command::new(std::env::current_exe().expect("current_exe")); + cmd.args([ + "identity_key_storage_tests::fixture_identity_key_under_hostile_umask", + "--exact", + "--ignored", + "--nocapture", + ]) + .env("GITLAWB_TEST_FIXTURE", "identity-key-umask") + .env("GITLAWB_TEST_BASE", base) + .env("GITLAWB_TEST_UMASK", umask) + .env("GITLAWB_TEST_LAYOUT", layout) + .env("GITLAWB_TEST_PHASE", phase); + let output = cmd.output().expect("spawn the identity-key fixture"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let row = format!("layout={layout} umask={umask} phase={phase}"); + + assert!( + output.status.success(), + "row {row}: the identity-key fixture must pass\n\ + --- stdout ---\n{stdout}\n--- stderr ---\n{stderr}" + ); + // A filter matching nothing exits 0, and the fixture's env gate + // returns early as a passing test, so neither alone is proof. + assert!( + stdout.contains("1 passed"), + "row {row}: filter must select one passing test\n{stdout}" + ); + let line = stdout + .lines() + .find(|l| l.starts_with(sentinel)) + .unwrap_or_else(|| { + panic!("row {row}: fixture must print its sentinel\n--- stdout ---\n{stdout}") + }); + line[sentinel.len()..].trim().to_string() + } + + let mut ran = 0usize; + for (layout, umasks, succeeds) in rows { + if layout_only.as_deref().is_some_and(|l| l != *layout) { + continue; + } + for umask in *umasks { + if umask_only.as_deref().is_some_and(|u| u != *umask) { + continue; + } + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)) + .unwrap(); + if *succeeds { + let created = run_row(base.path(), umask, layout, "create", SUCCESS_SENTINEL); + let reloaded = run_row(base.path(), umask, layout, "reload", SUCCESS_SENTINEL); + assert_eq!( + created, reloaded, + "layout={layout} umask={umask}: a fresh process must reload the same DID" + ); + } else { + run_row(base.path(), umask, layout, "create", REFUSED_SENTINEL); + } + ran += 1; + } + } + assert!( + ran > 0, + "the row and umask filters selected no child; a matrix that runs nothing is not a \ + passing matrix" + ); + } + + /// Two or more real first boots racing the same multi-level missing key + /// path. + /// + /// The rollback added by this round can remove an intermediate a concurrent + /// boot has already adopted, so the plan's section 7 claims both processes + /// still fail closed. That claim was reasoned, not executed. This drives it + /// with real processes: `umask` and the injection hooks are process-global, + /// so threads inside one process would not be the production path. + /// + /// The race is nondeterministic and the assertions are not. Nothing here + /// asserts a particular interleaving; every check is a property that must + /// hold for all of them, so the test is deterministic in PASS/FAIL: + /// + /// * a boot that reports success has its own identity on disk at 0600 and + /// parseable (asserted in the child, which is the process that claimed + /// it); + /// * every surviving directory this run created is exactly 0700, never + /// group or world writable; + /// * every successful boot agrees on the DID; + /// * if no boot succeeded, nothing partial is left (no key, no scratch) and + /// a retry in this process then succeeds, which is the explicit form of + /// "fail closed and recover on the next boot". + #[cfg(unix)] + #[test] + fn identity_concurrent_first_boots_fail_closed() { + use std::io::Read; + use std::os::unix::fs::PermissionsExt; + + const OK_SENTINEL: &str = "identity-key-race: ok did="; + const ERR_SENTINEL: &str = "identity-key-race: err "; + // 0002 is the mask that produced finding 1, so a directory this run + // creates without pinning it lands 0775 and the exact-mode assertions + // below catch it. + const RACE_UMASK: &str = "0002"; + + // (arm, one role per child). The pure arm is three unaided first boots. + // The other two add losers that fail after the walk and before any key + // is written, so their rollback reaches a directory a sibling may + // already hold open: that is the window the plan describes and the only + // injected part of the race. Two children hit it rarely, because the + // winner reaches its scratch file before a single loser reaches its + // rmdir; four losers against one unaided boot hit it often, so the + // pressure arm is what actually executes the interleaving. The + // assertions do not depend on which arm hits it. + let arms: &[(&str, &[&str])] = &[ + ("pure", &["pure", "pure", "pure"][..]), + ("loser-fails", &["fail-write", "pure"][..]), + ( + "rollback-pressure", + &[ + "fail-write", + "fail-write", + "fail-write", + "fail-write", + "pure", + ][..], + ), + ]; + + // A single run of a nondeterministic test proves very little, so the + // default loops the whole race; raise it through the env for a longer + // shake without editing the test. + let iterations: usize = std::env::var("GITLAWB_TEST_RACE_ITERATIONS") + .ok() + .map(|v| { + v.parse() + .expect("GITLAWB_TEST_RACE_ITERATIONS must be a count") + }) + .unwrap_or(50); + assert!(iterations > 0, "a race run zero times proves nothing"); + + #[derive(Default)] + struct Tally { + all_ok: usize, + some_ok: usize, + none_ok: usize, + /// A boot refused because the directory it was holding had been + /// removed under it: the interleaving section 7 describes. + adopted_dir_unlinked: usize, + } + let mut tallies: Vec<(&str, Tally)> = arms + .iter() + .map(|(name, _)| (*name, Tally::default())) + .collect(); + + for iteration in 0..iterations { + for (arm_idx, (arm, roles)) in arms.iter().enumerate() { + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)) + .unwrap(); + let barrier = tempfile::tempdir().unwrap(); + let one = base.path().join("one"); + let two = one.join("two"); + let key = two.join("identity.pem"); + let label = format!("arm={arm} iteration={iteration}"); + + let mut children = Vec::new(); + for (id, role) in roles.iter().enumerate() { + let mut cmd = + std::process::Command::new(std::env::current_exe().expect("current_exe")); + cmd.args([ + "identity_key_storage_tests::fixture_identity_key_under_hostile_umask", + "--exact", + "--ignored", + "--nocapture", + ]) + .env("GITLAWB_TEST_FIXTURE", "identity-key-umask") + .env("GITLAWB_TEST_BASE", base.path()) + .env("GITLAWB_TEST_UMASK", RACE_UMASK) + .env("GITLAWB_TEST_LAYOUT", "concurrent-first-boot") + .env("GITLAWB_TEST_PHASE", "create") + .env("GITLAWB_TEST_BARRIER", barrier.path()) + .env("GITLAWB_TEST_RACE_ID", id.to_string()) + .env("GITLAWB_TEST_RACE_ROLE", *role) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + children.push(cmd.spawn().expect("spawn a racing identity boot")); + } + + // Open the barrier only once every child has announced itself. + // If one died before announcing, the deadline lets the run + // proceed to the exit-status assertion below rather than hang. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(120); + loop { + let ready = (0..roles.len()) + .filter(|id| barrier.path().join(format!("ready.{id}")).exists()) + .count(); + if ready == roles.len() || std::time::Instant::now() >= deadline { + break; + } + std::thread::yield_now(); + } + std::fs::write(barrier.path().join("go"), b"").expect("open the starting barrier"); + + let mut oks = Vec::new(); + let mut errs = Vec::new(); + for (id, child) in children.into_iter().enumerate() { + let output = child.wait_with_output().expect("await a racing boot"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let who = format!("{label} child={id} role={}", roles[id]); + assert!( + output.status.success(), + "{who}: the racing fixture must pass whichever way the race went\n\ + --- stdout ---\n{stdout}\n--- stderr ---\n{stderr}" + ); + assert!( + stdout.contains("1 passed"), + "{who}: filter must select one passing test\n{stdout}" + ); + let line = stdout + .lines() + .find(|l| l.starts_with("identity-key-race: ")) + .unwrap_or_else(|| { + panic!("{who}: the child must report its outcome\n{stdout}") + }); + if let Some(did) = line.strip_prefix(OK_SENTINEL) { + oks.push(did.trim().to_string()); + } else if let Some(text) = line.strip_prefix(ERR_SENTINEL) { + errs.push((roles[id], text.to_string())); + } else { + panic!("{who}: unrecognized outcome line {line:?}"); + } + } + + // Invariant: every surviving directory this run created is + // exactly 0700. A missing one is legal (a loser rolled it + // back); a group or world writable one never is. + for d in [&one, &two] { + if d.exists() { + let mode = identity_mode_of(d); + assert_eq!( + mode, + IDENTITY_WANT_DIR_MODE, + "{label}: surviving directory {} achieved mode {mode:04o}, requested \ + {:04o}", + d.display(), + IDENTITY_WANT_DIR_MODE + ); + let residue: Vec = std::fs::read_dir(d) + .expect("read a surviving directory") + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|n| n.starts_with(".p2p.key.") && n.ends_with(".tmp")) + .collect(); + assert!( + residue.is_empty(), + "{label}: a finished race must leave no scratch behind in {}: \ + {residue:?}", + d.display() + ); + } + } + + if key.exists() { + // Invariant: the published key is 0600 and loadable, and + // every boot that reported success agrees on it. + let mode = identity_mode_of(&key); + assert_eq!( + mode, IDENTITY_WANT_KEY_MODE, + "{label}: the surviving key achieved mode {mode:04o}, requested 0600" + ); + let mut pem = String::new(); + std::fs::File::open(&key) + .expect("open the surviving key") + .read_to_string(&mut pem) + .expect("read the surviving key"); + let on_disk = Keypair::from_pem(&pem) + .expect("the surviving key must parse as a keypair") + .did() + .to_string(); + for did in &oks { + assert_eq!( + *did, on_disk, + "{label}: a successful boot reported an identity the disk does not \ + hold" + ); + } + } else { + // Invariant: no key means no boot may claim success, and + // the tree must be clean enough that the next boot works. + assert!( + oks.is_empty(), + "{label}: a boot reported success with no key on disk: {oks:?}" + ); + let kp = load_or_create_keypair_at(&key).unwrap_or_else(|e| { + panic!( + "{label}: every boot failed, so a retry must succeed, got: {e:#}\n\ + errors: {errs:?}\nkey storage:\n{}", + identity_key_tree(base.path(), &key) + ) + }); + assert_identity_success(base.path(), &key, &[one.clone(), two.clone()]); + assert!( + !kp.did().to_string().is_empty(), + "{label}: the recovering boot must produce an identity" + ); + } + + let tally = &mut tallies[arm_idx].1; + if oks.len() == roles.len() { + tally.all_ok += 1; + } else if oks.is_empty() { + tally.none_ok += 1; + } else { + tally.some_ok += 1; + } + // The rollback-under-an-adopted-directory window: a boot that + // was refused because the directory it held was unlinked. + if errs.iter().any(|(_, t)| { + t.contains("No such file or directory") || t.contains("(os error 2)") + }) { + tally.adopted_dir_unlinked += 1; + } + } + } + + for (arm, tally) in &tallies { + println!( + "identity-race-summary: arm={arm} iterations={iterations} all_ok={} some_ok={} \ + none_ok={} adopted_dir_unlinked={}", + tally.all_ok, tally.some_ok, tally.none_ok, tally.adopted_dir_unlinked + ); + } + } + + /// Bare `GITLAWB_KEY=identity.pem` (and `./identity.pem`) must still create + /// the identity in the working directory. The p2p key refuses that form on + /// purpose; the node identity has always allowed it. + #[cfg(unix)] + #[test] + #[ignore = "self-exec fixture: only runs under GITLAWB_TEST_FIXTURE=identity-key-bare"] + fn fixture_bare_identity_key_in_cwd() { + use std::os::unix::fs::PermissionsExt; + + if std::env::var("GITLAWB_TEST_FIXTURE").ok().as_deref() != Some("identity-key-bare") { + return; + } + let cwd = std::path::PathBuf::from( + std::env::var("GITLAWB_TEST_BASE").expect("GITLAWB_TEST_BASE"), + ); + std::env::set_current_dir(&cwd).expect("chdir into isolated tempdir"); + let cwd_mode_before = std::fs::symlink_metadata(".").unwrap().permissions().mode() & 0o7777; + + let name = std::env::var("GITLAWB_TEST_KEY_NAME").expect("GITLAWB_TEST_KEY_NAME"); + let kp = load_or_create_keypair_at(std::path::Path::new(&name)) + .unwrap_or_else(|e| panic!("bare identity path {name:?} must create, got: {e:#}")); + assert!( + std::path::Path::new(&name).exists() || std::path::Path::new("identity.pem").exists(), + "the key must land in the working directory" + ); + let key = if std::path::Path::new(&name).exists() { + std::path::PathBuf::from(&name) + } else { + std::path::PathBuf::from("identity.pem") + }; + assert_eq!( + std::fs::symlink_metadata(&key) + .unwrap() + .permissions() + .mode() + & 0o7777, + 0o600, + "the identity key must be owner-only" + ); + let cwd_mode_after = std::fs::symlink_metadata(".").unwrap().permissions().mode() & 0o7777; + assert_eq!( + cwd_mode_before, cwd_mode_after, + "creating a bare identity key must not chmod the working directory" + ); + let reloaded = load_or_create_keypair_at(std::path::Path::new(&name)).expect("reload"); + assert_eq!(kp.did(), reloaded.did(), "the identity must be stable"); + println!("identity-key-bare: asserted did={}", kp.did()); + } + + #[cfg(unix)] + #[test] + fn bare_identity_key_path_creates_in_cwd_without_chmodding_cwd() { + use std::os::unix::fs::PermissionsExt; + + for name in ["identity.pem", "./identity.pem"] { + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); + let mut cmd = std::process::Command::new(std::env::current_exe().expect("current_exe")); + cmd.args([ + "identity_key_storage_tests::fixture_bare_identity_key_in_cwd", + "--exact", + "--ignored", + "--nocapture", + ]) + .env("GITLAWB_TEST_FIXTURE", "identity-key-bare") + .env("GITLAWB_TEST_BASE", base.path()) + .env("GITLAWB_TEST_KEY_NAME", name); + let output = cmd.output().expect("spawn the bare-identity fixture"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "name={name}: bare identity path must create\n\ + --- stdout ---\n{stdout}\n--- stderr ---\n{stderr}" + ); + assert!( + stdout.contains("1 passed"), + "name={name}: filter must select one passing test\n{stdout}" + ); + assert!( + stdout.contains("identity-key-bare: asserted did="), + "name={name}: fixture must print its sentinel\n{stdout}" + ); + let cwd_mode = std::fs::symlink_metadata(base.path()) + .unwrap() + .permissions() + .mode() + & 0o7777; + assert_eq!( + cwd_mode, 0o755, + "name={name}: the parent test must also see cwd left at 0755" + ); + } + } + + /// A 0600 identity file is not protected if cwd is group/world-writable: + /// another local user can unlink or replace the entry. Creating into that + /// cwd must fail, and must not leave a key behind. Do not chmod cwd. + #[cfg(unix)] + #[test] + #[ignore = "self-exec fixture: only runs under GITLAWB_TEST_FIXTURE=identity-key-bare-writable"] + fn fixture_bare_identity_key_in_writable_cwd_is_refused() { + use std::os::unix::fs::PermissionsExt; + + if std::env::var("GITLAWB_TEST_FIXTURE").ok().as_deref() + != Some("identity-key-bare-writable") + { + return; + } + let cwd = std::path::PathBuf::from( + std::env::var("GITLAWB_TEST_BASE").expect("GITLAWB_TEST_BASE"), + ); + std::env::set_current_dir(&cwd).expect("chdir into isolated tempdir"); + std::fs::set_permissions(&cwd, std::fs::Permissions::from_mode(0o777)).unwrap(); + let cwd_mode_before = std::fs::symlink_metadata(".").unwrap().permissions().mode() & 0o7777; + + let Err(err) = load_or_create_keypair_at(std::path::Path::new("identity.pem")) else { + panic!("a bare identity path under a writable cwd must be refused"); + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("writable beyond its owner") || msg.contains("0777"), + "the refusal must name the writable-parent reason, got: {msg}" + ); + assert!( + !std::path::Path::new("identity.pem").exists(), + "a refused cwd must not have identity.pem created" + ); + let cwd_mode_after = std::fs::symlink_metadata(".").unwrap().permissions().mode() & 0o7777; + assert_eq!( + cwd_mode_before, cwd_mode_after, + "refusing a writable cwd must not chmod it" + ); + println!("identity-key-bare-writable: asserted"); + } + + #[cfg(unix)] + #[test] + fn bare_identity_key_path_refuses_a_writable_cwd() { + use std::os::unix::fs::PermissionsExt; + + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o777)).unwrap(); + let mut cmd = std::process::Command::new(std::env::current_exe().expect("current_exe")); + cmd.args([ + "identity_key_storage_tests::fixture_bare_identity_key_in_writable_cwd_is_refused", + "--exact", + "--ignored", + "--nocapture", + ]) + .env("GITLAWB_TEST_FIXTURE", "identity-key-bare-writable") + .env("GITLAWB_TEST_BASE", base.path()); + let output = cmd + .output() + .expect("spawn the writable-cwd identity fixture"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "writable cwd must refuse the bare identity path\n\ + --- stdout ---\n{stdout}\n--- stderr ---\n{stderr}" + ); + assert!( + stdout.contains("1 passed"), + "filter must select one passing test\n{stdout}" + ); + assert!( + stdout.contains("identity-key-bare-writable: asserted"), + "fixture must print its sentinel\n{stdout}" + ); + let leftovers: Vec<_> = std::fs::read_dir(base.path()) + .unwrap() + .filter_map(|e| e.ok()) + .map(|e| e.file_name()) + .collect(); + assert!( + leftovers.is_empty(), + "parent must also see no identity.pem, found: {leftovers:?}" + ); + let cwd_mode = std::fs::symlink_metadata(base.path()) + .unwrap() + .permissions() + .mode() + & 0o7777; + assert_eq!(cwd_mode, 0o777, "cwd must stay 0777"); + } + + /// An existing key in a writable directory still loads. Create is refused + /// there; turning that into a boot failure on upgrade would strand nodes + /// that already have a key. + #[cfg(unix)] + #[test] + fn existing_identity_in_a_writable_dir_still_loads() { + use std::os::unix::fs::PermissionsExt; + + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); + let key = base.path().join("identity.pem"); + let created = load_or_create_keypair_at(&key).expect("create under 0755"); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o777)).unwrap(); + let reloaded = load_or_create_keypair_at(&key) + .expect("an existing identity in a writable directory must still load"); + assert_eq!(created.did(), reloaded.did()); + assert_eq!( + std::fs::symlink_metadata(base.path()) + .unwrap() + .permissions() + .mode() + & 0o7777, + 0o777, + "load must not chmod the writable directory" + ); + } + + #[cfg(unix)] + #[test] + fn identity_symlink_key_is_refused() { + use std::os::unix::fs::PermissionsExt; + + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let real = base.path().join("real.pem"); + let created = load_or_create_keypair_at(&real).expect("create the symlink target"); + std::os::unix::fs::symlink(&real, base.path().join("identity.pem")).unwrap(); + let err = match load_or_create_keypair_at(&base.path().join("identity.pem")) { + Ok(kp) => panic!( + "a symlink at the identity path must be refused, not followed; loaded did={}", + kp.did() + ), + Err(e) => format!("{e:#}"), + }; + assert!( + err.contains("symlink") || err.to_lowercase().contains("too many levels"), + "symlink refusal must name the link, got: {err}" + ); + let reread = load_or_create_keypair_at(&real).expect("target still loads by its real path"); + assert_eq!( + created.did(), + reread.did(), + "the symlink target must be unchanged" + ); + } + + #[cfg(unix)] + #[test] + fn identity_symlinked_grandparent_is_refused() { + use std::os::unix::fs::PermissionsExt; + + let base = tempfile::tempdir().unwrap(); + let real = base.path().join("real"); + std::fs::create_dir(&real).unwrap(); + std::fs::set_permissions(&real, std::fs::Permissions::from_mode(0o700)).unwrap(); + let link = base.path().join("link"); + std::os::unix::fs::symlink(&real, &link).unwrap(); + let key = link.join("keys").join("identity.pem"); + let err = match load_or_create_keypair_at(&key) { + Ok(_) => panic!("a symlinked grandparent must be refused, not followed"), + Err(e) => format!("{e:#}"), + }; + assert!( + err.contains("symlink") + || err.to_lowercase().contains("too many levels") + || err.contains("loop"), + "symlinked grandparent refusal must name the link, got: {err}" + ); + assert!( + !real.join("keys").exists(), + "symlink target must be untouched" + ); + } + + /// Build `base/evil/one` plus `base/link -> evil`, the layout every + /// interior-symlink row shares. `link` is relative so the layout is + /// self-contained; the absolute row builds its own. + #[cfg(unix)] + fn plant_interior_symlink_layout(base: &std::path::Path) -> std::path::PathBuf { + use std::os::unix::fs::PermissionsExt; + + let evil = base.join("evil"); + std::fs::create_dir(&evil).expect("create the symlink target directory"); + std::fs::set_permissions(&evil, std::fs::Permissions::from_mode(0o700)) + .expect("chmod the symlink target directory"); + let one = evil.join("one"); + std::fs::create_dir(&one).expect("create the already-existing next level"); + std::fs::set_permissions(&one, std::fs::Permissions::from_mode(0o700)) + .expect("chmod the already-existing next level"); + std::os::unix::fs::symlink("evil", base.join("link")).expect("plant the interior symlink"); + evil + } + + /// `O_NOFOLLOW` binds the FINAL component only, so an interior symlink + /// whose next level already exists is resolved in one open and followed. + /// With `link -> evil` and `evil/one` present, the walk-up opens + /// `link/one` by pathname, anchors on `evil/one`, and creates the node + /// identity inside the attacker's tree. + #[cfg(unix)] + #[test] + fn identity_interior_symlink_with_existing_next_level_is_refused_not_followed() { + use std::os::unix::fs::PermissionsExt; + + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let evil = plant_interior_symlink_layout(base.path()); + + let key = base + .path() + .join("link") + .join("one") + .join("two") + .join("identity.pem"); + match load_or_create_keypair_at(&key) { + Ok(kp) => panic!( + "an interior symlink on the key path was followed, not refused: the identity \ + {} was created through {} into the symlink's target\nkey storage:\n{}", + kp.did(), + key.display(), + identity_key_tree(base.path(), &key) + ), + Err(e) => assert_names_symlink_refusal( + &format!("{e:#}"), + "an interior symlink whose next level already exists", + ), + } + assert_identity_absent(&evil.join("one").join("two")); + assert_no_identity_key_under( + base.path(), + "an interior symlink with an existing next level", + ); + } + + /// The same interior symlink pointed by absolute path at a directory + /// outside the base. Following it publishes the node's identity somewhere + /// the configured path does not name at all, so the assertion is that no + /// key appears outside the base. + #[cfg(unix)] + #[test] + fn identity_interior_symlink_escaping_the_base_must_not_publish_the_key_outside_it() { + use std::os::unix::fs::PermissionsExt; + + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let outside = tempfile::tempdir().unwrap(); + std::fs::set_permissions(outside.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let outside_one = outside.path().join("one"); + std::fs::create_dir(&outside_one).expect("create the escaped next level"); + std::fs::set_permissions(&outside_one, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::os::unix::fs::symlink(outside.path(), base.path().join("link")) + .expect("plant the absolute escaping symlink"); + + let key = base + .path() + .join("link") + .join("one") + .join("two") + .join("identity.pem"); + match load_or_create_keypair_at(&key) { + Ok(kp) => panic!( + "an absolute interior symlink was followed out of the base: the identity {} \ + was created outside {} through {}", + kp.did(), + base.path().display(), + key.display() + ), + Err(e) => assert_names_symlink_refusal( + &format!("{e:#}"), + "an interior symlink whose target is outside the base", + ), + } + assert_no_identity_key_under(outside.path(), "an escaping interior symlink"); + assert_identity_absent(&outside_one.join("two")); + } + + /// A chain, because refusing one link is not the same property as + /// refusing the path. `l1 -> l2 -> evil`, so the resolution that must be + /// refused takes two hops before it reaches the existing next level. + #[cfg(unix)] + #[test] + fn identity_chained_interior_symlinks_are_refused_not_followed() { + use std::os::unix::fs::PermissionsExt; + + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let evil = base.path().join("evil"); + std::fs::create_dir(&evil).unwrap(); + std::fs::set_permissions(&evil, std::fs::Permissions::from_mode(0o700)).unwrap(); + let one = evil.join("one"); + std::fs::create_dir(&one).unwrap(); + std::fs::set_permissions(&one, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::os::unix::fs::symlink("evil", base.path().join("l2")).expect("plant the second hop"); + std::os::unix::fs::symlink("l2", base.path().join("l1")).expect("plant the first hop"); + + let key = base + .path() + .join("l1") + .join("one") + .join("two") + .join("identity.pem"); + match load_or_create_keypair_at(&key) { + Ok(kp) => panic!( + "a two-link chain on the key path was followed, not refused: the identity {} \ + was created through {}", + kp.did(), + key.display() + ), + Err(e) => { + assert_names_symlink_refusal(&format!("{e:#}"), "a chain of two interior symlinks") + } + } + assert_identity_absent(&one.join("two")); + assert_no_identity_key_under(base.path(), "a chain of two interior symlinks"); + } + + /// The existing-named-parent fast path never enters the walk: it opens the + /// whole configured parent pathname in one call and publishes into + /// whatever that resolves to. `link/one` exists through the symlink, so + /// this row reaches the fast path and not the walk-up. + #[cfg(unix)] + #[test] + fn identity_existing_named_parent_reached_through_a_symlink_is_refused_not_followed() { + use std::os::unix::fs::PermissionsExt; + + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let evil = plant_interior_symlink_layout(base.path()); + + let key = base.path().join("link").join("one").join("identity.pem"); + match load_or_create_keypair_at(&key) { + Ok(kp) => panic!( + "the existing-named-parent fast path followed an interior symlink: the identity \ + {} was published into {} through {}", + kp.did(), + evil.join("one").display(), + key.display() + ), + Err(e) => assert_names_symlink_refusal( + &format!("{e:#}"), + "an interior symlink on the existing-named-parent fast path", + ), + } + assert_identity_absent(&evil.join("one").join("identity.pem")); + assert_no_identity_key_under(base.path(), "the existing-named-parent fast path"); + } + + /// Fixture: the relative-path row. `cwd` is process-global, so the row + /// that proves a relative configured path is judged the same way runs in + /// its own process. Double-gated like the other fixtures here: `#[ignore]` + /// keeps it out of a normal run and the env check keeps it inert under a + /// bare `--ignored` sweep, which would otherwise chdir the shared test + /// process. + #[cfg(unix)] + #[test] + #[ignore = "self-exec fixture: only runs under GITLAWB_TEST_FIXTURE=identity-symlink-relative"] + fn fixture_identity_relative_path_with_interior_symlink() { + if std::env::var("GITLAWB_TEST_FIXTURE").ok().as_deref() + != Some("identity-symlink-relative") + { + return; + } + let base = std::path::PathBuf::from( + std::env::var("GITLAWB_TEST_BASE").expect("GITLAWB_TEST_BASE"), + ); + let evil = plant_interior_symlink_layout(&base); + std::env::set_current_dir(&base).expect("chdir into the row's base"); + + let call_path = std::path::Path::new("link/one/two/identity.pem"); + match load_or_create_keypair_at(call_path) { + Ok(kp) => panic!( + "an interior symlink on a RELATIVE key path was followed, not refused: the \ + identity {} was created through {}", + kp.did(), + call_path.display() + ), + Err(e) => assert_names_symlink_refusal( + &format!("{e:#}"), + "an interior symlink on a relative configured path", + ), + } + assert_identity_absent(&evil.join("one").join("two")); + assert_no_identity_key_under(&base, "a relative path with an interior symlink"); + println!("identity-symlink-relative: refused"); + } + + /// Driver for the relative row. An absolute path is not the only shape an + /// operator configures, and the walk resolves a relative pathname against + /// the process cwd, so the same interior-symlink refusal has to hold there. + #[cfg(unix)] + #[test] + fn identity_relative_key_path_with_interior_symlink_is_refused_not_followed() { + use std::os::unix::fs::PermissionsExt; + + const REFUSED_SENTINEL: &str = "identity-symlink-relative: refused"; + + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + + let mut cmd = std::process::Command::new(std::env::current_exe().expect("current_exe")); + cmd.args([ + "identity_key_storage_tests::fixture_identity_relative_path_with_interior_symlink", + "--exact", + "--ignored", + "--nocapture", + ]) + .env("GITLAWB_TEST_FIXTURE", "identity-symlink-relative") + .env("GITLAWB_TEST_BASE", base.path()); + let output = cmd + .output() + .expect("spawn the relative-path symlink fixture"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + assert!( + output.status.success(), + "a relative key path with an interior symlink must be refused, not followed\n\ + --- stdout ---\n{stdout}\n--- stderr ---\n{stderr}" + ); + // A filter matching nothing exits 0, and the fixture's env gate returns + // early as a passing test, so neither alone is proof the row ran. + assert!( + stdout.contains("1 passed"), + "the filter must select one passing test\n{stdout}" + ); + assert!( + stdout.contains(REFUSED_SENTINEL), + "the fixture must print its refusal sentinel\n--- stdout ---\n{stdout}" + ); + } + + /// THE LOAD PATH, which is the identity-substitution primitive rather than + /// a permissions bug. The node's real key sits at `a/keys/identity.pem` + /// and the configured path is `link/keys/identity.pem`. An attacker who + /// can only repoint `link` from `a` to `b` makes the load resolve to a PEM + /// they supplied, and the node boots as THEIR DID while the real key is + /// still on disk, untouched, with nothing logged as wrong. + /// + /// The assertion is on the DID, not on a mode or an error string: a mode + /// assertion cannot see a substitution, because the substituted key is + /// a perfectly well-formed 0600 PEM. + #[cfg(unix)] + #[test] + fn identity_load_through_repointed_interior_symlink_must_not_adopt_the_substituted_key() { + use std::os::unix::fs::PermissionsExt; + + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + + let real_key = base.path().join("a").join("keys").join("identity.pem"); + let real_did = load_or_create_keypair_at(&real_key) + .expect("plant the node's real identity") + .did() + .to_string(); + let attacker_key = base.path().join("b").join("keys").join("identity.pem"); + let attacker_did = load_or_create_keypair_at(&attacker_key) + .expect("plant the attacker's identity") + .did() + .to_string(); + assert_ne!( + real_did, attacker_did, + "the row needs two distinct identities to tell substitution from a load" + ); + let real_bytes = std::fs::read(&real_key).expect("read the real key"); + + // The configured path never changes. Only the interior component does, + // which is exactly the authority an attacker with write access to one + // directory has. + let link = base.path().join("link"); + std::os::unix::fs::symlink("a", &link).expect("point the configured path at the real key"); + std::fs::remove_file(&link).expect("the attacker repoints the interior component"); + std::os::unix::fs::symlink("b", &link).expect("repoint the interior component"); + + let configured = link.join("keys").join("identity.pem"); + match load_or_create_keypair_at(&configured) { + Ok(kp) => { + let booted = kp.did().to_string(); + if booted == attacker_did { + panic!( + "identity substitution: repointing the interior symlink {} made the node \ + boot as the attacker's identity {attacker_did} (planted at {}) instead \ + of its own {real_did} at {}", + link.display(), + attacker_key.display(), + real_key.display() + ); + } + panic!( + "the load followed the interior symlink at {} and returned {booted} rather \ + than refusing the path", + link.display() + ); + } + Err(e) => assert_names_symlink_refusal( + &format!("{e:#}"), + "a repointed interior symlink on the load path", + ), + } + + assert_eq!( + std::fs::read(&real_key).expect("read the real key"), + real_bytes, + "the real key must be untouched by the refused boot" + ); + assert_eq!( + load_or_create_keypair_at(&real_key) + .expect("the real key must still load by its real path") + .did() + .to_string(), + real_did, + "the node's own identity must be unchanged" + ); + } + + /// The weaker variant of the same primitive. The attacker repoints the + /// interior component at an EMPTY directory, the load finds no key, and + /// the create path mints a fresh identity and logs it as a first boot. The + /// node comes up with a DID nobody knows and its real key is orphaned in + /// place with nothing reported. + #[cfg(unix)] + #[test] + fn identity_load_through_symlink_to_an_empty_directory_must_not_silently_mint_a_new_identity() { + use std::os::unix::fs::PermissionsExt; + + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + + let real_key = base.path().join("a").join("keys").join("identity.pem"); + let real_did = load_or_create_keypair_at(&real_key) + .expect("plant the node's real identity") + .did() + .to_string(); + let real_bytes = std::fs::read(&real_key).expect("read the real key"); + + let empty_keys = base.path().join("empty").join("keys"); + std::fs::create_dir_all(&empty_keys).expect("create the empty target tree"); + std::fs::set_permissions(&empty_keys, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::set_permissions( + base.path().join("empty"), + std::fs::Permissions::from_mode(0o700), + ) + .unwrap(); + let link = base.path().join("link"); + std::os::unix::fs::symlink("a", &link).expect("point the configured path at the real key"); + std::fs::remove_file(&link).expect("the attacker repoints the interior component"); + std::os::unix::fs::symlink("empty", &link).expect("repoint at an empty directory"); + + let configured = link.join("keys").join("identity.pem"); + match load_or_create_keypair_at(&configured) { + Ok(kp) => { + let booted = kp.did().to_string(); + panic!( + "silent identity loss: repointing the interior symlink {} at an empty \ + directory made the node mint a fresh identity {booted} as if this were a \ + first boot, orphaning its real identity {real_did} at {}", + link.display(), + real_key.display() + ); + } + Err(e) => assert_names_symlink_refusal( + &format!("{e:#}"), + "an interior symlink repointed at an empty directory", + ), + } + + assert_identity_absent(&empty_keys.join("identity.pem")); + assert_eq!( + std::fs::read(&real_key).expect("read the real key"), + real_bytes, + "the real key must be untouched by the refused boot" + ); + assert_eq!( + load_or_create_keypair_at(&real_key) + .expect("the real key must still load by its real path") + .did() + .to_string(), + real_did, + "the node's own identity must be unchanged" + ); + } + + /// Already correct, and here so it stays that way: a symlink at the key's + /// IMMEDIATE parent is the final component of the load's directory open, + /// which is the one position `O_NOFOLLOW` does bind. + #[cfg(unix)] + #[test] + fn identity_symlinked_immediate_parent_is_refused_at_load() { + use std::os::unix::fs::PermissionsExt; + + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let real = base.path().join("real"); + std::fs::create_dir(&real).unwrap(); + std::fs::set_permissions(&real, std::fs::Permissions::from_mode(0o700)).unwrap(); + let planted_did = load_or_create_keypair_at(&real.join("identity.pem")) + .expect("plant a key behind the symlink") + .did() + .to_string(); + std::os::unix::fs::symlink("real", base.path().join("link")).unwrap(); + + let configured = base.path().join("link").join("identity.pem"); + match load_or_create_keypair_at(&configured) { + Ok(kp) => panic!( + "a symlink at the key's immediate parent was followed: loaded {} (planted \ + {planted_did}) through {}", + kp.did(), + configured.display() + ), + Err(e) => assert_names_symlink_refusal( + &format!("{e:#}"), + "a symlink at the key's immediate parent", + ), + } + } + + /// Already correct, and here so it stays that way: a dangling symlink is + /// refused rather than replaced by a real directory of the same name, + /// both when it is the top component of the missing suffix and when it + /// sits under a real directory. + #[cfg(unix)] + #[test] + fn identity_dangling_symlink_on_the_key_path_is_refused_never_replaced() { + use std::os::unix::fs::PermissionsExt; + + for under_real_dir in [false, true] { + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let holder = if under_real_dir { + let real = base.path().join("real"); + std::fs::create_dir(&real).unwrap(); + std::fs::set_permissions(&real, std::fs::Permissions::from_mode(0o700)).unwrap(); + real + } else { + base.path().to_path_buf() + }; + let link = holder.join("link"); + std::os::unix::fs::symlink("nowhere", &link).expect("plant the dangling symlink"); + + let key = link.join("one").join("identity.pem"); + let where_ = if under_real_dir { + "a dangling symlink under a real directory" + } else { + "a dangling symlink at the top of the path" + }; + match load_or_create_keypair_at(&key) { + Ok(kp) => panic!( + "{where_} was not refused: the identity {} was created through {}", + kp.did(), + key.display() + ), + Err(e) => assert_names_symlink_refusal(&format!("{e:#}"), where_), + } + assert!( + std::fs::symlink_metadata(&link) + .expect("the dangling symlink must still be there") + .file_type() + .is_symlink(), + "{where_}: the refused boot replaced {} with something else", + link.display() + ); + assert_no_identity_key_under(base.path(), where_); + } + } + + /// 0111 grandparent cannot mkdir, so the key directory must already exist. + /// Opening that grandparent must not require directory-list permission. + #[cfg(all(unix, any(target_os = "linux", target_os = "android")))] + #[test] + fn identity_search_only_grandparent_publishes_into_existing_key_dir() { + use std::os::unix::fs::PermissionsExt; + + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let gp = base.path().join("searchonly"); + let keys = gp.join("keys"); + std::fs::create_dir_all(&keys).unwrap(); + std::fs::set_permissions(&keys, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::set_permissions(&gp, std::fs::Permissions::from_mode(0o111)).unwrap(); + let key = keys.join("identity.pem"); + let created = match load_or_create_keypair_at(&key) { + Ok(kp) => kp, + Err(e) => { + let _ = std::fs::set_permissions(&gp, std::fs::Permissions::from_mode(0o700)); + panic!("search-only grandparent must be enough to publish into an existing 0700 key dir: {e:#}"); + } + }; + let reloaded = load_or_create_keypair_at(&key); + std::fs::set_permissions(&gp, std::fs::Permissions::from_mode(0o700)).unwrap(); + let reloaded = reloaded.expect("reload"); + assert_eq!(created.did(), reloaded.did()); + } + + /// `/identity.pem` is a pathname contract, not a filesystem probe: its + /// parent is the root, which the operator has already nominated, so + /// `create_pinned_dir_and_publish` must take the already-nominated branch. + /// Asserted lexically because a probe against the real `/` cannot be + /// isolated and, run as root, leaves a real key at `/identity.pem`. + #[cfg(unix)] + #[test] + fn identity_root_adjacent_path_is_an_already_nominated_parent() { + use std::path::{Component, Path}; + + let root_adjacent = Path::new("/identity.pem"); + + assert!( + !p2p::path_denotes_a_directory(root_adjacent, None), + "the lexical gate in load_or_create_keypair_at must admit /identity.pem" + ); + assert!( + !root_adjacent + .components() + .any(|c| c == Component::ParentDir), + "/identity.pem walks back out through no `..`" + ); + assert_eq!( + p2p::key_parent(root_adjacent), + Path::new("/"), + "the lexical parent of /identity.pem is the filesystem root" + ); + assert!( + p2p::identity_parent_is_already_nominated(root_adjacent), + "root-adjacent identity path must take the already-nominated branch, not the named-parent branch that fails with 'names no final directory component'" + ); + + for nominated in ["identity.pem", "./identity.pem"] { + assert!( + p2p::identity_parent_is_already_nominated(Path::new(nominated)), + "{nominated} publishes into the working directory, which is already nominated" + ); + } + + for named in [ + "/data/identity.pem", + "keys/identity.pem", + "/identity/identity.pem", + ] { + assert!( + !p2p::identity_parent_is_already_nominated(Path::new(named)), + "{named} names a parent component this process may create" + ); + } + } + + /// A terminal `.` is a directory spelling. Path drops it, so this would + /// otherwise publish a 0600 file named `keys` under `data`. + #[cfg(unix)] + #[test] + fn identity_terminal_dot_path_is_refused_without_retargeting() { + use std::os::unix::fs::PermissionsExt; + + let base = tempfile::tempdir().unwrap(); + std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let data = base.path().join("data"); + std::fs::create_dir(&data).unwrap(); + std::fs::set_permissions(&data, std::fs::Permissions::from_mode(0o755)).unwrap(); + let spelling = format!("{}/keys/.", data.display()); + let key = std::path::Path::new(&spelling); + let Err(err) = load_or_create_keypair_at(key) else { + panic!("a terminal `.` identity path must be refused"); + }; + let text = format!("{err:#}"); + assert!( + text.contains("must name a key file") || text.contains("directory"), + "the refusal must name the directory spelling, got: {text}" + ); + let mode = std::fs::symlink_metadata(&data) + .unwrap() + .permissions() + .mode() + & 0o7777; + assert_eq!(mode, 0o755, "rejection must not chmod the parent"); + assert!( + !data.join("keys").exists(), + "rejection must not create a file named keys" + ); + } +} diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index a65ff4291..efd80ec4c 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -1,3985 +1,7 @@ -mod ans104; -mod api; -mod arweave; -mod arweave_v2; -mod auth; -mod bootstrap; -mod cert; -mod config; -mod db; -mod durable_outbox; -mod encrypted_pin; -mod error; -mod git; -mod graphql; -mod icaptcha; -mod ipfs_pin; -mod metrics; -mod operator; -mod p2p; -mod pinata; -mod rate_limit; -mod reconciliation; -mod server; -mod state; -mod sync; -#[cfg(test)] -mod test_support; -mod upstream_mirror; -mod visibility; -mod webhook_secret; -mod webhooks; - -use anyhow::{anyhow, Context, Result}; -use axum::extract::State; -use axum::http::StatusCode; -use axum::response::IntoResponse; -use axum::routing::get; -use axum::{Json, Router}; -use clap::Parser; -use std::collections::HashSet; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; -use tokio::net::TcpListener; -use tokio::sync::watch; -use tracing::{error, info, warn}; - -use gitlawb_core::http_sig::sign_request; -use gitlawb_core::identity::Keypair; - -use config::Config; -use db::Db; -use state::AppState; - -#[derive(Clone)] -struct DegradedState { - node_did: String, - db_startup: Arc, - web_url: Option, -} - -/// Two independent counters with no cross-field invariant — atomics, not a -/// lock, so the retry loop and the degraded handlers never contend. -#[derive(Default)] -struct DbStartupStatus { - attempts: AtomicU64, - next_retry_secs: AtomicU64, -} +//! Thin binary entry point. All boot logic lives in the library crate +//! (`src/lib.rs`) so out-of-crate integration tests can link the same code. #[tokio::main] -async fn main() -> Result<()> { - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::from_default_env() - .add_directive("gitlawb_node=debug".parse().unwrap()) - .add_directive("tower_http=info".parse().unwrap()), - ) - .init(); - - let mut config = Config::parse(); - - // Merge the embedded seed list of public network nodes into the runtime - // bootstrap peers. Operators can opt out via GITLAWB_BOOTSTRAP_DISABLE_SEEDS. - bootstrap::merge_seeds(&mut config); - - // Fail fast on config combinations that are individually in-range but jointly - // unsafe — notably a DB pool too small for the concurrent-write cap, which - // would let a push burst starve every other DB path (#174 F1). - config - .validate() - .map_err(|e| anyhow::anyhow!("invalid configuration: {e}"))?; - - if !config.public_read { - warn!( - "GITLAWB_PUBLIC_READ=false has no effect; reads are gated per repository by is_public and path-scoped visibility rules, not this flag" - ); - } - - // Load or generate the node's identity keypair - let keypair = Arc::new(load_or_create_keypair(&config)?); - // Sealing key for the legacy-scan continuation tokens, DERIVED from the identity - // just loaded so it is the same key after a restart (see `derive_scan_token_key`). - let scan_token_key = AppState::derive_scan_token_key(&keypair); - let webhook_secret_key = AppState::derive_webhook_secret_key(&keypair); - let node_did = keypair.did(); - - // One-time metrics init. Must run before any handler that calls into - // `metrics::record_*` so the registry exists when the first event fires. - // Safe to call even when GITLAWB_METRICS_ADDR is unset — those helpers - // are simply no-ops until something reads from the registry. - metrics::init(env!("CARGO_PKG_VERSION"), &node_did.to_string()); - - info!("╔══════════════════════════════════════════╗"); - info!( - "║ Twigpine node v{} ║", - env!("CARGO_PKG_VERSION") - ); - info!("╚══════════════════════════════════════════╝"); - // Process-wide shutdown signal. One sender lives in AppState (cloned - // into every handler); main() keeps a clone and flips it on SIGINT - // or SIGTERM. Tasks that hold a watch::Receiver get notified at - // their next await point. - let (shutdown_tx, _shutdown_rx_for_main) = watch::channel(false); - spawn_shutdown_signal(shutdown_tx.clone()); - - info!(did = %node_did, "node identity"); - info!(addr = %config.bind_addr(), "binding HTTP listener"); - - // Bind HTTP once, before dependency initialization, and keep this socket - // for the life of the process. The degraded server accepts on a dup of the - // same socket, so the degraded→full handoff never closes the port: while - // the full server initializes, connections queue in the shared backlog - // instead of being refused. - let listener = TcpListener::bind(config.bind_addr()) - .await - .with_context(|| format!("failed to bind to {}", config.bind_addr()))?; - let full_std_listener = listener.into_std()?; - let degraded_listener = TcpListener::from_std( - full_std_listener - .try_clone() - .context("failed to clone HTTP listener for degraded server")?, - )?; - - // Metrics must stay observable during a database outage — the degraded - // window is exactly when dashboards need data — so this listener starts - // before the DB connects. - let metrics_handle = if !config.metrics_addr.is_empty() { - match spawn_metrics_server(&config.metrics_addr, shutdown_tx.subscribe()).await { - Ok(handle) => { - info!(addr = %config.metrics_addr, "metrics endpoint listening"); - Some(handle) - } - Err(e) => { - warn!(err = %e, addr = %config.metrics_addr, "failed to start metrics endpoint — continuing without"); - None - } - } - } else { - info!("metrics endpoint disabled (GITLAWB_METRICS_ADDR not set)"); - None - }; - - let db_startup = Arc::new(DbStartupStatus::default()); - let (db_ready_tx, db_ready_rx) = watch::channel(false); - let mut degraded_handle = tokio::spawn(run_degraded_server( - degraded_listener, - node_did.to_string(), - Arc::clone(&db_startup), - config.web_url.clone(), - db_ready_rx, - shutdown_tx.subscribe(), - )); - - // Resolve the p2p identity BEFORE the database connect, both arms of the - // port gate together. - // - // The key load is pure filesystem work with no database dependency, and - // its failure is deliberately non-fatal. Leaving it behind the DB connect - // meant the "HTTP up, p2p off" outcome and the port-zero no-IO guarantee - // were both unobservable whenever the database was unreachable, because - // `connect_db_with_retry` retries indefinitely and never falls through. - // Moving only the load would have left the disabled arm stranded, so the - // whole gate moves and just `p2p::start` stays behind the database. - let p2p_local_key = if config.p2p_port > 0 { - match p2p::load_or_create_p2p_keypair(&config.resolved_p2p_key_path()) { - Ok(local_key) => { - metrics::set_p2p_key_load_failed(false); - Some(local_key) - } - // Non-fatal by policy, and the cost is named rather than hidden: - // the node keeps serving HTTP with a green /health while it is off - // the p2p network entirely. Logged at error with a stable event - // name and mirrored into a metric, so the outage is alertable - // without reading startup logs by hand. - Err(e) => { - error!( - err = %format!("{e:#}"), - event = "p2p_identity_key_load_failed", - "failed to load p2p identity key, continuing without p2p" - ); - metrics::set_p2p_key_load_failed(true); - None - } - } - } else { - info!("p2p disabled (p2p_port = 0)"); - metrics::set_p2p_key_load_failed(false); - None - }; - - // Connect to PostgreSQL database. A transient outage or bad secret should - // not crash-loop the process and hammer the database provider; permanent - // misconfiguration surfaces through error-level logs and the /ready check. - let db = tokio::select! { - db = connect_db_with_retry(&config, Arc::clone(&db_startup), shutdown_tx.subscribe()) => { - match db { - Some(db) => db, - None => { - // Shutdown requested while waiting for the database. The - // degraded server only serves one-shot 503s — abort it - // rather than drain, so a slow client can't stall exit. - degraded_handle.abort(); - return Ok(()); - } - } - } - degraded = &mut degraded_handle => { - if *shutdown_tx.borrow() { - return Ok(()); - } - return match degraded { - Ok(Ok(())) => Err(anyhow!("degraded HTTP server stopped before database became ready")), - Ok(Err(err)) => Err(err.context("degraded HTTP server failed")), - Err(err) => Err(anyhow!("degraded HTTP server task failed: {err}")), - }; - } - }; - - // Flip the degraded server into graceful shutdown, but do NOT await the - // drain: one slow in-flight request must not delay the full server, and - // the shared socket means there is no port gap to cover. The drain - // finishes (and logs) in the background. - db_ready_tx.send(true).ok(); - tokio::spawn(async move { - match degraded_handle.await { - Ok(Ok(())) => {} - Ok(Err(err)) => warn!(err = %err, "degraded HTTP server exited with error"), - Err(err) => warn!(err = %err, "degraded HTTP server task failed"), - } - }); - info!(addr = %config.bind_addr(), "database ready; starting full HTTP server"); - - // Prune peer rows that point back at this node (stale self-loop entries) - if let Some(public_url) = config.public_url.as_deref() { - match db.prune_self_peers(public_url).await { - Ok(0) => {} - Ok(n) => info!(removed = n, public_url, "pruned self-loop peer rows"), - Err(e) => warn!(err = %e, "prune_self_peers failed (non-fatal)"), - } - } - - // Prune peer rows with non-public hosts (loopback/private/internal) that - // were injected via the unauthenticated announce route — they poison the - // sync-notify fan-out (SSRF + crowding out real peers). - match db.prune_non_public_peers().await { - Ok(0) => {} - Ok(n) => info!(removed = n, "pruned non-public (poisoned) peer rows"), - Err(e) => warn!(err = %e, "prune_non_public_peers failed (non-fatal)"), - } - - // Ensure repos directory exists - std::fs::create_dir_all(&config.repos_dir).context("failed to create repos directory")?; - - // The identity was resolved before the database connect; this is only the - // swarm start, which genuinely needs the database handle. - let p2p_handle = match p2p_local_key { - Some(local_key) => { - let bootstrap_addrs = config - .p2p_bootstrap - .iter() - .filter_map(|s| s.parse().ok()) - .collect(); - let shutdown_rx = shutdown_tx.subscribe(); - match p2p::start( - local_key, - config.p2p_port, - bootstrap_addrs, - Arc::clone(&db), - config.auto_sync, - shutdown_rx, - Arc::clone(&keypair), - config.require_signed_peer_writes, - ) - .await - { - Ok(handle) => { - info!(port = config.p2p_port, peer_id = %handle.local_peer_id, "libp2p swarm started"); - Some(Arc::new(handle)) - } - Err(e) => { - tracing::warn!(err = %e, "failed to start libp2p swarm — continuing without p2p"); - None - } - } - } - None => None, - }; - - // Shared no-redirect HTTP client. See build_http_client for the SSRF rationale. - let http_client = Arc::new(build_http_client()?); - - let (ref_update_tx, _) = tokio::sync::broadcast::channel::(256); - let (task_event_tx, _) = tokio::sync::broadcast::channel::(256); - - // Node-keyed, so continuation tokens this node mints are only accepted by - // this node and survive its restarts without any configured secret. - let task_cursor_key = api::task_cursor::TaskCursorKey::derive(&keypair.to_seed()); - - let graphql_schema = Arc::new(graphql::build_schema( - Arc::clone(&db), - ref_update_tx.clone(), - task_event_tx.clone(), - task_cursor_key.clone(), - )); - - let machine_id = std::env::var("FLY_MACHINE_ID").ok(); - if let Some(ref mid) = machine_id { - info!(" fly machine: {mid}"); - } - - // Initialize Tigris S3 client if bucket is configured - let tigris = if !config.tigris_bucket.is_empty() { - match git::tigris::TigrisClient::new(&config.tigris_bucket).await { - Ok(client) => { - info!(bucket = %config.tigris_bucket, "tigris storage enabled"); - Some(client) - } - Err(e) => { - tracing::warn!(err = %e, "failed to initialize Tigris client — using local-only storage"); - None - } - } - } else { - info!("tigris storage disabled (no bucket configured)"); - None - }; - - // Advisory-lock connections come from their own pool: a write guard pins one - // for its whole lifetime, and sharing the application pool would let a push - // burst starve ordinary request handlers. - let lock_pool = db::Db::lock_pool( - &config.database_url, - config.db_lock_pool_max_connections, - std::time::Duration::from_secs(config.db_acquire_timeout_secs), - )?; - let repo_store = git::repo_store::RepoStore::new( - config.repos_dir.clone(), - tigris, - lock_pool, - std::time::Duration::from_secs(config.lock_held_transfer_timeout_secs), - ); - - // Per-DID limiter for the creation endpoints. Keyed on the authenticated - // DID (attacker-varied), so bound its key set to cap memory. Max and window - // are operator-configurable via GITLAWB_RATE_LIMIT_MAX / - // GITLAWB_RATE_LIMIT_WINDOW_SECS; both reject 0 at parse time. - let rate_limiter = rate_limit::RateLimiter::new_bounded( - config.rate_limit_max_requests as usize, - std::time::Duration::from_secs(config.rate_limit_window_secs), - 200_000, - ); - - // Per-client-IP flood brake for the creation endpoints. The per-DID limiter - // above is bypassed by a DID farm (one throwaway did:key per repo), which is - // exactly how the recurring spam-repo floods get past both it and the - // iCaptcha gate. Keyed on the resolved client IP so a single-source flood is - // capped regardless of how many identities it mints. Sized well above any - // legitimate per-IP creation rate; GITLAWB_CREATE_RATE_LIMIT overrides, 0 - // disables. Bounded key set — the key is a client-influenced IP. - let create_limit = std::env::var("GITLAWB_CREATE_RATE_LIMIT") - .ok() - .and_then(|v| v.trim().parse::().ok()) - .unwrap_or(120); - let create_ip_rate_limiter = rate_limit::RateLimiter::new_bounded( - create_limit, - std::time::Duration::from_secs(3600), - 200_000, - ); - if create_limit == 0 { - tracing::warn!("GITLAWB_CREATE_RATE_LIMIT=0 — per-IP creation rate limiting disabled"); - } - - // Push-path flood brake: max git-receive-pack requests per client IP per - // hour (counts both the info/refs advertisement and the push POST). Sized - // for heavy agent automation while still stopping flood traffic (the June - // 2026 attack pushed several times per second per IP). GITLAWB_PUSH_RATE_LIMIT - // overrides; 0 disables. Bounded key set — the key is a client-influenced IP. - let push_limit = std::env::var("GITLAWB_PUSH_RATE_LIMIT") - .ok() - .and_then(|v| v.trim().parse::().ok()) - .unwrap_or(600); - let push_rate_limiter = rate_limit::RateLimiter::new_bounded( - push_limit, - std::time::Duration::from_secs(3600), - 200_000, - ); - if push_limit == 0 { - tracing::warn!("GITLAWB_PUSH_RATE_LIMIT=0 — per-IP push rate limiting disabled"); - } - - // close_issue drives a full archive download for non-owner author pre-checks. - // Keep it on its own bucket so it cannot drain the receive-pack limit. - let close_issue_limit = std::env::var("GITLAWB_CLOSE_ISSUE_RATE_LIMIT") - .ok() - .and_then(|v| v.trim().parse::().ok()) - .unwrap_or(120); - let close_issue_rate_limiter = rate_limit::RateLimiter::new_bounded( - close_issue_limit, - std::time::Duration::from_secs(3600), - 200_000, - ); - if close_issue_limit == 0 { - tracing::warn!( - "GITLAWB_CLOSE_ISSUE_RATE_LIMIT=0 — per-IP close_issue rate limiting disabled" - ); - } - - // Which forwarded header the edge is trusted to set. Default None (trust - // nothing, key on the socket peer). Fly nodes set GITLAWB_TRUSTED_PROXY=fly; - // a node behind Caddy/NGINX sets it to x-forwarded-for. - let push_limiter_trust = rate_limit::TrustedProxy::from_env_value( - &std::env::var("GITLAWB_TRUSTED_PROXY").unwrap_or_default(), - ); - tracing::info!(trust = ?push_limiter_trust, push_limit, "push rate limiter configured"); - - // Peer-sync flood brakes, keyed on the resolved client IP (per-DID is useless - // here — a did:key farm self-registers). Two buckets so an unsigned notify - // flood can't drain the signed trigger caller's quota (#82). Bounded key sets - // (the key is a client-influenced IP); 0 disables each. - let sync_trigger_rate_limiter = rate_limit::RateLimiter::new_bounded( - config.sync_trigger_rate_limit, - std::time::Duration::from_secs(3600), - 200_000, - ); - let peer_write_rate_limiter = rate_limit::RateLimiter::new_bounded( - config.peer_write_rate_limit, - std::time::Duration::from_secs(3600), - 200_000, - ); - let federated_rate_limiter = - rate_limit::RateLimiter::new_bounded(12, std::time::Duration::from_secs(60), 10_000); - if config.sync_trigger_rate_limit == 0 { - tracing::warn!( - "GITLAWB_SYNC_TRIGGER_RATE_LIMIT=0 — /sync/trigger IP rate limiting disabled" - ); - } - if config.peer_write_rate_limit == 0 { - tracing::warn!("GITLAWB_PEER_WRITE_RATE_LIMIT=0 — peer-write IP rate limiting disabled"); - } - - // Initialize the iCaptcha proof gate (inert unless ICAPTCHA_MODE is set). - icaptcha::init().await; - - let state = AppState { - config: Arc::new(config.clone()), - db, - node_did: node_did.clone(), - node_keypair: keypair, - task_cursor_key, - p2p: p2p_handle, - http_client, - ref_update_tx, - task_event_tx, - graphql_schema, - machine_id, - repo_store, - rate_limiter, - create_ip_rate_limiter, - push_rate_limiter, - close_issue_rate_limiter, - ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, - ipfs_max_legacy_probes: AppState::ipfs_legacy_probe_budget(&config), - ipfs_legacy_scan_page_rows: crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, - ipfs_max_legacy_scan_rows: AppState::ipfs_legacy_scan_row_budget(&config), - ipfs_max_legacy_scan_rule_bytes: crate::api::ipfs::MAX_LEGACY_SCAN_RULE_BYTES_PER_REQUEST, - ipfs_scan_token_key: Arc::new(scan_token_key), - webhook_secret_key: Arc::new(webhook_secret_key), - ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, - push_limiter_trust, - sync_trigger_rate_limiter, - peer_write_rate_limiter, - federated_rate_limiter, - shutdown_tx: shutdown_tx.clone(), - git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), - git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( - config.max_concurrent_git_pushes, - )), - // Anon receive-pack advertisements get their OWN pool, same size as the - // write pool but disjoint, so filling it (which takes many source IPs, each - // capped by git_push_advert_per_caller) never occupies a permit the - // authenticated POST needs (#174). - git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new( - config.max_concurrent_git_pushes, - )), - // Bounds concurrent detached post-push encryption walks, sized from the push - // pool (no separate knob — Q1): completed pushes cannot outnumber active - // encryption walks past this (#174 P1-e). - git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new( - config.max_concurrent_git_pushes, - )), - // Bounds how many post-push pin loops run concurrently across all repos (#174 F6), - // independent of the per-repo encrypt-task coalescing below. Not a bound on the - // MB-scale object-id lists themselves: parked tasks still hold theirs (see the - // field doc on AppState::pin_semaphore). - pin_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_pin_tasks)), - // Coalesces the DETACHED post-push encryption tasks per repo so a rapid pusher - // cannot grow the outstanding parked-waiter set past one task per repo (#174 - // P2-2). No knob: it is a natural cap (one entry per distinct repo), not a - // sized pool. - encrypt_inflight: crate::state::EncryptInflight::new(), - // Per-repo in-process write-lease serializer (#174 U2/F3): supplements the pg - // advisory lock so a disconnected push's still-reaping git group can't be raced - // by a second same-node push. The map is naturally capped (one entry per contended - // repo, freed when unreferenced); the sized knob is how many pushes may PARK on - // one repo, since each parked push holds a fully buffered pack. - repo_write_leases: crate::state::RepoWriteLeases::new(config.repo_lease_max_waiters), - git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( - config.max_concurrent_reads_per_caller, - ), - // Per-source cap on the receive-pack advertisement, sized to an eighth of the - // write pool (min 1): one resolved client key (rate_limit::client_key) can hold - // at most this many slots in the DEDICATED advert pool (git_push_advert_semaphore, - // disjoint from the write pool), so saturating that pool takes ~8 distinct keys - // (#174). That bounds an IPv4 or single-address caller; a caller controlling many - // addresses (an IPv6 /64 is 2^64 keys) still gets one cap per address, since - // client_key uses the full IP with no prefix folding. Narrowing the keying is a - // deferred design call, not something these caps claim to solve. Sized off the - // write pool only because the advert pool is created at the same size; an advert - // flood cannot touch a write permit. - git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( - rate_limit::per_source_push_cap(config.max_concurrent_git_pushes), - ), - // Per-source cap on the authenticated receive-pack POST, sized like the advert - // cap: one resolved client key can hold at most this many write-pool slots, so - // monopolizing the pool takes ~8 distinct keys (#174 P1-d). Same residual as - // above: keys are full IPs, so a caller with many addresses has many caps. - git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( - rate_limit::per_source_push_cap(config.max_concurrent_git_pushes), - ), - // Bounds concurrent /ipfs visibility walks — a distinct public cost center, so - // its own pool + per-source sub-cap + per-IP rate limiter, never a git pool - // (#174 P1-3). The per-source map is bounded (reject-before-insert, INV-15). - git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new( - config.max_concurrent_ipfs_walks, - )), - git_ipfs_walk_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( - config.ipfs_walk_per_source, - ), - ipfs_rate_limiter: rate_limit::RateLimiter::new_bounded( - config.ipfs_rate_limit, - std::time::Duration::from_secs(3600), - 200_000, - ), - ipfs_work_rate_limiter: rate_limit::RateLimiter::new_bounded( - AppState::ipfs_work_budget(&config), - std::time::Duration::from_secs(3600), - 200_000, - ), - task_read_rate_limiter: rate_limit::RateLimiter::new_bounded( - config.task_read_rate_limit, - std::time::Duration::from_secs(3600), - 200_000, - ), - git_bin: "git".to_string(), - }; - - // One-time re-encryption of any webhook HMAC secrets still resting as - // plaintext (written before at-rest encryption shipped). A SQL migration - // cannot do it — Postgres never sees the node-held key — so it runs here. - match state - .db - .reencrypt_legacy_webhook_secrets(state.webhook_secret_key.as_ref()) - .await - { - Ok(0) => {} - Ok(n) => info!( - count = n, - "re-encrypted legacy webhook HMAC secrets at rest" - ), - Err(e) => warn!(err = %e, "failed to re-encrypt legacy webhook secrets"), - } - - if config.ipfs_rate_limit == 0 { - tracing::warn!("GITLAWB_IPFS_RATE_LIMIT=0 — per-IP /ipfs rate limiting disabled"); - } - if config.task_read_rate_limit == 0 { - tracing::warn!("GITLAWB_TASK_READ_RATE_LIMIT=0 — per-IP task read rate limiting disabled"); - } - - // Periodic peer-count poll for the metrics gauge. If p2p is disabled - // we still set the gauge to 0 so dashboards don't show "no data". - { - let p2p_for_metrics = state.p2p.clone(); - let mut shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - let mut interval = tokio::time::interval(std::time::Duration::from_secs(15)); - loop { - tokio::select! { - _ = interval.tick() => { - let count = match &p2p_for_metrics { - Some(h) => h.status().await.map(|s| s.connected_peers).unwrap_or(0), - None => 0, - }; - metrics::set_peers_connected(count as i64); - } - _ = shutdown_rx.changed() => { - if *shutdown_rx.borrow() { - return; - } - } - } - } - }); - } - - // Periodic cleanup of expired rate limit entries + consumed-proof ledger - { - let sweep_state = state.clone(); - let db = state.db.clone(); - let mut shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - loop { - tokio::select! { - _ = tokio::time::sleep(std::time::Duration::from_secs(300)) => { - sweep_state.sweep_rate_limiters().await; - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs() as i64) - .unwrap_or(0); - if let Err(e) = db.sweep_expired_proofs(now).await { - tracing::warn!(err = %e, "failed to sweep expired iCaptcha proofs"); - } - } - _ = shutdown_rx.changed() => { - if *shutdown_rx.borrow() { - break; - } - } - } - } - }); - } - - let _legacy_cid_sweep = spawn_legacy_cid_sweep(&state, &config); - let _queue_lifecycle_sweep = spawn_queue_lifecycle_sweep(&state, &config); - // NOTE: the due-request worker is spawned AFTER the one-shot startup - // reconcile/drain below, not here: its 5s interval would otherwise tick - // during operator/PoS setup and interleave with boot recovery on backlog - // rows (state gates prevent data loss, but the interleave costs recovery - // latency and widens the reconcile/drain race window). - let _marker_cleanup_worker = spawn_marker_cleanup_worker(&state); - - let router = server::build_router(state.clone()); - // Re-register the socket bound at startup — same fd, so there was never a - // moment with the port closed between the degraded and full servers. - let listener = TcpListener::from_std(full_std_listener) - .context("failed to re-register HTTP listener with the runtime")?; - - info!("✓ node started — did:{}", node_did); - info!(" repos dir: {}", config.repos_dir.display()); - info!( - " database: PostgreSQL ({})", - &config.database_url.split('@').next_back().unwrap_or("?") - ); - - // Publish our DID record to the Kademlia DHT shortly after startup - if let Some(p2p) = &state.p2p { - let did_record = p2p::DidRecord { - did: node_did.to_string(), - http_url: config.public_url.clone().unwrap_or_default(), - peer_id: p2p.local_peer_id.to_string(), - p2p_port: config.p2p_port, - timestamp: chrono::Utc::now().to_rfc3339(), - }; - let p2p_clone = Arc::clone(p2p); - let mut shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - // Small delay so Kademlia can find peers first - tokio::select! { - _ = tokio::time::sleep(std::time::Duration::from_secs(5)) => {} - _ = shutdown_rx.changed() => return, - } - p2p_clone.put_did(did_record).await; - info!("DID record published to Kademlia DHT"); - }); - } - - // Spawn background gossip: announce to bootstrap peers, then ping known peers periodically - { - let gossip_state = state.clone(); - let bootstrap_peers = config.bootstrap_peers.clone(); - let shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - gossip_task(gossip_state, bootstrap_peers, shutdown_rx).await; - }); - } - - // Start multi-node sync worker if auto_sync is enabled - if config.auto_sync { - sync::start( - Arc::clone(&state.db), - Arc::clone(&state.config), - Arc::clone(&state.node_keypair), - state.subscribe_shutdown(), - ); - info!("auto-sync worker started"); - } - - // External-forge mirroring is a separate, default-off worker. Startup - // validation already requires owner-only pushes when it is enabled; the - // worker additionally verifies the Git runtime can pin DNS answers before - // any URL reaches `git fetch`. - if config.upstream_mirror_enabled { - upstream_mirror::start( - Arc::clone(&state.db), - Arc::clone(&state.config), - state.repo_store.clone(), - state.repo_write_leases.clone(), - Arc::clone(&state.git_write_semaphore), - state.subscribe_shutdown(), - ) - .context("starting upstream mirror worker")?; - } - - // Periodic reconciliation sweep: re-derives pin/seal sets and fills gaps - // so a dropped replication job never means data loss. - { - let db = state.db.clone(); - let config = Arc::clone(&state.config); - let http_client = Arc::clone(&state.http_client); - let node_keypair = Arc::clone(&state.node_keypair); - let node_did = state.node_did.clone(); - let pin_sem = Arc::clone(&state.pin_semaphore); - let shutdown_rx = state.subscribe_shutdown(); - if reconciliation::spawn( - db, - config, - http_client, - node_keypair, - node_did, - pin_sem, - shutdown_rx, - Some(state.repo_store.clone()), - ) { - info!("reconciliation sweep worker started"); - } - } - - // On-chain operator setup: verify stake + spawn heartbeat loop - if !state.config.contract_node_staking.is_empty() - && !state.config.operator_private_key.is_empty() - { - match build_operator_client(&state.config, &state.node_did.to_string()) { - Ok(client) => match operator::startup_check(&client).await { - Ok(_) => { - let arc_client = Arc::new(client); - arc_client.spawn_heartbeat_loop(state.subscribe_shutdown()); - } - Err(e) => { - if state.config.operator_strict_mode { - return Err(e.context("strict-mode operator check failed")); - } - tracing::warn!(err = %e, "operator startup check failed — continuing without heartbeat loop"); - } - }, - Err(e) => { - if state.config.operator_strict_mode { - return Err(e.context("strict-mode: failed to build operator client")); - } - tracing::warn!(err = %e, "operator client could not be built — continuing without PoS"); - } - } - } else { - info!("on-chain PoS disabled (GITLAWB_CONTRACT_NODE_STAKING or GITLAWB_OPERATOR_PRIVATE_KEY unset)"); - } - - // axum's `with_graceful_shutdown` waits for in-flight requests to - // complete (up to the configured grace) once the future resolves. - let shutdown_signal_for_axum = state.subscribe_shutdown(); - let grace = std::time::Duration::from_secs(config.shutdown_grace_secs); - info!(grace_secs = config.shutdown_grace_secs, "axum server ready"); - - // #26 Split PR 1: drain any `applied` rows left by a previous - // process that crashed after Git applied a ref but before the - // bookkeeping landed. Runs once, BEFORE the server accepts new - // pushes, so a recovery re-derivation does not race a fresh push. - // Non-fatal: a transient drain failure is logged and the rows - // remain `applied` for the next startup to pick up. - // - // P1-A: the reconcile step runs FIRST and promotes any `prepared` - // or `uncertain` row whose target SHA actually landed on disk. - // This is the path that recovers a ref when the post-receive - // `mark_pending_ref_transitions_applied` call errored or was - // interrupted after `receive_pack` returned Ok, or when - // receive-pack returned Err but some refs may have landed - // (the `uncertain` state). Without this step, the drain (gated - // on `state = 'applied'`) would never see those rows. - // - // P2 (reviewer-1/2 round 3): use the multi-pass reconcile so - // prepared/uncertain rows beyond the first 1000-row page are - // processed in the same startup, rather than waiting for the - // next restart (where they might age out of MAX_RECONCILE_AGE). - match durable_outbox::reconcile_prepared_from_disk_all( - state.clone(), - durable_outbox::DRAIN_PER_PASS_LIMIT, - durable_outbox::DRAIN_MAX_PASSES, - ) - .await - { - Ok(0) => {} - Ok(n) => info!( - n, - "reconciled prepared/uncertain -> applied via on-disk ref match" - ), - Err(e) => warn!( - err = %e, - "pending ref transition reconcile failed at startup (non-fatal; will retry on next start)" - ), - } - match durable_outbox::drain_receive_pack_requests_all( - state.clone(), - durable_outbox::DRAIN_PER_PASS_LIMIT, - durable_outbox::DRAIN_MAX_PASSES, - ) - .await - { - Ok(0) => {} - Ok(n) => info!(n, "drained pending ref transitions from prior run"), - Err(e) => warn!( - err = %e, - "pending ref transition drain failed at startup (non-fatal; will retry on next start)" - ), - } - - // Start the due-request worker only now that boot recovery is done (see - // the note at the other spawns above): its first tick must not interleave - // with the one-shot reconcile/drain on backlog rows. - let _due_request_worker = spawn_due_request_worker(&state); - - // `into_make_service_with_connect_info` exposes the socket peer address as - // `ConnectInfo` so the push limiter can key on the real client - // when no trusted proxy header applies (see `rate_limit::client_key`). - let serve_result = axum::serve( - listener, - router.into_make_service_with_connect_info::(), - ) - .with_graceful_shutdown(async move { - let mut rx = shutdown_signal_for_axum; - // Wait until the watcher flips to true, then return so axum - // can begin draining. - while !*rx.borrow_and_update() { - if rx.changed().await.is_err() { - // Sender dropped — treat as shutdown. - break; - } - } - }) - .await; - - // Server has stopped accepting new connections and drained in-flight - // requests. Tear the rest of the system down. - info!("HTTP server stopped, beginning process shutdown"); - if let Some(h) = metrics_handle { - h.abort(); - } - let _ = grace; // recorded for operators in the log above; not enforced - serve_result?; - info!("clean exit"); - Ok(()) -} - -/// U4 (#173): spawn the periodic legacy provider-CID repair sweep. Releases before this -/// version stored the PROVIDER CID (Kubo dag-pb / Pinata CIDv0) in `pinned_cids.cid`, -/// and this version's `/ipfs/{cid}` resolver withholds any row whose stored key is not -/// the raw-content CID. The opportunistic repair on the pin path only fires when a push -/// re-carries the object, which normal git negotiation makes it not do, so those rows -/// need a walk. DETACHED, never on the boot path: the caller keeps serving while this -/// runs, and the sweep's own batch bound plus inter-batch delay keep it off the DB's -/// critical path. Its cursor is durable, so a restart mid-walk resumes instead of -/// rewinding. It re-arms after every run, including a failed one, so it never returns -/// and there is no awaited value to log here; the shutdown watcher below is what ends -/// it. -/// -/// A named function rather than an inline block in `main` so the WIRING has a seam a -/// test can call: that the task is spawned at all, that it reads its batch and delay -/// from the config knobs rather than some other field, that the caller is not blocked -/// on it, and that the shutdown watcher actually ends it mid-walk. The sweep's own -/// behavior is covered elsewhere; this is the boot-path half. -fn spawn_legacy_cid_sweep(state: &AppState, config: &Config) -> tokio::task::JoinHandle<()> { - let db = state.db.clone(); - let repos_dir = config.repos_dir.clone(); - let git_bin = state.git_bin.clone(); - let git_timeout = std::time::Duration::from_secs(config.git_service_timeout_secs); - let batch = config.pin_repair_sweep_batch; - let delay = std::time::Duration::from_secs(config.pin_repair_sweep_delay_secs); - let mut shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - tokio::select! { - _ = ipfs_pin::run_sweep_rearmed( - &repos_dir, &git_bin, git_timeout, batch, delay, - ipfs_pin::SWEEP_REARM_DELAY, &db, - ) => {} - // Shutdown mid-walk simply drops the run; the persisted cursor means the - // next boot picks up where this one stopped. - _ = shutdown_rx.changed() => {} - } - }) -} - -/// #26 Split PR 1 step 4 — periodic queue-lifecycle purge. Runs on -/// the same detached task pattern as `spawn_legacy_cid_sweep`: -/// tokio::spawn with a shutdown watcher, never on the boot path. The -/// interval is fixed at 24 hours (the spec calls for "one per -/// cluster per day"); the inter-batch delay is implicit in the -/// batch size plus the wall-clock cost of each pass. The drain -/// (`drain_receive_pack_requests_all`) and the purge -/// (`purge_request_queue`) share the same `DRAIN_PER_PASS_LIMIT` -/// budget so a 1000-row purge pass takes roughly the same time as -/// a 1000-row drain pass. -fn spawn_queue_lifecycle_sweep(state: &AppState, config: &Config) -> tokio::task::JoinHandle<()> { - let app_state = state.clone(); - let retention_days = config.queue_retention_days; - let batch = config.queue_purge_batch; - let git_timeout = std::time::Duration::from_secs(config.git_service_timeout_secs.min(30)); - let mut shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - let mut interval = tokio::time::interval(std::time::Duration::from_secs(24 * 3600)); - interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - // Consume the immediate first tick: Tokio intervals fire - // immediately on creation, which would run the purge - // concurrently with startup reconcile/drain. The child purge - // is terminal-parent-gated in SQL, but delaying the first - // sweep a full period removes the race entirely. - interval.tick().await; - loop { - tokio::select! { - _ = interval.tick() => { - if let Err(e) = durable_outbox::purge_request_queue( - &app_state.db, - retention_days, - batch, - ).await { - tracing::warn!(err = %e, "queue lifecycle purge failed; will retry on next tick"); - } - // Drain marker tombstones with the bounded deleter; - // failures retain the tombstone for the next tick. - if let Err(e) = durable_outbox::drain_marker_cleanup_queue( - &app_state, - batch, - git_timeout, - ).await { - tracing::warn!(err = %e, "marker cleanup drain failed"); - } - } - _ = shutdown_rx.changed() => { - break; - } - } - } - }) -} - -/// Shutdown-aware background due-request loop. The startup drain runs -/// once; without this, `effects_pending` rows with a future -/// `next_attempt_at` would wait until the next process restart. -/// Polls the indexed due query with bounded batches and failure -/// isolation (one request's error never aborts the batch). -fn spawn_due_request_worker(state: &AppState) -> tokio::task::JoinHandle<()> { - let app_state = state.clone(); - let mut shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - let mut interval = tokio::time::interval(std::time::Duration::from_secs(5)); - interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - loop { - tokio::select! { - _ = interval.tick() => { - match durable_outbox::drain_receive_pack_requests( - app_state.clone(), - durable_outbox::DRAIN_PER_PASS_LIMIT, - ) - .await - { - Ok((0, _)) => {} - Ok((n, _)) => tracing::info!(n, "due request worker drained"), - Err(e) => tracing::warn!(err = %e, "due request worker drain failed"), - } - } - _ = shutdown_rx.changed() => break, - } - } - }) -} - -/// Dedicated marker-tombstone worker at a shorter cadence than the -/// daily purge, so external Git refs do not linger a full period -/// after their SQL owner is gone. -fn spawn_marker_cleanup_worker(state: &AppState) -> tokio::task::JoinHandle<()> { - let app_state = state.clone(); - let git_timeout = - std::time::Duration::from_secs(app_state.config.git_service_timeout_secs.min(30)); - let mut shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - let mut interval = tokio::time::interval(std::time::Duration::from_secs(60)); - interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - loop { - tokio::select! { - _ = interval.tick() => { - if let Err(e) = durable_outbox::drain_marker_cleanup_queue( - &app_state, - durable_outbox::DRAIN_PER_PASS_LIMIT, - git_timeout, - ) - .await - { - tracing::warn!(err = %e, "marker cleanup worker failed"); - } - } - _ = shutdown_rx.changed() => break, - } - } - }) -} - -fn spawn_shutdown_signal(tx: watch::Sender) { - tokio::spawn(async move { - #[cfg(unix)] - { - use tokio::signal::unix::{signal as unix_signal, SignalKind}; - let mut sigterm = - unix_signal(SignalKind::terminate()).expect("install SIGTERM handler"); - let mut sigint = unix_signal(SignalKind::interrupt()).expect("install SIGINT handler"); - tokio::select! { - _ = sigterm.recv() => info!("SIGTERM received, shutting down"), - _ = sigint.recv() => info!("SIGINT received, shutting down"), - } - } - #[cfg(not(unix))] - { - use tokio::signal; - let _ = signal::ctrl_c().await; - info!("Ctrl-C received, shutting down"); - } - tx.send(true).ok(); - }); -} - -async fn connect_db_with_retry( - config: &Config, - db_startup: Arc, - mut shutdown_rx: watch::Receiver, -) -> Option> { - let initial_retry_secs = config.db_retry_initial_secs; - let max_retry_secs = config.db_retry_max_secs.max(initial_retry_secs); - let acquire_timeout = std::time::Duration::from_secs(config.db_acquire_timeout_secs); - let attempt_timeout = std::time::Duration::from_secs(config.db_connect_timeout_secs); - let mut attempts = 0_u64; - - loop { - if *shutdown_rx.borrow() { - return None; - } - - attempts = attempts.saturating_add(1); - db_startup.attempts.store(attempts, Ordering::Relaxed); - - // Bound the whole attempt, not just the pool connect: migrations - // block on a cross-instance advisory lock, and an unbounded wait - // there would wedge this loop — no retries, no logs, no recovery. - // Timing out and retrying is safe; migrations are idempotent. - let attempt = match tokio::time::timeout( - attempt_timeout, - Db::connect( - &config.database_url, - config.db_max_connections, - acquire_timeout, - ), - ) - .await - { - Ok(result) => result, - Err(_) => Err(anyhow!( - "connect + migrate attempt exceeded {}s (GITLAWB_DB_CONNECT_TIMEOUT_SECS); \ - is another instance holding the migration lock?", - attempt_timeout.as_secs() - )), - }; - - match attempt { - Ok(db) => { - info!(attempts, "database connection established"); - return Some(Arc::new(db)); - } - Err(err) => { - // A bad DATABASE_URL or rejected credentials won't heal on - // their own. Still retry (exiting would crash-loop and hammer - // the provider — and take liveness down with it), but log at - // error level and skip straight to the maximum backoff; the - // /ready health check is what surfaces this to deploys. - let permanent = is_likely_permanent_db_error(&err); - let retry_secs = if permanent { - max_retry_secs - } else { - database_retry_delay_secs(initial_retry_secs, max_retry_secs, attempts) - }; - db_startup - .next_retry_secs - .store(retry_secs, Ordering::Relaxed); - if permanent { - tracing::error!( - attempts, - retry_secs, - err = %err, - "database rejected our configuration (bad DATABASE_URL or credentials?) — retrying, but operator action is likely required" - ); - } else { - warn!( - attempts, - retry_secs, - err = %err, - "database unavailable during startup; retrying" - ); - } - - tokio::select! { - _ = tokio::time::sleep(std::time::Duration::from_secs(retry_secs)) => {} - changed = shutdown_rx.changed() => { - if changed.is_err() || *shutdown_rx.borrow() { - return None; - } - } - } - } - } - } -} - -/// Errors that indicate misconfiguration rather than a transient outage: a -/// malformed DATABASE_URL, or a server that answered and rejected us — -/// Postgres error class 28xxx (invalid authorization) or 3D000 (database -/// does not exist). Best-effort: an error that anyhow can't downcast back to -/// sqlx just counts as transient. -fn is_likely_permanent_db_error(err: &anyhow::Error) -> bool { - match err.downcast_ref::() { - Some(sqlx::Error::Configuration(_)) => true, - Some(sqlx::Error::Database(db)) => db - .code() - .map(|c| c.starts_with("28") || c.starts_with("3D")) - .unwrap_or(false), - _ => false, - } -} - -fn database_retry_delay_secs(initial_secs: u64, max_secs: u64, attempts: u64) -> u64 { - // The exponent bound only keeps the u32 cast safe — max_secs is the real - // (operator-configurable) cap, and saturating math handles overflow. - let exponent = attempts.saturating_sub(1).min(63) as u32; - initial_secs - .saturating_mul(2_u64.saturating_pow(exponent)) - .min(max_secs) -} - -async fn run_degraded_server( - listener: TcpListener, - node_did: String, - db_startup: Arc, - web_url: Option, - mut db_ready_rx: watch::Receiver, - mut shutdown_rx: watch::Receiver, -) -> Result<()> { - let addr = listener.local_addr().ok(); - let router = build_degraded_router(node_did, db_startup, web_url); - info!(?addr, "degraded HTTP server ready"); - - axum::serve(listener, router) - .with_graceful_shutdown(async move { - // wait_for resolves on predicate-true or sender-drop; either way - // this phase is over. - tokio::select! { - _ = db_ready_rx.wait_for(|ready| *ready) => {} - _ = shutdown_rx.wait_for(|stop| *stop) => {} - } - }) - .await?; - - Ok(()) -} - -fn build_degraded_router( - node_did: String, - db_startup: Arc, - web_url: Option, -) -> Router { - let state = DegradedState { - node_did, - db_startup, - web_url, - }; - // Everything answers 503 with the same body — including /health and - // /ready, so peer readiness probes and uptime monitors correctly see a - // node that cannot serve traffic. - // `/` additionally carries the node identity for probing peers. - Router::new() - .route("/", get(degraded_node_info)) - .fallback(degraded_unavailable) - .with_state(state) -} - -/// One source of truth for the degraded 503 body, sharing the error -/// vocabulary with error.rs so clients see the same code/message for -/// "database unavailable" regardless of which phase produced it. -fn degraded_body(db_startup: &DbStartupStatus) -> serde_json::Value { - serde_json::json!({ - "status": "degraded", - "database": "initializing", - "error": error::DB_UNAVAILABLE_CODE, - "message": error::DB_UNAVAILABLE_MESSAGE, - "db_attempts": db_startup.attempts.load(Ordering::Relaxed), - "db_next_retry_secs": db_startup.next_retry_secs.load(Ordering::Relaxed), - }) -} - -async fn degraded_node_info(State(state): State) -> impl IntoResponse { - let mut body = degraded_body(&state.db_startup); - if let Some(obj) = body.as_object_mut() { - obj.insert("name".into(), "gitlawb-node".into()); - obj.insert("version".into(), env!("CARGO_PKG_VERSION").into()); - obj.insert("did".into(), state.node_did.clone().into()); - if let Some(web_url) = &state.web_url { - obj.insert("web_url".into(), web_url.clone().into()); - } - } - (StatusCode::SERVICE_UNAVAILABLE, Json(body)) -} - -async fn degraded_unavailable(State(state): State) -> impl IntoResponse { - ( - StatusCode::SERVICE_UNAVAILABLE, - Json(degraded_body(&state.db_startup)), - ) -} - -/// Spawn a small axum router that exposes only `GET /metrics` on its own -/// listener. Returns the JoinHandle so `main()` can abort it on shutdown. -/// This is deliberately separate from the main router so the metrics port -/// can be firewalled differently from the API port — bind to localhost -/// or a private interface only. -async fn spawn_metrics_server( - addr: &str, - mut shutdown_rx: watch::Receiver, -) -> Result> { - use axum::{response::IntoResponse, routing::get, Router}; - - async fn metrics_handler() -> impl IntoResponse { - match metrics::encode() { - Ok(body) => ( - axum::http::StatusCode::OK, - [( - axum::http::header::CONTENT_TYPE, - "text/plain; version=0.0.4; charset=utf-8", - )], - body, - ), - Err(e) => ( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - [( - axum::http::header::CONTENT_TYPE, - "text/plain; charset=utf-8", - )], - format!("metrics encode error: {e}"), - ), - } - } - - let listener = TcpListener::bind(addr) - .await - .with_context(|| format!("failed to bind metrics listener to {addr}"))?; - let app = Router::new().route("/metrics", get(metrics_handler)); - - let handle = tokio::spawn(async move { - if let Err(e) = axum::serve(listener, app) - .with_graceful_shutdown(async move { - while !*shutdown_rx.borrow_and_update() { - if shutdown_rx.changed().await.is_err() { - break; - } - } - }) - .await - { - warn!(err = %e, "metrics server exited with error"); - } - }); - Ok(handle) -} - -fn build_operator_client( - config: &config::Config, - node_did: &str, -) -> Result { - use alloy::primitives::Address; - use std::str::FromStr; - - let contract_address = Address::from_str(&config.contract_node_staking) - .with_context(|| format!("invalid contract address: {}", config.contract_node_staking))?; - - let cfg = operator::OperatorConfig { - rpc_url: config.chain_rpc_url.clone(), - private_key: config.operator_private_key.clone(), - contract_address, - node_did: node_did.to_string(), - heartbeat_interval: std::time::Duration::from_secs(config.heartbeat_interval_hours * 3600), - strict_mode: config.operator_strict_mode, - }; - Ok(operator::OperatorClient::new(cfg)) -} - -/// Announce to bootstrap peers on startup, then periodically ping all known peers. -async fn gossip_task( - state: AppState, - bootstrap_peers: Vec, - mut shutdown_rx: tokio::sync::watch::Receiver, -) { - // If shutdown arrives during the initial delay, exit before announcing. - tokio::select! { - _ = tokio::time::sleep(std::time::Duration::from_secs(2)) => {} - _ = shutdown_rx.changed() => { - if *shutdown_rx.borrow() { - info!("gossip: shutdown during startup delay, exiting"); - return; - } - } - } - - // Reuse the shared no-redirect client for every gossip outbound call (the - // bootstrap announce POST and the periodic peer /ready ping). Peer URLs are - // attacker-influenceable, so a 3xx to a private address must not be followed. - // Do NOT fall back to reqwest::Client::new(): its default follows redirects - // and would reintroduce the SSRF closed here (#93). - let client = state.http_client.clone(); - let my_did = state.node_did.to_string(); - let my_url = state.config.public_url.clone().unwrap_or_default(); - - // Announce ourselves to each bootstrap peer - for peer_url in &bootstrap_peers { - // Cooperative shutdown between peers — a slow peer shouldn't - // block the node exiting. - if *shutdown_rx.borrow() { - info!("gossip: shutdown signalled during peer announce, exiting"); - return; - } - let path = "/api/v1/peers/announce"; - let announce_url = format!("{}{}", peer_url.trim_end_matches('/'), path); - let body = serde_json::json!({ - "did": my_did.clone(), - "http_url": my_url.clone(), - }); - let body_bytes = match serde_json::to_vec(&body) { - Ok(bytes) => bytes, - Err(e) => { - tracing::warn!(err = %e, "failed to serialize peer announce body"); - continue; - } - }; - let signed = sign_request(state.node_keypair.as_ref(), "POST", path, &body_bytes); - // Per-request timeout inside the loop; do not let one hung peer - // block others. The request itself is a normal tokio future so - // it's cancel-safe on shutdown. - match tokio::time::timeout( - std::time::Duration::from_secs(5), - client - .post(&announce_url) - .header("Content-Type", "application/json") - .header("Content-Digest", signed.content_digest) - .header("Signature-Input", signed.signature_input) - .header("Signature", signed.signature) - .body(body_bytes) - .send(), - ) - .await - { - Ok(Ok(resp)) => { - if resp.status().is_success() { - if let Ok(json) = resp.json::().await { - // Add them back to our peer list - if let (Some(their_did), Some(their_url)) = ( - json.get("node_did").and_then(|v| v.as_str()), - json.get("node_url").and_then(|v| v.as_str()), - ) { - if !their_url.is_empty() { - // Unproven, unconditionally: their_did and - // their_url come straight out of the contacted - // peer's JSON response body, with no signature - // and no proof the claimed DID belongs to the - // peer we reached. This path therefore seeds - // new peers and can never repoint an existing - // row. The upsert's result is matched rather - // than discarded, because "bootstrap peer - // added" printed over a refused write is a - // denial rendering as success. Non-fatal to the - // loop either way. - match state - .db - .upsert_peer( - their_did, - their_url, - db::PeerWriteAuthority::Unproven, - ) - .await - { - Ok(()) => { - tracing::info!(did = %their_did, url = %their_url, "bootstrap peer added") - } - Err(e) => { - tracing::warn!(did = %their_did, url = %their_url, err = %e, "bootstrap peer announce-back rejected") - } - } - } - } - } - } - } - Ok(Err(e)) => { - tracing::warn!(url = %announce_url, err = %e, "failed to announce to bootstrap peer") - } - Err(_) => tracing::warn!(url = %announce_url, "bootstrap peer announce timed out (5s)"), - } - } - - // Periodic ping every 5 minutes — exit on shutdown. - let mut interval = tokio::time::interval(std::time::Duration::from_secs(300)); - interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - let mut failed_once: HashSet = HashSet::new(); - loop { - tokio::select! { - _ = interval.tick() => { - let peers = match state.db.list_peers().await { - Ok(p) => p, - Err(_) => continue, - }; - gossip_ping_round( - state.db.as_ref(), - client.as_ref(), - &mut failed_once, - peers, - ) - .await; - } - _ = shutdown_rx.changed() => { - if *shutdown_rx.borrow() { - info!("gossip task: shutdown signal received, exiting"); - return; - } - } - } - } -} - -#[cfg(test)] -mod rate_limiter_sweep_tests { - use crate::rate_limit::RateLimiter; - use std::time::Duration; - - // Every per-key limiter the router mounts must be swept by the periodic - // task, the `/ipfs` one included: a limiter left out keeps expired keys - // until its map fills and the inline capacity sweep fires. Fails on the - // pre-fix sweeper, which skipped `ipfs_rate_limiter`. - #[tokio::test] - async fn sweep_evicts_expired_keys_from_every_limiter() { - let window = Duration::from_millis(30); - let mut state = crate::test_support::test_state_lazy(); - state.rate_limiter = RateLimiter::new(10, window); - state.create_ip_rate_limiter = RateLimiter::new(10, window); - state.push_rate_limiter = RateLimiter::new(10, window); - state.close_issue_rate_limiter = RateLimiter::new(10, window); - state.sync_trigger_rate_limiter = RateLimiter::new(10, window); - state.peer_write_rate_limiter = RateLimiter::new(10, window); - state.federated_rate_limiter = RateLimiter::new(10, window); - state.ipfs_rate_limiter = RateLimiter::new(10, window); - state.ipfs_work_rate_limiter = RateLimiter::new(10, window); - state.task_read_rate_limiter = RateLimiter::new(10, window); - - let limiters = |s: &crate::state::AppState| { - [ - s.rate_limiter.clone(), - s.create_ip_rate_limiter.clone(), - s.push_rate_limiter.clone(), - s.close_issue_rate_limiter.clone(), - s.sync_trigger_rate_limiter.clone(), - s.peer_write_rate_limiter.clone(), - s.federated_rate_limiter.clone(), - s.ipfs_rate_limiter.clone(), - s.ipfs_work_rate_limiter.clone(), - s.task_read_rate_limiter.clone(), - ] - }; - for l in limiters(&state) { - assert!(l.check("1.2.3.4").await); - assert_eq!(l.tracked_keys().await, 1); - } - - tokio::time::sleep(window * 3).await; - state.sweep_rate_limiters().await; - - for (i, l) in limiters(&state).into_iter().enumerate() { - assert_eq!(l.tracked_keys().await, 0, "limiter {i} was not swept"); - } - } -} - -/// it. -fn peer_ping_db_update(failed_once: &mut HashSet, did: &str, ready: bool) -> Option { - if ready { - failed_once.remove(did); - Some(true) - } else if failed_once.insert(did.to_owned()) { - None - } else { - Some(false) - } -} - -/// Build the shared node HTTP client used for every outbound fan-out (sync -/// trigger, profile/repo fetches, gossip announce + peer pings). -/// -/// No redirects: peer URLs are attacker-influenceable, so a `3xx` to a private -/// address must not be followed (SSRF guard, #78/#93). Do NOT replace with -/// `reqwest::Client::new()` — its default follows redirects. Kept as a named -/// builder so tests bind the redirect guarantee to the real client the node -/// runs, not a hand-rolled equivalent. -fn build_http_client() -> reqwest::Result { - reqwest::Client::builder() - .timeout(OUTBOUND_HTTP_TIMEOUT) - .redirect(reqwest::redirect::Policy::none()) - .build() -} - -const OUTBOUND_HTTP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); - -/// Ping a peer's DB-aware `/ready` endpoint and report whether it answered 2xx. -/// -/// A 404 falls back to `/health` for compatibility with nodes released before -/// `/ready` existed. Other readiness failures, including 503 during a database -/// outage, fail closed and must not use the liveness-only endpoint. The complete -/// `/ready`-then-`/health` probe shares one timeout budget. -/// -/// Takes the client by reference so callers supply the shared, no-redirect -/// `state.http_client`. Peer URLs are attacker-influenceable, so a `3xx` to a -/// private address must not be followed. Do NOT call this with a bare -/// `reqwest::Client::new()`: its default follows redirects and would -/// reintroduce the SSRF this guards against (#93). -pub(crate) async fn ping_peer_readiness(client: &reqwest::Client, http_url: &str) -> bool { - ping_peer_readiness_with_timeout(client, http_url, OUTBOUND_HTTP_TIMEOUT).await -} - -async fn ping_peer_readiness_with_timeout( - client: &reqwest::Client, - http_url: &str, - timeout: std::time::Duration, -) -> bool { - let base_url = http_url.trim_end_matches('/'); - let result = tokio::time::timeout(timeout, async { - let readiness = client.get(format!("{base_url}/ready")).send().await; - - match readiness { - Ok(response) if response.status().is_success() => true, - Ok(response) if response.status() == reqwest::StatusCode::NOT_FOUND => { - info!( - peer_url = %http_url, - "peer has no readiness endpoint; falling back to legacy health probe" - ); - match client.get(format!("{base_url}/health")).send().await { - Ok(response) if response.status().is_success() => true, - Ok(response) => { - warn!( - peer_url = %http_url, - status = %response.status(), - "legacy peer health probe reported unhealthy" - ); - false - } - Err(error) => { - warn!( - peer_url = %http_url, - err = %error, - "legacy peer health probe failed" - ); - false - } - } - } - Ok(response) => { - warn!( - peer_url = %http_url, - status = %response.status(), - "peer readiness probe reported unready" - ); - false - } - Err(error) => { - warn!( - peer_url = %http_url, - err = %error, - "peer readiness probe failed" - ); - false - } - } - }) - .await; - - match result { - Ok(ready) => ready, - Err(_) => { - warn!( - peer_url = %http_url, - timeout_ms = timeout.as_millis(), - "peer readiness probe timed out" - ); - false - } - } -} - -fn load_or_create_keypair(config: &Config) -> Result { - load_or_create_keypair_at(&config.resolved_key_path()) -} - -/// The node identity key's load-or-create, taken by path so the storage -/// contract can be tested without building a whole `Config`. -fn load_or_create_keypair_at(key_path: &std::path::Path) -> Result { - if p2p::path_denotes_a_directory(key_path, None) - || key_path - .components() - .any(|c| matches!(c, std::path::Component::ParentDir)) - { - anyhow::bail!( - "GITLAWB_KEY ({}) must name a key file, not a directory; a trailing separator, \ - a final `.` or `..` component, and `..` traversal are refused so the path \ - cannot retarget a parent", - key_path.display() - ); - } - - #[cfg(unix)] - if let Some(pem) = p2p::load_identity_pem_if_present(key_path)? { - let kp = Keypair::from_pem(&pem).map_err(|e| anyhow::anyhow!("invalid PEM key: {e}"))?; - info!(path = %key_path.display(), "loaded existing identity"); - return Ok(kp); - } - - #[cfg(not(unix))] - if key_path.exists() { - let pem = std::fs::read_to_string(key_path) - .with_context(|| format!("failed to read key from {}", key_path.display()))?; - let kp = Keypair::from_pem(&pem).map_err(|e| anyhow::anyhow!("invalid PEM key: {e}"))?; - info!(path = %key_path.display(), "loaded existing identity"); - return Ok(kp); - } - - let kp = Keypair::generate(); - let pem = kp - .to_pem() - .map_err(|e| anyhow::anyhow!("failed to serialize key: {e}"))?; - - // The directory is created pinned to 0700 and the PEM is published - // through the same scratch-then-link path the p2p key uses, at a - // verified 0600. The previous flow (create_dir_all with no mode, then - // write, then set_permissions) is the sequence INV-23 prohibits: it - // left the directory world-writable under a permissive umask, and - // under a restrictive one it could not be opened at all, which failed - // the whole node here, before the listener binds. - #[cfg(unix)] - p2p::create_pinned_dir_and_publish(key_path, pem.as_bytes())?; - - #[cfg(not(unix))] - { - if let Some(parent) = key_path.parent() { - std::fs::create_dir_all(parent)?; - } - std::fs::write(key_path, pem.as_bytes())?; - } - - info!(path = %key_path.display(), did = %kp.did(), "generated new node identity"); - Ok(kp) -} - -async fn gossip_ping_round( - db: &Db, - client: &reqwest::Client, - failed_once: &mut HashSet, - peers: Vec, -) { - { - let current_dids: HashSet<&str> = peers.iter().map(|peer| peer.did.as_str()).collect(); - failed_once.retain(|did| current_dids.contains(did.as_str())); - } - for peer in peers { - let ok = ping_peer_readiness(client, &peer.http_url).await; - match peer_ping_db_update(failed_once, &peer.did, ok) { - Some(reachable) => { - if let Err(error) = db.mark_peer_ping(&peer.did, reachable).await { - warn!(did = %peer.did, err = %error, "failed to persist peer readiness"); - } - } - None => warn!( - did = %peer.did, - "peer readiness probe failed once; preserving previous federation status" - ), - } - } -} - -#[cfg(test)] -mod gossip_ssrf_tests { - use super::{ - gossip_ping_round, peer_ping_db_update, ping_peer_readiness, - ping_peer_readiness_with_timeout, - }; - use axum::{http::StatusCode, routing::get, Router}; - use sqlx::PgPool; - use std::collections::HashSet; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::Arc; - use std::time::Duration; - - // Build the client exactly as production does (super::build_http_client) so - // these tests bind the redirect guarantee to the real shared client the - // node runs. A regression that makes build_http_client follow redirects - // fails ping_peer_readiness_does_not_follow_redirect. - fn production_http_client() -> reqwest::Client { - super::build_http_client().expect("failed to build production http client") - } - - #[test] - fn peer_ping_requires_two_failures_before_marking_unreachable() { - let mut failed_once = HashSet::new(); - let did = "did:key:z6MkPeer"; - - assert_eq!( - peer_ping_db_update(&mut failed_once, did, false), - None, - "one transient failure must preserve the persisted federation gate" - ); - assert_eq!( - peer_ping_db_update(&mut failed_once, did, false), - Some(false), - "a sustained failure must mark the peer unreachable" - ); - assert_eq!( - peer_ping_db_update(&mut failed_once, did, true), - Some(true), - "a success must restore reachability immediately" - ); - assert_eq!( - peer_ping_db_update(&mut failed_once, did, false), - None, - "a success must reset the consecutive-failure state" - ); - } - - #[sqlx::test] - async fn gossip_ping_round_requires_two_failures_before_persisting_unreachable(pool: PgPool) { - let mut server = mockito::Server::new_async().await; - let ready = server - .mock("GET", "/ready") - .with_status(503) - .expect(2) - .create_async() - .await; - let state = crate::test_support::test_state(pool.clone()).await; - let did = "did:key:z6MkGossipHysteresis"; - let now = chrono::Utc::now().to_rfc3339(); - sqlx::query( - "INSERT INTO peers (did, http_url, last_seen, last_ping_ok, announced_at) - VALUES ($1, $2, $3, TRUE, $3)", - ) - .bind(did) - .bind(server.url()) - .bind(now) - .execute(&pool) - .await - .unwrap(); - - let mut failed_once = HashSet::from(["did:key:z6MkRemovedPeer".to_owned()]); - gossip_ping_round( - state.db.as_ref(), - &production_http_client(), - &mut failed_once, - state.db.list_peers().await.unwrap(), - ) - .await; - assert!( - !failed_once.contains("did:key:z6MkRemovedPeer"), - "failure tracking must prune peers outside the current snapshot" - ); - assert!( - failed_once.contains(did), - "the first failed sample must be retained for the next round" - ); - assert!( - state - .db - .list_peers() - .await - .unwrap() - .into_iter() - .find(|peer| peer.did == did) - .unwrap() - .last_ping_ok, - "one transient failure must preserve the persisted federation gate" - ); - - gossip_ping_round( - state.db.as_ref(), - &production_http_client(), - &mut failed_once, - state.db.list_peers().await.unwrap(), - ) - .await; - assert!( - !state - .db - .list_peers() - .await - .unwrap() - .into_iter() - .find(|peer| peer.did == did) - .unwrap() - .last_ping_ok, - "two consecutive failed rounds must persist the peer as unreachable" - ); - ready.assert_async().await; - } - - // A peer answering `/ready` with a 302 toward an internal address must not - // be followed: the redirect target must never be requested (#93). - #[tokio::test] - async fn ping_peer_readiness_does_not_follow_redirect() { - let mut server = mockito::Server::new_async().await; - let internal = server - .mock("GET", "/internal-metadata") - .with_status(200) - .expect(0) - .create_async() - .await; - let _ready = server - .mock("GET", "/ready") - .with_status(302) - .with_header("location", &format!("{}/internal-metadata", server.url())) - .create_async() - .await; - - let ok = ping_peer_readiness(&production_http_client(), &server.url()).await; - - assert!(!ok, "a 302 must not count as a healthy peer"); - // expect(0) is enforced only at assert time; this fails if the redirect - // was followed to the internal target. - internal.assert_async().await; - } - - #[tokio::test] - async fn ping_peer_readiness_reports_success_on_200() { - let mut server = mockito::Server::new_async().await; - let _ready = server - .mock("GET", "/ready") - .with_status(200) - .create_async() - .await; - - let ok = ping_peer_readiness(&production_http_client(), &server.url()).await; - - assert!(ok, "a 200 /ready must count as a ready peer"); - } - - #[tokio::test] - async fn ping_peer_readiness_ignores_liveness_only_health() { - let mut server = mockito::Server::new_async().await; - let health = server - .mock("GET", "/health") - .with_status(200) - .expect(0) - .create_async() - .await; - let ready = server - .mock("GET", "/ready") - .with_status(503) - .expect(1) - .create_async() - .await; - - let ok = ping_peer_readiness(&production_http_client(), &server.url()).await; - - assert!(!ok, "a peer with an unavailable database must not be ready"); - health.assert_async().await; - ready.assert_async().await; - } - - #[tokio::test] - async fn ping_peer_readiness_falls_back_for_legacy_peer() { - let mut server = mockito::Server::new_async().await; - let ready = server - .mock("GET", "/ready") - .with_status(404) - .expect(1) - .create_async() - .await; - let health = server - .mock("GET", "/health") - .with_status(200) - .expect(1) - .create_async() - .await; - - let ok = ping_peer_readiness(&production_http_client(), &server.url()).await; - - assert!( - ok, - "a legacy peer with a healthy /health must remain reachable" - ); - ready.assert_async().await; - health.assert_async().await; - } - - #[tokio::test] - async fn ping_peer_readiness_legacy_fallback_shares_deadline() { - let health_requests = Arc::new(AtomicUsize::new(0)); - let health_requests_for_route = Arc::clone(&health_requests); - let app = Router::new() - .route( - "/ready", - get(|| async { - tokio::time::sleep(Duration::from_millis(100)).await; - StatusCode::NOT_FOUND - }), - ) - .route( - "/health", - get(move || { - let health_requests = Arc::clone(&health_requests_for_route); - async move { - health_requests.fetch_add(1, Ordering::Relaxed); - tokio::time::sleep(Duration::from_millis(300)).await; - StatusCode::OK - } - }), - ); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("failed to bind delayed legacy peer"); - let peer_url = format!( - "http://{}", - listener - .local_addr() - .expect("delayed legacy peer has no local address") - ); - let server = tokio::spawn(async move { - axum::serve(listener, app) - .await - .expect("delayed legacy peer failed"); - }); - - let ok = ping_peer_readiness_with_timeout( - &production_http_client(), - &peer_url, - // Each response arrives within 350 ms, but the two serial requests - // cannot both finish within one 350 ms probe budget. - Duration::from_millis(350), - ) - .await; - - assert!(!ok, "the fallback must not start a fresh timeout budget"); - assert_eq!( - health_requests.load(Ordering::Relaxed), - 1, - "the delayed 404 should reach the legacy fallback" - ); - server.abort(); - } - - #[tokio::test] - async fn ping_peer_readiness_legacy_fallback_does_not_follow_redirect() { - let mut server = mockito::Server::new_async().await; - let internal = server - .mock("GET", "/internal-metadata") - .with_status(200) - .expect(0) - .create_async() - .await; - let _ready = server - .mock("GET", "/ready") - .with_status(404) - .create_async() - .await; - let _health = server - .mock("GET", "/health") - .with_status(302) - .with_header("location", &format!("{}/internal-metadata", server.url())) - .create_async() - .await; - - let ok = ping_peer_readiness(&production_http_client(), &server.url()).await; - - assert!(!ok, "the legacy fallback must not follow redirects"); - internal.assert_async().await; - } - - // A transport error (nothing listening) must map to unhealthy, never a - // spurious healthy — the .unwrap_or(false) arm. - #[tokio::test] - async fn ping_peer_readiness_reports_unready_on_connection_error() { - let ok = ping_peer_readiness(&production_http_client(), "http://127.0.0.1:1").await; - assert!(!ok, "a connection error must count as an unready peer"); - } -} - -#[cfg(test)] -mod identity_key_storage_tests { - use super::*; - - /// Requested modes for the objects this invocation creates. Named once so - /// a failure prints achieved against requested rather than a bare boolean. - #[cfg(unix)] - const IDENTITY_WANT_DIR_MODE: u32 = 0o700; - #[cfg(unix)] - const IDENTITY_WANT_KEY_MODE: u32 = 0o600; - /// Mode a PRE-EXISTING ancestor is built at. Not group-writable, so the - /// write-authority check accepts it, and nothing this process does may - /// change it: only directories this invocation creates are its business. - #[cfg(unix)] - const IDENTITY_PREEXISTING_MODE: u32 = 0o755; - /// Mode the group-writable rows build. A parent carrying it must be - /// refused before anything is created inside it. - #[cfg(unix)] - const IDENTITY_LOOSE_MODE: u32 = 0o775; - - #[cfg(unix)] - fn identity_mode_of(path: &std::path::Path) -> u32 { - use std::os::unix::fs::PermissionsExt; - std::fs::symlink_metadata(path) - .unwrap_or_else(|e| panic!("stat {}: {e}", path.display())) - .permissions() - .mode() - & 0o7777 - } - - /// Every directory on the key path with its achieved mode against what the - /// contract requested, so a RED is attributable to the mode rather than to - /// "something failed". - #[cfg(unix)] - fn identity_key_tree(base: &std::path::Path, key: &std::path::Path) -> String { - let mut dirs = Vec::new(); - let mut cur = key.parent(); - while let Some(d) = cur { - dirs.push(d.to_path_buf()); - if d == base { - break; - } - cur = d.parent(); - } - dirs.reverse(); - let mut out = String::new(); - for d in dirs { - match std::fs::symlink_metadata(&d) { - Ok(_) => out.push_str(&format!( - " dir {} achieved mode {:04o}, requested {:04o}\n", - d.display(), - identity_mode_of(&d), - IDENTITY_WANT_DIR_MODE - )), - Err(e) => out.push_str(&format!(" dir {} absent ({e})\n", d.display())), - } - } - match std::fs::symlink_metadata(key) { - Ok(_) => out.push_str(&format!( - " key {} achieved mode {:04o}, requested {:04o}\n", - key.display(), - identity_mode_of(key), - IDENTITY_WANT_KEY_MODE - )), - Err(e) => out.push_str(&format!(" key {} absent ({e})\n", key.display())), - } - out - } - - /// An object a refused boot must not have left behind. The message is the - /// fixture's own: the anyhow text never carries the phrase, so a `contains` - /// on the error would be a check that can never pass. - #[cfg(unix)] - fn assert_identity_absent(path: &std::path::Path) { - assert!( - std::fs::symlink_metadata(path).is_err(), - "{} left behind by a refused boot", - path.display() - ); - } - - /// A symlink refusal must name the symlink. An unrelated EACCES, ENOENT, - /// or write-authority failure satisfies "an error happened" while proving - /// nothing about whether the interior component was followed, so every - /// symlink row asserts the reason rather than the bare failure. - #[cfg(unix)] - fn assert_names_symlink_refusal(text: &str, what: &str) { - let lower = text.to_lowercase(); - assert!( - lower.contains("symlink") || lower.contains("symbolic link"), - "{what} must be refused for the symlink on the path, and the refusal must say so, \ - got: {text}" - ); - } - - /// Every regular file under `root`, following no symlink out of it. Used - /// to prove a refused boot published nothing where a followed interior - /// link would have put it, including a target outside the base. - #[cfg(unix)] - fn identity_collect_files(root: &std::path::Path, found: &mut Vec) { - let Ok(entries) = std::fs::read_dir(root) else { - return; - }; - for entry in entries.flatten() { - let path = entry.path(); - let Ok(md) = std::fs::symlink_metadata(&path) else { - continue; - }; - if md.is_dir() { - identity_collect_files(&path, found); - } else if md.is_file() { - found.push(path); - } - } - } - - /// No key anywhere beneath `root`. Stronger than naming one expected path, - /// because a followed link can land the key at a name the test did not - /// predict. - #[cfg(unix)] - fn assert_no_identity_key_under(root: &std::path::Path, what: &str) { - let mut found = Vec::new(); - identity_collect_files(root, &mut found); - let keys: Vec = found - .iter() - .filter(|p| p.file_name().is_some_and(|n| n == "identity.pem")) - .map(|p| p.display().to_string()) - .collect(); - assert!( - keys.is_empty(), - "{what}: a refused boot published an identity under {}: {keys:?}", - root.display() - ); - } - - /// The assertions every success row shares: exact 0700 on each directory - /// this invocation created, 0600 on the key, no scratch residue beside it, - /// and a base this process never touched. - #[cfg(unix)] - fn assert_identity_success( - base: &std::path::Path, - key: &std::path::Path, - created: &[std::path::PathBuf], - ) { - for d in created { - let mode = identity_mode_of(d); - assert_eq!( - mode, - IDENTITY_WANT_DIR_MODE, - "created directory {} achieved mode {:04o}, requested {:04o}\n{}", - d.display(), - mode, - IDENTITY_WANT_DIR_MODE, - identity_key_tree(base, key) - ); - } - let key_mode = identity_mode_of(key); - assert_eq!( - key_mode, - IDENTITY_WANT_KEY_MODE, - "identity key achieved mode {key_mode:04o}, requested 0600\n{}", - identity_key_tree(base, key) - ); - let leaf = key.parent().expect("the key names a parent"); - let entries: Vec = std::fs::read_dir(leaf) - .expect("read the key directory") - .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) - .collect(); - assert_eq!( - entries, - vec![key.file_name().unwrap().to_string_lossy().into_owned()], - "the key directory must hold only the published key, no scratch residue: {entries:?}" - ); - let base_mode = identity_mode_of(base); - assert_eq!( - base_mode, 0o700, - "the base directory must be left at 0700, achieved {base_mode:04o}" - ); - } - - /// Boot a success row and check its contract. `call_path` is what the code - /// is handed (relative for the relative row); `key` is the same key under - /// `base`, so the mode checks never depend on the child's cwd. - #[cfg(unix)] - fn run_identity_success_row( - base: &std::path::Path, - call_path: &std::path::Path, - key: &std::path::Path, - created: &[std::path::PathBuf], - phase: &str, - ) -> String { - if phase == "reload" { - assert!( - key.exists(), - "the reload phase requires the create phase to have published a key at {}", - key.display() - ); - } - let kp = load_or_create_keypair_at(call_path).unwrap_or_else(|e| { - panic!( - "{} the node identity, got: {e:#}\nkey storage at failure:\n{}", - if phase == "reload" { - "reload in a fresh process must load" - } else { - "first boot must create" - }, - identity_key_tree(base, key) - ) - }); - assert_identity_success(base, key, created); - kp.did().to_string() - } - - /// Fixture: one (layout, umask, phase) row of the identity lifecycle - /// matrix. - /// - /// `~/.gitlawb` holds BOTH keys, and `load_or_create_keypair` runs before - /// the listener binds, so a mask that strips owner bits here takes the - /// whole node down before any p2p code is reached. `umask` is - /// process-global, so every row is a child. Double-gated like the p2p - /// fixtures: `#[ignore]` keeps it out of a normal run and the env check - /// keeps it inert under a bare `--ignored` sweep, which would otherwise set - /// a process-global umask inside the shared test process. - #[cfg(unix)] - #[test] - #[ignore = "self-exec fixture: only runs under GITLAWB_TEST_FIXTURE=identity-key-umask"] - fn fixture_identity_key_under_hostile_umask() { - use std::os::unix::fs::PermissionsExt; - - if std::env::var("GITLAWB_TEST_FIXTURE").ok().as_deref() != Some("identity-key-umask") { - return; - } - let base = std::path::PathBuf::from( - std::env::var("GITLAWB_TEST_BASE").expect("GITLAWB_TEST_BASE"), - ); - let umask_val = - u32::from_str_radix(&std::env::var("GITLAWB_TEST_UMASK").unwrap(), 8).unwrap(); - let layout = std::env::var("GITLAWB_TEST_LAYOUT").expect("GITLAWB_TEST_LAYOUT"); - let phase = std::env::var("GITLAWB_TEST_PHASE").expect("GITLAWB_TEST_PHASE"); - - // SAFETY: `umask` only reads and replaces the process-wide value, and - // this process exists solely for this probe. No restore: the value dies - // with the child. Set FIRST, before any layout is built, so the - // explicit chmods below are what give a pre-existing directory its - // mode regardless of the mask. - unsafe { libc::umask(umask_val as libc::mode_t) }; - - match layout.as_str() { - "one-missing" => { - let dir = base.join(".gitlawb"); - let key = dir.join("identity.pem"); - let did = run_identity_success_row(&base, &key, &key, &[dir], &phase); - println!("identity-key-umask: asserted did={did}"); - } - "multi-missing" => { - let one = base.join("one"); - let two = one.join("two"); - let key = two.join("identity.pem"); - let did = run_identity_success_row(&base, &key, &key, &[one, two], &phase); - println!("identity-key-umask: asserted did={did}"); - } - "three-missing" => { - let one = base.join("one"); - let two = one.join("two"); - let three = two.join("three"); - let key = three.join("identity.pem"); - let did = run_identity_success_row(&base, &key, &key, &[one, two, three], &phase); - println!("identity-key-umask: asserted did={did}"); - } - "under-preexisting" => { - let pre = base.join("pre"); - if phase == "create" { - std::fs::create_dir(&pre).expect("create the pre-existing ancestor"); - std::fs::set_permissions( - &pre, - std::fs::Permissions::from_mode(IDENTITY_PREEXISTING_MODE), - ) - .expect("chmod the pre-existing ancestor"); - } - let one = pre.join("one"); - let two = one.join("two"); - let key = two.join("identity.pem"); - let did = run_identity_success_row(&base, &key, &key, &[one, two], &phase); - let pre_mode = identity_mode_of(&pre); - assert_eq!( - pre_mode, - IDENTITY_PREEXISTING_MODE, - "a pre-existing ancestor is neither chmodded nor judged: {} achieved \ - {pre_mode:04o}, expected {:04o}", - pre.display(), - IDENTITY_PREEXISTING_MODE - ); - println!("identity-key-umask: asserted did={did}"); - } - "relative-multi-missing" => { - std::env::set_current_dir(&base).expect("chdir into the row's base"); - let one = base.join("one"); - let two = one.join("two"); - let key = two.join("identity.pem"); - let call_path = std::path::PathBuf::from("one/two/identity.pem"); - let did = run_identity_success_row(&base, &call_path, &key, &[one, two], &phase); - println!("identity-key-umask: asserted did={did}"); - } - "loose-ancestor" => { - let loose = base.join("loose"); - std::fs::create_dir(&loose).expect("create the group-writable ancestor"); - std::fs::set_permissions( - &loose, - std::fs::Permissions::from_mode(IDENTITY_LOOSE_MODE), - ) - .expect("chmod the group-writable ancestor"); - let one = loose.join("one"); - let two = one.join("two"); - let key = two.join("identity.pem"); - let Err(err) = load_or_create_keypair_at(&key) else { - panic!( - "a group-writable deepest ancestor must be refused\nkey storage:\n{}", - identity_key_tree(&base, &key) - ) - }; - let text = format!("{err:#}"); - // The reason is checked before the leftovers so a refusal for - // the wrong reason is reported as that, not as a stray file. - assert!( - text.contains("writable beyond its owner"), - "a group-writable deepest ancestor must be refused for the write-authority \ - reason, got: {text}" - ); - assert_identity_absent(&one); - assert_identity_absent(&two); - assert_identity_absent(&key); - let loose_mode = identity_mode_of(&loose); - assert_eq!( - loose_mode, - IDENTITY_LOOSE_MODE, - "a refusal must not chmod {}: achieved {loose_mode:04o}", - loose.display() - ); - println!("identity-key-umask: refused"); - } - "write-failure" => { - let one = base.join("one"); - let two = one.join("two"); - let three = two.join("three"); - let key = three.join("identity.pem"); - // Armed on this thread only, and disarmed before the - // assertions so a later boot in this process is unaffected. - p2p::FAIL_KEY_WRITE.with(|f| f.set(true)); - let result = load_or_create_keypair_at(&key); - p2p::FAIL_KEY_WRITE.with(|f| f.set(false)); - let Err(err) = result else { - panic!("an injected key-write failure must not report a created identity") - }; - let text = format!("{err:#}"); - assert!( - text.contains("injected key-write failure"), - "the failure must name the injected key write, got: {text}" - ); - assert_identity_absent(&one); - assert_identity_absent(&two); - assert_identity_absent(&three); - assert_identity_absent(&key); - let residue: Vec = std::fs::read_dir(&base) - .expect("read the base directory") - .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) - .collect(); - assert!( - residue.is_empty(), - "a failed first boot must leave nothing under the base: {residue:?}" - ); - println!("identity-key-umask: refused"); - } - "race-loose-intermediate" => { - let one = base.join("one"); - let two = one.join("two"); - let key = two.join("identity.pem"); - // A concurrent process wins the mkdir with a group-writable - // mode. The loser adopts nothing it did not create, so its - // rollback must leave the winner's directory alone. - p2p::pin::RACE_CREATE_MODE.with(|c| c.set(Some(IDENTITY_LOOSE_MODE))); - let result = load_or_create_keypair_at(&key); - let Err(err) = result else { - panic!("a race-won group-writable intermediate must be refused") - }; - let text = format!("{err:#}"); - assert!( - text.contains("writable beyond its owner"), - "the refusal must name the write-authority problem, got: {text}" - ); - assert!( - one.exists(), - "a directory this invocation did not create must not be removed: {} is gone", - one.display() - ); - let one_mode = identity_mode_of(&one); - assert_eq!( - one_mode, - IDENTITY_LOOSE_MODE, - "a directory this invocation did not create must not be removed or chmodded: \ - {} achieved {one_mode:04o}", - one.display() - ); - assert_identity_absent(&two); - assert_identity_absent(&key); - println!("identity-key-umask: refused"); - } - "symlinked-deepest-ancestor" => { - let real = base.join("real"); - std::fs::create_dir(&real).expect("create the real ancestor"); - std::fs::set_permissions(&real, std::fs::Permissions::from_mode(0o700)) - .expect("chmod the real ancestor"); - let link = base.join("link"); - std::os::unix::fs::symlink("real", &link).expect("plant the symlinked ancestor"); - let key = link.join("one").join("two").join("identity.pem"); - let Err(err) = load_or_create_keypair_at(&key) else { - panic!("a symlinked deepest existing ancestor must be refused, not followed") - }; - let text = format!("{err:#}"); - assert!( - text.contains("a symlink here is refused rather than followed"), - "a symlinked deepest existing ancestor must be refused, not followed, and the \ - refusal must say so, got: {text}" - ); - for p in [ - link.join("one"), - link.join("one").join("two"), - key.clone(), - real.join("one"), - real.join("one").join("two"), - real.join("one").join("two").join("identity.pem"), - ] { - assert_identity_absent(&p); - } - println!("identity-key-umask: refused"); - } - "populated-leaf-refuses-rollback" => { - let one = base.join("one"); - let two = one.join("two"); - let key = two.join("identity.pem"); - // The key is published and only the scratch removal fails, so - // the leaf this invocation created is no longer empty when the - // rollback reaches it. `AT_REMOVEDIR` must refuse it. - p2p::FAIL_SCRATCH_UNLINK.with(|f| f.set(true)); - let result = load_or_create_keypair_at(&key); - p2p::FAIL_SCRATCH_UNLINK.with(|f| f.set(false)); - let Err(err) = result else { - panic!("an injected scratch-unlink failure must not report success") - }; - let text = format!("{err:#}"); - assert!( - text.contains("failed to remove the scratch name"), - "the failure must name the scratch it could not remove, got: {text}" - ); - assert!( - text.contains("also failed to remove") - && text.contains("which this process created"), - "the rollback must report the populated leaf it was refused, got: {text}" - ); - let key_mode = identity_mode_of(&key); - assert_eq!( - key_mode, IDENTITY_WANT_KEY_MODE, - "a published key must survive its own boot's rollback: achieved \ - {key_mode:04o}, requested 0600" - ); - for d in [&one, &two] { - let mode = identity_mode_of(d); - assert_eq!( - mode, - IDENTITY_WANT_DIR_MODE, - "created directory {} achieved mode {mode:04o}, requested {:04o}", - d.display(), - IDENTITY_WANT_DIR_MODE - ); - } - let scratch: Vec = std::fs::read_dir(&two) - .expect("read the key directory") - .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) - .filter(|n| n.starts_with(".p2p.key.") && n.ends_with(".tmp")) - .collect(); - assert_eq!( - scratch.len(), - 1, - "the refused unlink must leave exactly one scratch beside the key: {scratch:?}" - ); - let kp = load_or_create_keypair_at(&key) - .expect("a boot after the injected failure must load the published key"); - println!("identity-key-umask: refused did={}", kp.did()); - } - // One process of a real concurrent first boot. Every child of a - // race runs this arm; the driver owns the tree assertions, because - // only it knows the race has finished. - "concurrent-first-boot" => { - let barrier = std::path::PathBuf::from( - std::env::var("GITLAWB_TEST_BARRIER").expect("GITLAWB_TEST_BARRIER"), - ); - let id = std::env::var("GITLAWB_TEST_RACE_ID").expect("GITLAWB_TEST_RACE_ID"); - let role = std::env::var("GITLAWB_TEST_RACE_ROLE").expect("GITLAWB_TEST_RACE_ROLE"); - let key = base.join("one").join("two").join("identity.pem"); - - // Two-phase rendezvous: each child announces itself with - // `ready.` and then spins on `go`, which the driver creates - // only once every ready file is present. Chosen over a shared - // wake-up timestamp because a deadline bounds only the skew it - // cannot observe: a child that is slow to exec still arrives - // late and the race never overlaps. Here every child is already - // inside the spin loop before `go` can appear, so the spread - // across the racing call is one loop iteration rather than the - // tens of milliseconds a process takes to start. The deadline - // exists so a child whose sibling died never hangs the suite. - std::fs::write(barrier.join(format!("ready.{id}")), b"") - .expect("announce readiness at the starting barrier"); - let go = barrier.join("go"); - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(120); - while !go.exists() { - assert!( - std::time::Instant::now() < deadline, - "the starting barrier never opened" - ); - std::hint::spin_loop(); - } - - // The `fail-write` role is the only injected part of the race, - // and it is what puts a REAL second process into the window - // section 7 of the plan describes: this child creates a - // component, the other adopts it, this one fails before either - // has written, and its rollback reaches the still-empty - // directory the other is holding open. - if role == "fail-write" { - p2p::FAIL_KEY_WRITE.with(|f| f.set(true)); - } - let result = load_or_create_keypair_at(&key); - p2p::FAIL_KEY_WRITE.with(|f| f.set(false)); - - match result { - Ok(kp) => { - // A boot that reports success must have the identity it - // returned on disk, at 0600, and readable as a keypair. - // Checked here rather than in the driver so a success - // that raced with a rollback is attributed to the - // process that claimed it. - let mode = identity_mode_of(&key); - assert_eq!( - mode, - IDENTITY_WANT_KEY_MODE, - "a successful concurrent boot published {} at {mode:04o}, requested \ - 0600", - key.display() - ); - let pem = std::fs::read_to_string(&key) - .unwrap_or_else(|e| panic!("read {}: {e}", key.display())); - let on_disk = Keypair::from_pem(&pem) - .expect("a successful concurrent boot must leave a loadable keypair"); - assert_eq!( - on_disk.did().to_string(), - kp.did().to_string(), - "a successful boot must return the identity that is on disk" - ); - println!("identity-key-race: ok did={}", kp.did()); - } - Err(err) => { - // One line so the driver can classify it; the anyhow - // chain is multi-line under `{:#}` only when a context - // carries a newline, but flattening is free insurance. - let text = format!("{err:#}").replace('\n', " "); - println!("identity-key-race: err {text}"); - } - } - } - other => panic!("unknown layout {other}"), - } - } - - /// GITLAWB_KEY names a PEM file, not a dedicated key directory. An existing - /// 0755 parent is usable (no group/world write) and must not be chmodded. - #[cfg(unix)] - #[test] - fn existing_identity_parent_is_not_chmodded() { - use std::os::unix::fs::PermissionsExt; - - for parent_name in [".gitlawb", "shared"] { - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); - let dir = base.path().join(parent_name); - std::fs::create_dir(&dir).unwrap(); - std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o755)).unwrap(); - - let key = dir.join("identity.pem"); - load_or_create_keypair_at(&key).expect("first boot into an existing 0755 parent"); - - let mode = std::fs::symlink_metadata(&dir) - .unwrap() - .permissions() - .mode() - & 0o7777; - assert_eq!( - mode, 0o755, - "{parent_name}: an existing 0755 identity parent must stay 0755, found {mode:04o}" - ); - assert_eq!( - std::fs::symlink_metadata(&key) - .unwrap() - .permissions() - .mode() - & 0o7777, - 0o600, - "the identity key must be owner-only" - ); - } - } - - /// A parent this process creates is still pinned to 0700. That is the - /// missing-directory path, not an adopt of an operator-owned tree. - #[cfg(unix)] - #[test] - fn missing_identity_parent_is_created_0700() { - use std::os::unix::fs::PermissionsExt; - - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); - let dir = base.path().join(".gitlawb"); - let key = dir.join("identity.pem"); - load_or_create_keypair_at(&key).expect("first boot creates the identity parent"); - let mode = std::fs::symlink_metadata(&dir) - .unwrap() - .permissions() - .mode() - & 0o7777; - assert_eq!(mode, 0o700, "a parent this process created must be 0700"); - } - - /// Group/world write on the parent is replacement authority over a 0600 - /// key. Refuse, and leave the directory untouched. - #[cfg(unix)] - #[test] - fn writable_identity_parent_is_refused_unchanged() { - use std::os::unix::fs::PermissionsExt; - - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); - let dir = base.path().join("tmp"); - std::fs::create_dir(&dir).unwrap(); - std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o777)).unwrap(); - let key = dir.join("identity.pem"); - let Err(err) = load_or_create_keypair_at(&key) else { - panic!("a world-writable identity parent must be refused"); - }; - let text = format!("{err:#}"); - assert!( - text.contains("writable") || text.contains("replace"), - "the refusal must name the write-authority problem, got: {text}" - ); - let mode = std::fs::symlink_metadata(&dir) - .unwrap() - .permissions() - .mode() - & 0o7777; - assert_eq!(mode, 0o777, "a refusal must not chmod the parent"); - assert!(!key.exists(), "a refusal must not publish the identity key"); - } - - /// An existing key is loaded, never rewritten and never chmodded: the - /// creation path is the only thing this change touches. - #[cfg(unix)] - #[test] - fn existing_identity_key_is_loaded_unchanged() { - use std::os::unix::fs::PermissionsExt; - - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); - let key = base.path().join(".gitlawb").join("identity.pem"); - - let created = load_or_create_keypair_at(&key).expect("first boot"); - let before = std::fs::read(&key).unwrap(); - - // A deliberately odd but readable mode must survive: an existing key is - // the operator's, and this path does not repair it. - std::fs::set_permissions(&key, std::fs::Permissions::from_mode(0o400)).unwrap(); - let reloaded = load_or_create_keypair_at(&key).expect("reload"); - - assert_eq!(created.did(), reloaded.did(), "the identity must be stable"); - assert_eq!( - std::fs::read(&key).unwrap(), - before, - "the key must not be rewritten" - ); - assert_eq!( - std::fs::symlink_metadata(&key) - .unwrap() - .permissions() - .mode() - & 0o7777, - 0o400, - "an existing key's mode must not be changed" - ); - } - - /// Parent for the identity lifecycle matrix: drives every row as a child - /// process and requires create and reload to agree on the DID. - #[cfg(unix)] - #[test] - fn identity_key_storage_is_umask_independent() { - use std::os::unix::fs::PermissionsExt; - - const SUCCESS_SENTINEL: &str = "identity-key-umask: asserted did="; - const REFUSED_SENTINEL: &str = "identity-key-umask: refused"; - // Fixed order, and part of the contract: 0000 and 0002 are the masks - // that leave a group/world-writable ancestor behind, 0022 is the one - // that boots with the wrong mode, and 0777 strips owner bits. Rows - // abort at the first failing mask, so several load-bearing proofs are - // pinned to which mask that is. - const ALL_UMASKS: &[&str] = &["0000", "0002", "0022", "0777"]; - - // (layout, umasks, succeeds). A success row runs create then reload - // against one base and must return the same DID; a reject row runs - // create alone and must print the refusal sentinel. - let rows: &[(&str, &[&str], bool)] = &[ - ("one-missing", ALL_UMASKS, true), - ("multi-missing", ALL_UMASKS, true), - ("three-missing", ALL_UMASKS, true), - ("under-preexisting", ALL_UMASKS, true), - ("relative-multi-missing", ALL_UMASKS, true), - ("loose-ancestor", ALL_UMASKS, false), - ("write-failure", ALL_UMASKS, false), - ("race-loose-intermediate", &["0002"], false), - ("symlinked-deepest-ancestor", &["0022"], false), - ("populated-leaf-refuses-rollback", &["0022"], false), - ]; - - // Both filters are optional and both panic on a value that names - // nothing: a typo that quietly ran no child would be an empty green, - // which is the failure this matrix exists to rule out. - let layout_only = std::env::var("GITLAWB_TEST_LAYOUT_ONLY").ok(); - let umask_only = std::env::var("GITLAWB_TEST_UMASK_ONLY").ok(); - if let Some(l) = layout_only.as_deref() { - assert!( - rows.iter().any(|r| r.0 == l), - "GITLAWB_TEST_LAYOUT_ONLY={l} names no row in the matrix" - ); - } - if let Some(u) = umask_only.as_deref() { - assert!( - ALL_UMASKS.contains(&u), - "GITLAWB_TEST_UMASK_ONLY={u} names no umask in the matrix" - ); - } - - // Run one row in a child and return whatever follows its sentinel. - fn run_row( - base: &std::path::Path, - umask: &str, - layout: &str, - phase: &str, - sentinel: &str, - ) -> String { - let mut cmd = std::process::Command::new(std::env::current_exe().expect("current_exe")); - cmd.args([ - "identity_key_storage_tests::fixture_identity_key_under_hostile_umask", - "--exact", - "--ignored", - "--nocapture", - ]) - .env("GITLAWB_TEST_FIXTURE", "identity-key-umask") - .env("GITLAWB_TEST_BASE", base) - .env("GITLAWB_TEST_UMASK", umask) - .env("GITLAWB_TEST_LAYOUT", layout) - .env("GITLAWB_TEST_PHASE", phase); - let output = cmd.output().expect("spawn the identity-key fixture"); - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - let row = format!("layout={layout} umask={umask} phase={phase}"); - - assert!( - output.status.success(), - "row {row}: the identity-key fixture must pass\n\ - --- stdout ---\n{stdout}\n--- stderr ---\n{stderr}" - ); - // A filter matching nothing exits 0, and the fixture's env gate - // returns early as a passing test, so neither alone is proof. - assert!( - stdout.contains("1 passed"), - "row {row}: filter must select one passing test\n{stdout}" - ); - let line = stdout - .lines() - .find(|l| l.starts_with(sentinel)) - .unwrap_or_else(|| { - panic!("row {row}: fixture must print its sentinel\n--- stdout ---\n{stdout}") - }); - line[sentinel.len()..].trim().to_string() - } - - let mut ran = 0usize; - for (layout, umasks, succeeds) in rows { - if layout_only.as_deref().is_some_and(|l| l != *layout) { - continue; - } - for umask in *umasks { - if umask_only.as_deref().is_some_and(|u| u != *umask) { - continue; - } - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)) - .unwrap(); - if *succeeds { - let created = run_row(base.path(), umask, layout, "create", SUCCESS_SENTINEL); - let reloaded = run_row(base.path(), umask, layout, "reload", SUCCESS_SENTINEL); - assert_eq!( - created, reloaded, - "layout={layout} umask={umask}: a fresh process must reload the same DID" - ); - } else { - run_row(base.path(), umask, layout, "create", REFUSED_SENTINEL); - } - ran += 1; - } - } - assert!( - ran > 0, - "the row and umask filters selected no child; a matrix that runs nothing is not a \ - passing matrix" - ); - } - - /// Two or more real first boots racing the same multi-level missing key - /// path. - /// - /// The rollback added by this round can remove an intermediate a concurrent - /// boot has already adopted, so the plan's section 7 claims both processes - /// still fail closed. That claim was reasoned, not executed. This drives it - /// with real processes: `umask` and the injection hooks are process-global, - /// so threads inside one process would not be the production path. - /// - /// The race is nondeterministic and the assertions are not. Nothing here - /// asserts a particular interleaving; every check is a property that must - /// hold for all of them, so the test is deterministic in PASS/FAIL: - /// - /// * a boot that reports success has its own identity on disk at 0600 and - /// parseable (asserted in the child, which is the process that claimed - /// it); - /// * every surviving directory this run created is exactly 0700, never - /// group or world writable; - /// * every successful boot agrees on the DID; - /// * if no boot succeeded, nothing partial is left (no key, no scratch) and - /// a retry in this process then succeeds, which is the explicit form of - /// "fail closed and recover on the next boot". - #[cfg(unix)] - #[test] - fn identity_concurrent_first_boots_fail_closed() { - use std::io::Read; - use std::os::unix::fs::PermissionsExt; - - const OK_SENTINEL: &str = "identity-key-race: ok did="; - const ERR_SENTINEL: &str = "identity-key-race: err "; - // 0002 is the mask that produced finding 1, so a directory this run - // creates without pinning it lands 0775 and the exact-mode assertions - // below catch it. - const RACE_UMASK: &str = "0002"; - - // (arm, one role per child). The pure arm is three unaided first boots. - // The other two add losers that fail after the walk and before any key - // is written, so their rollback reaches a directory a sibling may - // already hold open: that is the window the plan describes and the only - // injected part of the race. Two children hit it rarely, because the - // winner reaches its scratch file before a single loser reaches its - // rmdir; four losers against one unaided boot hit it often, so the - // pressure arm is what actually executes the interleaving. The - // assertions do not depend on which arm hits it. - let arms: &[(&str, &[&str])] = &[ - ("pure", &["pure", "pure", "pure"][..]), - ("loser-fails", &["fail-write", "pure"][..]), - ( - "rollback-pressure", - &[ - "fail-write", - "fail-write", - "fail-write", - "fail-write", - "pure", - ][..], - ), - ]; - - // A single run of a nondeterministic test proves very little, so the - // default loops the whole race; raise it through the env for a longer - // shake without editing the test. - let iterations: usize = std::env::var("GITLAWB_TEST_RACE_ITERATIONS") - .ok() - .map(|v| { - v.parse() - .expect("GITLAWB_TEST_RACE_ITERATIONS must be a count") - }) - .unwrap_or(50); - assert!(iterations > 0, "a race run zero times proves nothing"); - - #[derive(Default)] - struct Tally { - all_ok: usize, - some_ok: usize, - none_ok: usize, - /// A boot refused because the directory it was holding had been - /// removed under it: the interleaving section 7 describes. - adopted_dir_unlinked: usize, - } - let mut tallies: Vec<(&str, Tally)> = arms - .iter() - .map(|(name, _)| (*name, Tally::default())) - .collect(); - - for iteration in 0..iterations { - for (arm_idx, (arm, roles)) in arms.iter().enumerate() { - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)) - .unwrap(); - let barrier = tempfile::tempdir().unwrap(); - let one = base.path().join("one"); - let two = one.join("two"); - let key = two.join("identity.pem"); - let label = format!("arm={arm} iteration={iteration}"); - - let mut children = Vec::new(); - for (id, role) in roles.iter().enumerate() { - let mut cmd = - std::process::Command::new(std::env::current_exe().expect("current_exe")); - cmd.args([ - "identity_key_storage_tests::fixture_identity_key_under_hostile_umask", - "--exact", - "--ignored", - "--nocapture", - ]) - .env("GITLAWB_TEST_FIXTURE", "identity-key-umask") - .env("GITLAWB_TEST_BASE", base.path()) - .env("GITLAWB_TEST_UMASK", RACE_UMASK) - .env("GITLAWB_TEST_LAYOUT", "concurrent-first-boot") - .env("GITLAWB_TEST_PHASE", "create") - .env("GITLAWB_TEST_BARRIER", barrier.path()) - .env("GITLAWB_TEST_RACE_ID", id.to_string()) - .env("GITLAWB_TEST_RACE_ROLE", *role) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()); - children.push(cmd.spawn().expect("spawn a racing identity boot")); - } - - // Open the barrier only once every child has announced itself. - // If one died before announcing, the deadline lets the run - // proceed to the exit-status assertion below rather than hang. - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(120); - loop { - let ready = (0..roles.len()) - .filter(|id| barrier.path().join(format!("ready.{id}")).exists()) - .count(); - if ready == roles.len() || std::time::Instant::now() >= deadline { - break; - } - std::thread::yield_now(); - } - std::fs::write(barrier.path().join("go"), b"").expect("open the starting barrier"); - - let mut oks = Vec::new(); - let mut errs = Vec::new(); - for (id, child) in children.into_iter().enumerate() { - let output = child.wait_with_output().expect("await a racing boot"); - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - let who = format!("{label} child={id} role={}", roles[id]); - assert!( - output.status.success(), - "{who}: the racing fixture must pass whichever way the race went\n\ - --- stdout ---\n{stdout}\n--- stderr ---\n{stderr}" - ); - assert!( - stdout.contains("1 passed"), - "{who}: filter must select one passing test\n{stdout}" - ); - let line = stdout - .lines() - .find(|l| l.starts_with("identity-key-race: ")) - .unwrap_or_else(|| { - panic!("{who}: the child must report its outcome\n{stdout}") - }); - if let Some(did) = line.strip_prefix(OK_SENTINEL) { - oks.push(did.trim().to_string()); - } else if let Some(text) = line.strip_prefix(ERR_SENTINEL) { - errs.push((roles[id], text.to_string())); - } else { - panic!("{who}: unrecognized outcome line {line:?}"); - } - } - - // Invariant: every surviving directory this run created is - // exactly 0700. A missing one is legal (a loser rolled it - // back); a group or world writable one never is. - for d in [&one, &two] { - if d.exists() { - let mode = identity_mode_of(d); - assert_eq!( - mode, - IDENTITY_WANT_DIR_MODE, - "{label}: surviving directory {} achieved mode {mode:04o}, requested \ - {:04o}", - d.display(), - IDENTITY_WANT_DIR_MODE - ); - let residue: Vec = std::fs::read_dir(d) - .expect("read a surviving directory") - .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) - .filter(|n| n.starts_with(".p2p.key.") && n.ends_with(".tmp")) - .collect(); - assert!( - residue.is_empty(), - "{label}: a finished race must leave no scratch behind in {}: \ - {residue:?}", - d.display() - ); - } - } - - if key.exists() { - // Invariant: the published key is 0600 and loadable, and - // every boot that reported success agrees on it. - let mode = identity_mode_of(&key); - assert_eq!( - mode, IDENTITY_WANT_KEY_MODE, - "{label}: the surviving key achieved mode {mode:04o}, requested 0600" - ); - let mut pem = String::new(); - std::fs::File::open(&key) - .expect("open the surviving key") - .read_to_string(&mut pem) - .expect("read the surviving key"); - let on_disk = Keypair::from_pem(&pem) - .expect("the surviving key must parse as a keypair") - .did() - .to_string(); - for did in &oks { - assert_eq!( - *did, on_disk, - "{label}: a successful boot reported an identity the disk does not \ - hold" - ); - } - } else { - // Invariant: no key means no boot may claim success, and - // the tree must be clean enough that the next boot works. - assert!( - oks.is_empty(), - "{label}: a boot reported success with no key on disk: {oks:?}" - ); - let kp = load_or_create_keypair_at(&key).unwrap_or_else(|e| { - panic!( - "{label}: every boot failed, so a retry must succeed, got: {e:#}\n\ - errors: {errs:?}\nkey storage:\n{}", - identity_key_tree(base.path(), &key) - ) - }); - assert_identity_success(base.path(), &key, &[one.clone(), two.clone()]); - assert!( - !kp.did().to_string().is_empty(), - "{label}: the recovering boot must produce an identity" - ); - } - - let tally = &mut tallies[arm_idx].1; - if oks.len() == roles.len() { - tally.all_ok += 1; - } else if oks.is_empty() { - tally.none_ok += 1; - } else { - tally.some_ok += 1; - } - // The rollback-under-an-adopted-directory window: a boot that - // was refused because the directory it held was unlinked. - if errs.iter().any(|(_, t)| { - t.contains("No such file or directory") || t.contains("(os error 2)") - }) { - tally.adopted_dir_unlinked += 1; - } - } - } - - for (arm, tally) in &tallies { - println!( - "identity-race-summary: arm={arm} iterations={iterations} all_ok={} some_ok={} \ - none_ok={} adopted_dir_unlinked={}", - tally.all_ok, tally.some_ok, tally.none_ok, tally.adopted_dir_unlinked - ); - } - } - - /// Bare `GITLAWB_KEY=identity.pem` (and `./identity.pem`) must still create - /// the identity in the working directory. The p2p key refuses that form on - /// purpose; the node identity has always allowed it. - #[cfg(unix)] - #[test] - #[ignore = "self-exec fixture: only runs under GITLAWB_TEST_FIXTURE=identity-key-bare"] - fn fixture_bare_identity_key_in_cwd() { - use std::os::unix::fs::PermissionsExt; - - if std::env::var("GITLAWB_TEST_FIXTURE").ok().as_deref() != Some("identity-key-bare") { - return; - } - let cwd = std::path::PathBuf::from( - std::env::var("GITLAWB_TEST_BASE").expect("GITLAWB_TEST_BASE"), - ); - std::env::set_current_dir(&cwd).expect("chdir into isolated tempdir"); - let cwd_mode_before = std::fs::symlink_metadata(".").unwrap().permissions().mode() & 0o7777; - - let name = std::env::var("GITLAWB_TEST_KEY_NAME").expect("GITLAWB_TEST_KEY_NAME"); - let kp = load_or_create_keypair_at(std::path::Path::new(&name)) - .unwrap_or_else(|e| panic!("bare identity path {name:?} must create, got: {e:#}")); - assert!( - std::path::Path::new(&name).exists() || std::path::Path::new("identity.pem").exists(), - "the key must land in the working directory" - ); - let key = if std::path::Path::new(&name).exists() { - std::path::PathBuf::from(&name) - } else { - std::path::PathBuf::from("identity.pem") - }; - assert_eq!( - std::fs::symlink_metadata(&key) - .unwrap() - .permissions() - .mode() - & 0o7777, - 0o600, - "the identity key must be owner-only" - ); - let cwd_mode_after = std::fs::symlink_metadata(".").unwrap().permissions().mode() & 0o7777; - assert_eq!( - cwd_mode_before, cwd_mode_after, - "creating a bare identity key must not chmod the working directory" - ); - let reloaded = load_or_create_keypair_at(std::path::Path::new(&name)).expect("reload"); - assert_eq!(kp.did(), reloaded.did(), "the identity must be stable"); - println!("identity-key-bare: asserted did={}", kp.did()); - } - - #[cfg(unix)] - #[test] - fn bare_identity_key_path_creates_in_cwd_without_chmodding_cwd() { - use std::os::unix::fs::PermissionsExt; - - for name in ["identity.pem", "./identity.pem"] { - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); - let mut cmd = std::process::Command::new(std::env::current_exe().expect("current_exe")); - cmd.args([ - "identity_key_storage_tests::fixture_bare_identity_key_in_cwd", - "--exact", - "--ignored", - "--nocapture", - ]) - .env("GITLAWB_TEST_FIXTURE", "identity-key-bare") - .env("GITLAWB_TEST_BASE", base.path()) - .env("GITLAWB_TEST_KEY_NAME", name); - let output = cmd.output().expect("spawn the bare-identity fixture"); - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!( - output.status.success(), - "name={name}: bare identity path must create\n\ - --- stdout ---\n{stdout}\n--- stderr ---\n{stderr}" - ); - assert!( - stdout.contains("1 passed"), - "name={name}: filter must select one passing test\n{stdout}" - ); - assert!( - stdout.contains("identity-key-bare: asserted did="), - "name={name}: fixture must print its sentinel\n{stdout}" - ); - let cwd_mode = std::fs::symlink_metadata(base.path()) - .unwrap() - .permissions() - .mode() - & 0o7777; - assert_eq!( - cwd_mode, 0o755, - "name={name}: the parent test must also see cwd left at 0755" - ); - } - } - - /// A 0600 identity file is not protected if cwd is group/world-writable: - /// another local user can unlink or replace the entry. Creating into that - /// cwd must fail, and must not leave a key behind. Do not chmod cwd. - #[cfg(unix)] - #[test] - #[ignore = "self-exec fixture: only runs under GITLAWB_TEST_FIXTURE=identity-key-bare-writable"] - fn fixture_bare_identity_key_in_writable_cwd_is_refused() { - use std::os::unix::fs::PermissionsExt; - - if std::env::var("GITLAWB_TEST_FIXTURE").ok().as_deref() - != Some("identity-key-bare-writable") - { - return; - } - let cwd = std::path::PathBuf::from( - std::env::var("GITLAWB_TEST_BASE").expect("GITLAWB_TEST_BASE"), - ); - std::env::set_current_dir(&cwd).expect("chdir into isolated tempdir"); - std::fs::set_permissions(&cwd, std::fs::Permissions::from_mode(0o777)).unwrap(); - let cwd_mode_before = std::fs::symlink_metadata(".").unwrap().permissions().mode() & 0o7777; - - let Err(err) = load_or_create_keypair_at(std::path::Path::new("identity.pem")) else { - panic!("a bare identity path under a writable cwd must be refused"); - }; - let msg = format!("{err:#}"); - assert!( - msg.contains("writable beyond its owner") || msg.contains("0777"), - "the refusal must name the writable-parent reason, got: {msg}" - ); - assert!( - !std::path::Path::new("identity.pem").exists(), - "a refused cwd must not have identity.pem created" - ); - let cwd_mode_after = std::fs::symlink_metadata(".").unwrap().permissions().mode() & 0o7777; - assert_eq!( - cwd_mode_before, cwd_mode_after, - "refusing a writable cwd must not chmod it" - ); - println!("identity-key-bare-writable: asserted"); - } - - #[cfg(unix)] - #[test] - fn bare_identity_key_path_refuses_a_writable_cwd() { - use std::os::unix::fs::PermissionsExt; - - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o777)).unwrap(); - let mut cmd = std::process::Command::new(std::env::current_exe().expect("current_exe")); - cmd.args([ - "identity_key_storage_tests::fixture_bare_identity_key_in_writable_cwd_is_refused", - "--exact", - "--ignored", - "--nocapture", - ]) - .env("GITLAWB_TEST_FIXTURE", "identity-key-bare-writable") - .env("GITLAWB_TEST_BASE", base.path()); - let output = cmd - .output() - .expect("spawn the writable-cwd identity fixture"); - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!( - output.status.success(), - "writable cwd must refuse the bare identity path\n\ - --- stdout ---\n{stdout}\n--- stderr ---\n{stderr}" - ); - assert!( - stdout.contains("1 passed"), - "filter must select one passing test\n{stdout}" - ); - assert!( - stdout.contains("identity-key-bare-writable: asserted"), - "fixture must print its sentinel\n{stdout}" - ); - let leftovers: Vec<_> = std::fs::read_dir(base.path()) - .unwrap() - .filter_map(|e| e.ok()) - .map(|e| e.file_name()) - .collect(); - assert!( - leftovers.is_empty(), - "parent must also see no identity.pem, found: {leftovers:?}" - ); - let cwd_mode = std::fs::symlink_metadata(base.path()) - .unwrap() - .permissions() - .mode() - & 0o7777; - assert_eq!(cwd_mode, 0o777, "cwd must stay 0777"); - } - - /// An existing key in a writable directory still loads. Create is refused - /// there; turning that into a boot failure on upgrade would strand nodes - /// that already have a key. - #[cfg(unix)] - #[test] - fn existing_identity_in_a_writable_dir_still_loads() { - use std::os::unix::fs::PermissionsExt; - - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); - let key = base.path().join("identity.pem"); - let created = load_or_create_keypair_at(&key).expect("create under 0755"); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o777)).unwrap(); - let reloaded = load_or_create_keypair_at(&key) - .expect("an existing identity in a writable directory must still load"); - assert_eq!(created.did(), reloaded.did()); - assert_eq!( - std::fs::symlink_metadata(base.path()) - .unwrap() - .permissions() - .mode() - & 0o7777, - 0o777, - "load must not chmod the writable directory" - ); - } - - #[cfg(unix)] - #[test] - fn identity_symlink_key_is_refused() { - use std::os::unix::fs::PermissionsExt; - - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); - let real = base.path().join("real.pem"); - let created = load_or_create_keypair_at(&real).expect("create the symlink target"); - std::os::unix::fs::symlink(&real, base.path().join("identity.pem")).unwrap(); - let err = match load_or_create_keypair_at(&base.path().join("identity.pem")) { - Ok(kp) => panic!( - "a symlink at the identity path must be refused, not followed; loaded did={}", - kp.did() - ), - Err(e) => format!("{e:#}"), - }; - assert!( - err.contains("symlink") || err.to_lowercase().contains("too many levels"), - "symlink refusal must name the link, got: {err}" - ); - let reread = load_or_create_keypair_at(&real).expect("target still loads by its real path"); - assert_eq!( - created.did(), - reread.did(), - "the symlink target must be unchanged" - ); - } - - #[cfg(unix)] - #[test] - fn identity_symlinked_grandparent_is_refused() { - use std::os::unix::fs::PermissionsExt; - - let base = tempfile::tempdir().unwrap(); - let real = base.path().join("real"); - std::fs::create_dir(&real).unwrap(); - std::fs::set_permissions(&real, std::fs::Permissions::from_mode(0o700)).unwrap(); - let link = base.path().join("link"); - std::os::unix::fs::symlink(&real, &link).unwrap(); - let key = link.join("keys").join("identity.pem"); - let err = match load_or_create_keypair_at(&key) { - Ok(_) => panic!("a symlinked grandparent must be refused, not followed"), - Err(e) => format!("{e:#}"), - }; - assert!( - err.contains("symlink") - || err.to_lowercase().contains("too many levels") - || err.contains("loop"), - "symlinked grandparent refusal must name the link, got: {err}" - ); - assert!( - !real.join("keys").exists(), - "symlink target must be untouched" - ); - } - - /// Build `base/evil/one` plus `base/link -> evil`, the layout every - /// interior-symlink row shares. `link` is relative so the layout is - /// self-contained; the absolute row builds its own. - #[cfg(unix)] - fn plant_interior_symlink_layout(base: &std::path::Path) -> std::path::PathBuf { - use std::os::unix::fs::PermissionsExt; - - let evil = base.join("evil"); - std::fs::create_dir(&evil).expect("create the symlink target directory"); - std::fs::set_permissions(&evil, std::fs::Permissions::from_mode(0o700)) - .expect("chmod the symlink target directory"); - let one = evil.join("one"); - std::fs::create_dir(&one).expect("create the already-existing next level"); - std::fs::set_permissions(&one, std::fs::Permissions::from_mode(0o700)) - .expect("chmod the already-existing next level"); - std::os::unix::fs::symlink("evil", base.join("link")).expect("plant the interior symlink"); - evil - } - - /// `O_NOFOLLOW` binds the FINAL component only, so an interior symlink - /// whose next level already exists is resolved in one open and followed. - /// With `link -> evil` and `evil/one` present, the walk-up opens - /// `link/one` by pathname, anchors on `evil/one`, and creates the node - /// identity inside the attacker's tree. - #[cfg(unix)] - #[test] - fn identity_interior_symlink_with_existing_next_level_is_refused_not_followed() { - use std::os::unix::fs::PermissionsExt; - - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); - let evil = plant_interior_symlink_layout(base.path()); - - let key = base - .path() - .join("link") - .join("one") - .join("two") - .join("identity.pem"); - match load_or_create_keypair_at(&key) { - Ok(kp) => panic!( - "an interior symlink on the key path was followed, not refused: the identity \ - {} was created through {} into the symlink's target\nkey storage:\n{}", - kp.did(), - key.display(), - identity_key_tree(base.path(), &key) - ), - Err(e) => assert_names_symlink_refusal( - &format!("{e:#}"), - "an interior symlink whose next level already exists", - ), - } - assert_identity_absent(&evil.join("one").join("two")); - assert_no_identity_key_under( - base.path(), - "an interior symlink with an existing next level", - ); - } - - /// The same interior symlink pointed by absolute path at a directory - /// outside the base. Following it publishes the node's identity somewhere - /// the configured path does not name at all, so the assertion is that no - /// key appears outside the base. - #[cfg(unix)] - #[test] - fn identity_interior_symlink_escaping_the_base_must_not_publish_the_key_outside_it() { - use std::os::unix::fs::PermissionsExt; - - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); - let outside = tempfile::tempdir().unwrap(); - std::fs::set_permissions(outside.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); - let outside_one = outside.path().join("one"); - std::fs::create_dir(&outside_one).expect("create the escaped next level"); - std::fs::set_permissions(&outside_one, std::fs::Permissions::from_mode(0o700)).unwrap(); - std::os::unix::fs::symlink(outside.path(), base.path().join("link")) - .expect("plant the absolute escaping symlink"); - - let key = base - .path() - .join("link") - .join("one") - .join("two") - .join("identity.pem"); - match load_or_create_keypair_at(&key) { - Ok(kp) => panic!( - "an absolute interior symlink was followed out of the base: the identity {} \ - was created outside {} through {}", - kp.did(), - base.path().display(), - key.display() - ), - Err(e) => assert_names_symlink_refusal( - &format!("{e:#}"), - "an interior symlink whose target is outside the base", - ), - } - assert_no_identity_key_under(outside.path(), "an escaping interior symlink"); - assert_identity_absent(&outside_one.join("two")); - } - - /// A chain, because refusing one link is not the same property as - /// refusing the path. `l1 -> l2 -> evil`, so the resolution that must be - /// refused takes two hops before it reaches the existing next level. - #[cfg(unix)] - #[test] - fn identity_chained_interior_symlinks_are_refused_not_followed() { - use std::os::unix::fs::PermissionsExt; - - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); - let evil = base.path().join("evil"); - std::fs::create_dir(&evil).unwrap(); - std::fs::set_permissions(&evil, std::fs::Permissions::from_mode(0o700)).unwrap(); - let one = evil.join("one"); - std::fs::create_dir(&one).unwrap(); - std::fs::set_permissions(&one, std::fs::Permissions::from_mode(0o700)).unwrap(); - std::os::unix::fs::symlink("evil", base.path().join("l2")).expect("plant the second hop"); - std::os::unix::fs::symlink("l2", base.path().join("l1")).expect("plant the first hop"); - - let key = base - .path() - .join("l1") - .join("one") - .join("two") - .join("identity.pem"); - match load_or_create_keypair_at(&key) { - Ok(kp) => panic!( - "a two-link chain on the key path was followed, not refused: the identity {} \ - was created through {}", - kp.did(), - key.display() - ), - Err(e) => { - assert_names_symlink_refusal(&format!("{e:#}"), "a chain of two interior symlinks") - } - } - assert_identity_absent(&one.join("two")); - assert_no_identity_key_under(base.path(), "a chain of two interior symlinks"); - } - - /// The existing-named-parent fast path never enters the walk: it opens the - /// whole configured parent pathname in one call and publishes into - /// whatever that resolves to. `link/one` exists through the symlink, so - /// this row reaches the fast path and not the walk-up. - #[cfg(unix)] - #[test] - fn identity_existing_named_parent_reached_through_a_symlink_is_refused_not_followed() { - use std::os::unix::fs::PermissionsExt; - - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); - let evil = plant_interior_symlink_layout(base.path()); - - let key = base.path().join("link").join("one").join("identity.pem"); - match load_or_create_keypair_at(&key) { - Ok(kp) => panic!( - "the existing-named-parent fast path followed an interior symlink: the identity \ - {} was published into {} through {}", - kp.did(), - evil.join("one").display(), - key.display() - ), - Err(e) => assert_names_symlink_refusal( - &format!("{e:#}"), - "an interior symlink on the existing-named-parent fast path", - ), - } - assert_identity_absent(&evil.join("one").join("identity.pem")); - assert_no_identity_key_under(base.path(), "the existing-named-parent fast path"); - } - - /// Fixture: the relative-path row. `cwd` is process-global, so the row - /// that proves a relative configured path is judged the same way runs in - /// its own process. Double-gated like the other fixtures here: `#[ignore]` - /// keeps it out of a normal run and the env check keeps it inert under a - /// bare `--ignored` sweep, which would otherwise chdir the shared test - /// process. - #[cfg(unix)] - #[test] - #[ignore = "self-exec fixture: only runs under GITLAWB_TEST_FIXTURE=identity-symlink-relative"] - fn fixture_identity_relative_path_with_interior_symlink() { - if std::env::var("GITLAWB_TEST_FIXTURE").ok().as_deref() - != Some("identity-symlink-relative") - { - return; - } - let base = std::path::PathBuf::from( - std::env::var("GITLAWB_TEST_BASE").expect("GITLAWB_TEST_BASE"), - ); - let evil = plant_interior_symlink_layout(&base); - std::env::set_current_dir(&base).expect("chdir into the row's base"); - - let call_path = std::path::Path::new("link/one/two/identity.pem"); - match load_or_create_keypair_at(call_path) { - Ok(kp) => panic!( - "an interior symlink on a RELATIVE key path was followed, not refused: the \ - identity {} was created through {}", - kp.did(), - call_path.display() - ), - Err(e) => assert_names_symlink_refusal( - &format!("{e:#}"), - "an interior symlink on a relative configured path", - ), - } - assert_identity_absent(&evil.join("one").join("two")); - assert_no_identity_key_under(&base, "a relative path with an interior symlink"); - println!("identity-symlink-relative: refused"); - } - - /// Driver for the relative row. An absolute path is not the only shape an - /// operator configures, and the walk resolves a relative pathname against - /// the process cwd, so the same interior-symlink refusal has to hold there. - #[cfg(unix)] - #[test] - fn identity_relative_key_path_with_interior_symlink_is_refused_not_followed() { - use std::os::unix::fs::PermissionsExt; - - const REFUSED_SENTINEL: &str = "identity-symlink-relative: refused"; - - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); - - let mut cmd = std::process::Command::new(std::env::current_exe().expect("current_exe")); - cmd.args([ - "identity_key_storage_tests::fixture_identity_relative_path_with_interior_symlink", - "--exact", - "--ignored", - "--nocapture", - ]) - .env("GITLAWB_TEST_FIXTURE", "identity-symlink-relative") - .env("GITLAWB_TEST_BASE", base.path()); - let output = cmd - .output() - .expect("spawn the relative-path symlink fixture"); - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - - assert!( - output.status.success(), - "a relative key path with an interior symlink must be refused, not followed\n\ - --- stdout ---\n{stdout}\n--- stderr ---\n{stderr}" - ); - // A filter matching nothing exits 0, and the fixture's env gate returns - // early as a passing test, so neither alone is proof the row ran. - assert!( - stdout.contains("1 passed"), - "the filter must select one passing test\n{stdout}" - ); - assert!( - stdout.contains(REFUSED_SENTINEL), - "the fixture must print its refusal sentinel\n--- stdout ---\n{stdout}" - ); - } - - /// THE LOAD PATH, which is the identity-substitution primitive rather than - /// a permissions bug. The node's real key sits at `a/keys/identity.pem` - /// and the configured path is `link/keys/identity.pem`. An attacker who - /// can only repoint `link` from `a` to `b` makes the load resolve to a PEM - /// they supplied, and the node boots as THEIR DID while the real key is - /// still on disk, untouched, with nothing logged as wrong. - /// - /// The assertion is on the DID, not on a mode or an error string: a mode - /// assertion cannot see a substitution, because the substituted key is - /// a perfectly well-formed 0600 PEM. - #[cfg(unix)] - #[test] - fn identity_load_through_repointed_interior_symlink_must_not_adopt_the_substituted_key() { - use std::os::unix::fs::PermissionsExt; - - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); - - let real_key = base.path().join("a").join("keys").join("identity.pem"); - let real_did = load_or_create_keypair_at(&real_key) - .expect("plant the node's real identity") - .did() - .to_string(); - let attacker_key = base.path().join("b").join("keys").join("identity.pem"); - let attacker_did = load_or_create_keypair_at(&attacker_key) - .expect("plant the attacker's identity") - .did() - .to_string(); - assert_ne!( - real_did, attacker_did, - "the row needs two distinct identities to tell substitution from a load" - ); - let real_bytes = std::fs::read(&real_key).expect("read the real key"); - - // The configured path never changes. Only the interior component does, - // which is exactly the authority an attacker with write access to one - // directory has. - let link = base.path().join("link"); - std::os::unix::fs::symlink("a", &link).expect("point the configured path at the real key"); - std::fs::remove_file(&link).expect("the attacker repoints the interior component"); - std::os::unix::fs::symlink("b", &link).expect("repoint the interior component"); - - let configured = link.join("keys").join("identity.pem"); - match load_or_create_keypair_at(&configured) { - Ok(kp) => { - let booted = kp.did().to_string(); - if booted == attacker_did { - panic!( - "identity substitution: repointing the interior symlink {} made the node \ - boot as the attacker's identity {attacker_did} (planted at {}) instead \ - of its own {real_did} at {}", - link.display(), - attacker_key.display(), - real_key.display() - ); - } - panic!( - "the load followed the interior symlink at {} and returned {booted} rather \ - than refusing the path", - link.display() - ); - } - Err(e) => assert_names_symlink_refusal( - &format!("{e:#}"), - "a repointed interior symlink on the load path", - ), - } - - assert_eq!( - std::fs::read(&real_key).expect("read the real key"), - real_bytes, - "the real key must be untouched by the refused boot" - ); - assert_eq!( - load_or_create_keypair_at(&real_key) - .expect("the real key must still load by its real path") - .did() - .to_string(), - real_did, - "the node's own identity must be unchanged" - ); - } - - /// The weaker variant of the same primitive. The attacker repoints the - /// interior component at an EMPTY directory, the load finds no key, and - /// the create path mints a fresh identity and logs it as a first boot. The - /// node comes up with a DID nobody knows and its real key is orphaned in - /// place with nothing reported. - #[cfg(unix)] - #[test] - fn identity_load_through_symlink_to_an_empty_directory_must_not_silently_mint_a_new_identity() { - use std::os::unix::fs::PermissionsExt; - - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); - - let real_key = base.path().join("a").join("keys").join("identity.pem"); - let real_did = load_or_create_keypair_at(&real_key) - .expect("plant the node's real identity") - .did() - .to_string(); - let real_bytes = std::fs::read(&real_key).expect("read the real key"); - - let empty_keys = base.path().join("empty").join("keys"); - std::fs::create_dir_all(&empty_keys).expect("create the empty target tree"); - std::fs::set_permissions(&empty_keys, std::fs::Permissions::from_mode(0o700)).unwrap(); - std::fs::set_permissions( - base.path().join("empty"), - std::fs::Permissions::from_mode(0o700), - ) - .unwrap(); - let link = base.path().join("link"); - std::os::unix::fs::symlink("a", &link).expect("point the configured path at the real key"); - std::fs::remove_file(&link).expect("the attacker repoints the interior component"); - std::os::unix::fs::symlink("empty", &link).expect("repoint at an empty directory"); - - let configured = link.join("keys").join("identity.pem"); - match load_or_create_keypair_at(&configured) { - Ok(kp) => { - let booted = kp.did().to_string(); - panic!( - "silent identity loss: repointing the interior symlink {} at an empty \ - directory made the node mint a fresh identity {booted} as if this were a \ - first boot, orphaning its real identity {real_did} at {}", - link.display(), - real_key.display() - ); - } - Err(e) => assert_names_symlink_refusal( - &format!("{e:#}"), - "an interior symlink repointed at an empty directory", - ), - } - - assert_identity_absent(&empty_keys.join("identity.pem")); - assert_eq!( - std::fs::read(&real_key).expect("read the real key"), - real_bytes, - "the real key must be untouched by the refused boot" - ); - assert_eq!( - load_or_create_keypair_at(&real_key) - .expect("the real key must still load by its real path") - .did() - .to_string(), - real_did, - "the node's own identity must be unchanged" - ); - } - - /// Already correct, and here so it stays that way: a symlink at the key's - /// IMMEDIATE parent is the final component of the load's directory open, - /// which is the one position `O_NOFOLLOW` does bind. - #[cfg(unix)] - #[test] - fn identity_symlinked_immediate_parent_is_refused_at_load() { - use std::os::unix::fs::PermissionsExt; - - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); - let real = base.path().join("real"); - std::fs::create_dir(&real).unwrap(); - std::fs::set_permissions(&real, std::fs::Permissions::from_mode(0o700)).unwrap(); - let planted_did = load_or_create_keypair_at(&real.join("identity.pem")) - .expect("plant a key behind the symlink") - .did() - .to_string(); - std::os::unix::fs::symlink("real", base.path().join("link")).unwrap(); - - let configured = base.path().join("link").join("identity.pem"); - match load_or_create_keypair_at(&configured) { - Ok(kp) => panic!( - "a symlink at the key's immediate parent was followed: loaded {} (planted \ - {planted_did}) through {}", - kp.did(), - configured.display() - ), - Err(e) => assert_names_symlink_refusal( - &format!("{e:#}"), - "a symlink at the key's immediate parent", - ), - } - } - - /// Already correct, and here so it stays that way: a dangling symlink is - /// refused rather than replaced by a real directory of the same name, - /// both when it is the top component of the missing suffix and when it - /// sits under a real directory. - #[cfg(unix)] - #[test] - fn identity_dangling_symlink_on_the_key_path_is_refused_never_replaced() { - use std::os::unix::fs::PermissionsExt; - - for under_real_dir in [false, true] { - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); - let holder = if under_real_dir { - let real = base.path().join("real"); - std::fs::create_dir(&real).unwrap(); - std::fs::set_permissions(&real, std::fs::Permissions::from_mode(0o700)).unwrap(); - real - } else { - base.path().to_path_buf() - }; - let link = holder.join("link"); - std::os::unix::fs::symlink("nowhere", &link).expect("plant the dangling symlink"); - - let key = link.join("one").join("identity.pem"); - let where_ = if under_real_dir { - "a dangling symlink under a real directory" - } else { - "a dangling symlink at the top of the path" - }; - match load_or_create_keypair_at(&key) { - Ok(kp) => panic!( - "{where_} was not refused: the identity {} was created through {}", - kp.did(), - key.display() - ), - Err(e) => assert_names_symlink_refusal(&format!("{e:#}"), where_), - } - assert!( - std::fs::symlink_metadata(&link) - .expect("the dangling symlink must still be there") - .file_type() - .is_symlink(), - "{where_}: the refused boot replaced {} with something else", - link.display() - ); - assert_no_identity_key_under(base.path(), where_); - } - } - - /// 0111 grandparent cannot mkdir, so the key directory must already exist. - /// Opening that grandparent must not require directory-list permission. - #[cfg(all(unix, any(target_os = "linux", target_os = "android")))] - #[test] - fn identity_search_only_grandparent_publishes_into_existing_key_dir() { - use std::os::unix::fs::PermissionsExt; - - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); - let gp = base.path().join("searchonly"); - let keys = gp.join("keys"); - std::fs::create_dir_all(&keys).unwrap(); - std::fs::set_permissions(&keys, std::fs::Permissions::from_mode(0o700)).unwrap(); - std::fs::set_permissions(&gp, std::fs::Permissions::from_mode(0o111)).unwrap(); - let key = keys.join("identity.pem"); - let created = match load_or_create_keypair_at(&key) { - Ok(kp) => kp, - Err(e) => { - let _ = std::fs::set_permissions(&gp, std::fs::Permissions::from_mode(0o700)); - panic!("search-only grandparent must be enough to publish into an existing 0700 key dir: {e:#}"); - } - }; - let reloaded = load_or_create_keypair_at(&key); - std::fs::set_permissions(&gp, std::fs::Permissions::from_mode(0o700)).unwrap(); - let reloaded = reloaded.expect("reload"); - assert_eq!(created.did(), reloaded.did()); - } - - /// `/identity.pem` is a pathname contract, not a filesystem probe: its - /// parent is the root, which the operator has already nominated, so - /// `create_pinned_dir_and_publish` must take the already-nominated branch. - /// Asserted lexically because a probe against the real `/` cannot be - /// isolated and, run as root, leaves a real key at `/identity.pem`. - #[cfg(unix)] - #[test] - fn identity_root_adjacent_path_is_an_already_nominated_parent() { - use std::path::{Component, Path}; - - let root_adjacent = Path::new("/identity.pem"); - - assert!( - !p2p::path_denotes_a_directory(root_adjacent, None), - "the lexical gate in load_or_create_keypair_at must admit /identity.pem" - ); - assert!( - !root_adjacent - .components() - .any(|c| c == Component::ParentDir), - "/identity.pem walks back out through no `..`" - ); - assert_eq!( - p2p::key_parent(root_adjacent), - Path::new("/"), - "the lexical parent of /identity.pem is the filesystem root" - ); - assert!( - p2p::identity_parent_is_already_nominated(root_adjacent), - "root-adjacent identity path must take the already-nominated branch, not the named-parent branch that fails with 'names no final directory component'" - ); - - for nominated in ["identity.pem", "./identity.pem"] { - assert!( - p2p::identity_parent_is_already_nominated(Path::new(nominated)), - "{nominated} publishes into the working directory, which is already nominated" - ); - } - - for named in [ - "/data/identity.pem", - "keys/identity.pem", - "/identity/identity.pem", - ] { - assert!( - !p2p::identity_parent_is_already_nominated(Path::new(named)), - "{named} names a parent component this process may create" - ); - } - } - - /// A terminal `.` is a directory spelling. Path drops it, so this would - /// otherwise publish a 0600 file named `keys` under `data`. - #[cfg(unix)] - #[test] - fn identity_terminal_dot_path_is_refused_without_retargeting() { - use std::os::unix::fs::PermissionsExt; - - let base = tempfile::tempdir().unwrap(); - std::fs::set_permissions(base.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); - let data = base.path().join("data"); - std::fs::create_dir(&data).unwrap(); - std::fs::set_permissions(&data, std::fs::Permissions::from_mode(0o755)).unwrap(); - let spelling = format!("{}/keys/.", data.display()); - let key = std::path::Path::new(&spelling); - let Err(err) = load_or_create_keypair_at(key) else { - panic!("a terminal `.` identity path must be refused"); - }; - let text = format!("{err:#}"); - assert!( - text.contains("must name a key file") || text.contains("directory"), - "the refusal must name the directory spelling, got: {text}" - ); - let mode = std::fs::symlink_metadata(&data) - .unwrap() - .permissions() - .mode() - & 0o7777; - assert_eq!(mode, 0o755, "rejection must not chmod the parent"); - assert!( - !data.join("keys").exists(), - "rejection must not create a file named keys" - ); - } +async fn main() -> anyhow::Result<()> { + gitlawb_node::run().await } diff --git a/crates/gitlawb-node/src/test_git_shim.rs b/crates/gitlawb-node/src/test_git_shim.rs index a53b05d64..41aebfb7d 100644 --- a/crates/gitlawb-node/src/test_git_shim.rs +++ b/crates/gitlawb-node/src/test_git_shim.rs @@ -6,6 +6,7 @@ use std::sync::OnceLock; /// The fixture delays or hangs in its own process, so the production watchdog /// can terminate it without depending on a shell or descendant process support. +#[allow(dead_code)] pub(super) enum Behavior<'a> { Delay(u64), Hang, @@ -14,6 +15,7 @@ pub(super) enum Behavior<'a> { LogTypes(&'a Path), } +#[allow(dead_code)] pub(super) struct GitShim { _directory: tempfile::TempDir, executable: PathBuf, @@ -27,6 +29,7 @@ impl Deref for GitShim { } } +#[allow(dead_code)] pub(super) fn create(name: &str, behavior: Behavior<'_>) -> GitShim { static COMPILED: OnceLock = OnceLock::new(); let compiled = COMPILED.get_or_init(|| { diff --git a/crates/gitlawb-node/src/test_harness.rs b/crates/gitlawb-node/src/test_harness.rs new file mode 100644 index 000000000..64da604be --- /dev/null +++ b/crates/gitlawb-node/src/test_harness.rs @@ -0,0 +1,601 @@ +//! Test-only spawn surface for the real-node deny harness (see +//! `tests/deny_harness.rs`). Feature-gated behind `test-harness` so it never +//! compiles into the production binary. It exposes a single boot constructor so +//! the out-of-crate integration test can bring up a real node over a bound TCP +//! socket without the integration crate needing access to `graphql`, +//! `rate_limit`, `repo_store`, or the other internals `AppState` is built from. +//! +//! The node is built the same way `test_support::build_state` builds it (p2p +//! disabled, real migrated pool), but bound on `127.0.0.1:0` and served through +//! the real `axum::serve` stack with connect-info, so the middleware order, +//! body limits, and per-IP rate limiters all run exactly as in production. + +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use chrono::Utc; +use clap::Parser; +use sqlx::PgPool; +use tokio::net::TcpListener; +use tokio::sync::watch; +use uuid::Uuid; + +use gitlawb_core::identity::Keypair; + +use crate::config::Config; +use crate::db::{Db, PullRequest, RepoRecord, VisibilityMode}; +use crate::rate_limit::{RateLimiter, TrustedProxy}; +use crate::state::AppState; + +/// A running node bound to an ephemeral port. End every test with +/// [`TestNode::shutdown`]: it joins the serve task and closes the pool, so +/// `#[sqlx::test]`'s `DROP DATABASE` cleanup (which runs the moment the test +/// future returns) never races the server's still-open connections. Dropping +/// without it signals shutdown, aborts any remaining serve task, and removes +/// the temp repository directory, a fallback for tests that return or panic +/// before reaching teardown. +pub struct TestNode { + /// Base URL of the running node, e.g. `http://127.0.0.1:54321`. + pub base_url: String, + /// The node's own DID (for building requests that reference it). + pub node_did: String, + shutdown_tx: watch::Sender, + repos_dir: PathBuf, + /// Seeding handle over the same pool the node serves from. Kept private so + /// the integration crate seeds through the methods below rather than + /// naming `Db`/`RepoRecord` directly. + db: Arc, + /// The detached `axum::serve` task, joined by [`Self::shutdown`]. `None` + /// once taken, so the `Drop` fallback never overlaps a completed teardown. + /// Carries `serve`'s own `io::Result<()>` so `shutdown()` can observe it + /// rather than discard it (see the coverage note there). + server: Option>>, + /// A clone of the served pool, kept so [`Self::shutdown`] can close it + /// (sqlx pool clones share one inner, so closing this closes the clones + /// inside `AppState`/`Db` too). + pool: PgPool, + /// Test-only: when set, `Drop` skips the graceful watch signal so ONLY + /// `handle.abort()` tears the serve task down. Defaults false, so every + /// other test keeps both redundant teardown mechanisms. Set via + /// [`Self::force_abort_only_teardown`] to fence that the abort alone + /// releases the database when the graceful chain is broken. + suppress_graceful_shutdown_on_drop: bool, +} + +impl Drop for TestNode { + fn drop(&mut self) { + // Fallback for tests that return or panic before reaching + // [`TestNode::shutdown`]: flip the shared shutdown signal so the serve + // task exits gracefully, abort whatever is left of it, and remove the + // temp repos dir. Drop cannot await, so it cannot join the server or + // close the pool; a test that already failed should not panic again in + // teardown. All three actions are idempotent or no-ops after + // `shutdown()` (which takes the handle), so running afterwards is + // harmless. + // + // The abort matters when the graceful chain does not run to completion + // before sqlx's `DROP DATABASE` cleanup: `JoinHandle::abort()` is a + // synchronous cancellation request callable from non-async Drop, and + // the aborted task's future (with the `PgPool` clones inside its + // router/state) is dropped on a subsequent scheduler tick, which sqlx + // cleanup's own awaits provide before the drop statement runs. The + // race is closed by those scheduling points, not by the abort alone. + // + // `suppress_graceful_shutdown_on_drop` (test-only) forces the abort to + // carry teardown by itself, so a test can fence that the abort line is + // load-bearing; it is false everywhere else. + if !self.suppress_graceful_shutdown_on_drop { + let _ = self.shutdown_tx.send(true); + } + if let Some(handle) = self.server.take() { + handle.abort(); + } + let _ = std::fs::remove_dir_all(&self.repos_dir); + } +} + +/// Outcome of joining the serve task during teardown, split so the wedged-task +/// path is observable to a test (which injects a short timeout) without waiting +/// the production teardown window. +enum ServeTeardown { + /// The task returned (carrying `serve`'s `io::Result<()>`) — either within + /// the timeout, or in the race between the timeout firing and the abort + /// landing. A task that actually finished is never a wedge. + Graceful(std::io::Result<()>), + /// The task did not finish within the timeout and was aborted (cancelled) + /// and reaped. + TimedOutAborted, + /// The serve task panicked. The payload is carried out so the caller can + /// resume unwinding only AFTER the pool is closed and the temp dir removed. + Panicked(Box), +} + +/// Join `handle` within `timeout`; on expiry `abort()` it and await the aborted +/// handle so the serve task (and the `PgPool` clones inside its router/state) is +/// reaped BEFORE the caller returns and `#[sqlx::test]` runs `DROP DATABASE`. +/// Takes `&mut` (a `JoinHandle` is `Unpin`) so ownership is retained across the +/// timeout and the handle can be aborted on expiry — the previous code moved the +/// handle into `timeout` and, on elapse, dropped it, which DETACHES the still +/// running task rather than aborting it, leaking the test database (#194 F2). +async fn join_or_abort( + handle: &mut tokio::task::JoinHandle>, + timeout: Duration, +) -> ServeTeardown { + match tokio::time::timeout(timeout, &mut *handle).await { + // A panic in the serve task is carried, not unwound here: the caller + // must close the pool and remove the repos dir before resuming it. + Ok(join_result) => match join_result { + Ok(result) => ServeTeardown::Graceful(result), + Err(e) => ServeTeardown::Panicked(e.into_panic()), + }, + Err(_elapsed) => { + handle.abort(); + match handle.await { + // The task finished in the race between the timeout firing and + // the abort landing — honor its result, not a false wedge. + Ok(result) => ServeTeardown::Graceful(result), + Err(e) if e.is_cancelled() => ServeTeardown::TimedOutAborted, + Err(e) => ServeTeardown::Panicked(e.into_panic()), + } + } + } +} + +/// Allocate a process-unique temp directory for a spawned node's repositories +/// without pulling in the `tempfile` dev-dependency (which is unavailable to +/// the library crate under `--features test-harness`). +fn unique_repos_dir() -> PathBuf { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + std::env::temp_dir().join(format!("gitlawb-deny-harness-{}-{}", std::process::id(), n)) +} + +/// Build an [`AppState`] over the given migrated pool, mirroring +/// `test_support::build_state` but with a real on-disk repository directory so +/// git smart-HTTP routes can serve. P2P is always disabled. +fn build_state(db: Arc, pool: PgPool, repos_dir: PathBuf) -> AppState { + let keypair = Keypair::generate(); + let scan_token_key = crate::state::AppState::derive_scan_token_key(&keypair); + let webhook_secret_key = crate::state::AppState::derive_webhook_secret_key(&keypair); + let task_cursor_key = crate::api::task_cursor::TaskCursorKey::derive(&keypair.to_seed()); + let node_did = keypair.did(); + let (ref_tx, _) = tokio::sync::broadcast::channel(1); + let (task_tx, _) = tokio::sync::broadcast::channel(1); + let schema = Arc::new(crate::graphql::build_schema( + db.clone(), + ref_tx.clone(), + task_tx.clone(), + task_cursor_key.clone(), + )); + AppState { + config: Arc::new(Config::parse_from(["gitlawb-node"])), + db, + node_did, + node_keypair: Arc::new(keypair), + p2p: None, + http_client: Arc::new(reqwest::Client::new()), + ref_update_tx: ref_tx, + task_event_tx: task_tx, + graphql_schema: schema, + task_cursor_key, + machine_id: None, + repo_store: crate::git::repo_store::RepoStore::for_testing(repos_dir, pool), + rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), + create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), + push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + close_issue_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + federated_rate_limiter: RateLimiter::new(12, Duration::from_secs(60)), + task_read_rate_limiter: RateLimiter::new(1200, Duration::from_secs(3600)), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_work_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_max_history_walks: crate::api::ipfs::MAX_HISTORY_WALKS_PER_REQUEST, + ipfs_max_legacy_probes: crate::api::ipfs::MAX_LEGACY_PROBES_PER_REQUEST, + ipfs_legacy_scan_page_rows: crate::api::ipfs::LEGACY_SCAN_PAGE_ROWS, + ipfs_max_legacy_scan_rows: crate::api::ipfs::MAX_LEGACY_SCAN_ROWS_PER_REQUEST, + ipfs_max_legacy_scan_rule_bytes: crate::api::ipfs::MAX_LEGACY_SCAN_RULE_BYTES_PER_REQUEST, + ipfs_scan_token_key: Arc::new(scan_token_key), + webhook_secret_key: Arc::new(webhook_secret_key), + ipfs_max_served_object_bytes: crate::api::ipfs::MAX_SERVED_OBJECT_BYTES, + push_limiter_trust: TrustedProxy::None, + sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), + peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + shutdown_tx: watch::channel(false).0, + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + encrypt_inflight: crate::state::EncryptInflight::new(), + repo_write_leases: crate::state::RepoWriteLeases::new(8), + git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( + 8, + ), + git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( + 16, + ), + git_bin: "git".to_string(), + } +} + +/// Spawn a real node bound to `127.0.0.1:0` over the given (already-created, +/// empty) test pool. Runs the schema migrations, builds the router, and serves +/// it on a background task through the production `axum::serve` stack with +/// connect-info so per-IP layers key on the real peer. Returns once the socket +/// is bound and accepting connections. +pub async fn spawn_node(pool: PgPool) -> TestNode { + let db = Arc::new(crate::db::Db::for_testing(pool.clone())); + db.run_migrations() + .await + .expect("test schema migrations should apply"); + + let repos_dir = unique_repos_dir(); + std::fs::create_dir_all(&repos_dir).expect("create temp repos dir"); + + let state = build_state(db.clone(), pool.clone(), repos_dir.clone()); + let node_did = state.node_did.to_string(); + let shutdown_tx = state.shutdown_tx.clone(); + let mut shutdown_rx = shutdown_tx.subscribe(); + + let router = crate::server::build_router(state); + + let listener = TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("bind ephemeral port"); + let addr = listener.local_addr().expect("read bound addr"); + + let server = tokio::spawn(async move { + axum::serve( + listener, + router.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(async move { + let _ = shutdown_rx.changed().await; + }) + .await + }); + + TestNode { + base_url: format!("http://{addr}"), + node_did, + shutdown_tx, + repos_dir, + db, + server: Some(server), + pool, + suppress_graceful_shutdown_on_drop: false, + } +} + +impl TestNode { + /// Test-only: make `Drop` tear the serve task down through `handle.abort()` + /// alone by suppressing the graceful watch signal. Used to prove the abort + /// is load-bearing (that the database is released even when the graceful + /// chain is broken); no production or normal-teardown path calls this. + pub fn force_abort_only_teardown(&mut self) { + self.suppress_graceful_shutdown_on_drop = true; + } + + /// Graceful async teardown: signal shutdown, join the serve task, close + /// the pool, remove the temp repos dir. Call this at the end of every + /// test. `#[sqlx::test]` issues `DROP DATABASE` as soon as the test + /// future returns, so the server (which owns pool clones inside + /// `AppState`/`Db`) must be gone and the pool closed before then, or the + /// cleanup fails and leaks per-test databases. A serve task that does not + /// exit within 10s panics loudly: a wedged graceful shutdown must fail + /// the test, not hang CI. + /// + /// When `self` drops on return, `Drop` re-sends the signal and re-removes + /// the dir; both are idempotent, and the serve handle was already taken, + /// so the overlap is harmless. + pub async fn shutdown(self) { + self.shutdown_with_timeout(Duration::from_secs(10)).await; + } + + /// Body of [`Self::shutdown`] with an injectable teardown timeout so a test + /// can drive the wedged-shutdown path without waiting the production 10s. + /// Production always calls this with 10s. + async fn shutdown_with_timeout(mut self, timeout: Duration) { + let _ = self.shutdown_tx.send(true); + let outcome = match self.server.take() { + Some(mut handle) => Some(join_or_abort(&mut handle, timeout).await), + None => None, + }; + // Release the pool and temp dir on EVERY path before returning OR + // panicking. A wedged graceful shutdown is exactly when in-flight + // per-connection tasks still hold `PgPool` clones; aborting the serve + // future alone does not reliably drop them, so the explicit + // `pool.close()` (which closes the shared inner) is what frees them + // before `#[sqlx::test]` runs `DROP DATABASE`. Panicking before this + // (the previous behavior) skipped the close on the one path that needs + // it most (#194 F2). + self.pool.close().await; + let _ = std::fs::remove_dir_all(&self.repos_dir); + match outcome { + // Honest coverage contract: on axum 0.8.8, + // `WithGracefulShutdown::into_future` always resolves `Ok(())` (the + // shutdown signal is treated as success and `accept()` errors are + // retried forever rather than returned), so this `Err` arm is + // unreachable today - every existing harness test exercises only the + // `Ok` path here. Forward-compat insurance for a future axum where + // `serve` can fail: that should surface as a clear panic, not be + // silently discarded. + Some(ServeTeardown::Graceful(result)) => { + result.expect("axum::serve exited with an error"); + } + Some(ServeTeardown::TimedOutAborted) => { + panic!( + "serve task did not exit within {timeout:?} of the shutdown \ + signal (wedged graceful shutdown)" + ); + } + Some(ServeTeardown::Panicked(payload)) => std::panic::resume_unwind(payload), + None => {} + } + } + + /// Insert a repo owned by `owner_did` and return its repo id. Mirrors + /// `test_support::seed_repo` (the `disk_path` field is unused by the + /// integration path — `RepoStore` computes the on-disk path from + /// `repos_dir`/owner/name, see [`Self::seed_bare_repo`]). + pub async fn seed_repo(&self, owner_did: &str, name: &str, is_public: bool) -> String { + let now = Utc::now(); + let record = RepoRecord { + id: Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: owner_did.to_string(), + description: None, + is_public, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/tmp/{name}"), + forked_from: None, + machine_id: None, + }; + self.db.create_repo(&record).await.expect("seed repo"); + record.id + } + + /// Mark `branch` as protected on `repo_id`, so a non-owner push to it is + /// rejected by the branch-protection gate (used by the deny-prober's + /// protected-push probe — #195, F3). + pub async fn seed_protected_branch(&self, repo_id: &str, branch: &str, created_by: &str) { + self.db + .protect_branch(repo_id, branch, created_by) + .await + .expect("seed protected branch"); + } + + /// Seed an open PR `number` in `repo_id`, authored by `author_did`. The + /// author-or-owner close gate (`close_pr`) loads the PR *before* it runs, so a + /// deny-prober row for `.../pulls/{number}/close` needs the PR to exist or a + /// stranger gets a 404 (absent entity) instead of the 403 the gate emits. + pub async fn seed_pr(&self, repo_id: &str, number: i64, author_did: &str) { + let now = Utc::now().to_rfc3339(); + let pr = PullRequest { + id: Uuid::new_v4().to_string(), + repo_id: repo_id.to_string(), + number, + title: "seed pr".to_string(), + body: None, + author_did: author_did.to_string(), + source_branch: "feature".to_string(), + target_branch: "main".to_string(), + status: "open".to_string(), + merged_by_did: None, + merged_at: None, + created_at: now.clone(), + updated_at: now, + }; + self.db.create_pr(&pr).await.expect("seed pr"); + } + + /// Seed issue `issue_id` (a git-ref JSON blob authored by `author_did`) in the + /// repo's bare on-disk store. Like [`Self::seed_pr`], the `close_issue` + /// owner-or-author gate loads the issue before it runs, so the deny-prober row + /// for `.../issues/{id}/close` needs the issue present to reach the 403. + pub fn seed_issue(&self, owner_did: &str, name: &str, issue_id: &str, author_did: &str) { + let slug = owner_did.replace([':', '/'], "_"); + let bare = self.repos_dir.join(&slug).join(format!("{name}.git")); + let json = format!(r#"{{"author":"{author_did}"}}"#); + crate::git::issues::create_issue(&bare, issue_id, &json).expect("seed issue"); + } + + /// Add a path-scoped visibility rule restricting `path_glob` to + /// `reader_dids` (empty = only the owner). Mode B keeps object SHAs intact + /// and withholds blob content, which is what the read/replication gates + /// enforce. + pub async fn withhold_path( + &self, + repo_id: &str, + path_glob: &str, + reader_dids: &[String], + created_by: &str, + ) { + self.db + .set_visibility_rule( + repo_id, + path_glob, + VisibilityMode::B, + reader_dids, + created_by, + ) + .await + .expect("set visibility rule"); + } + + /// Record `pinned_cids` rows for every blob OID returned by + /// [`Self::seed_bare_repo`] so `/ipfs/{cid}` can resolve via provenance on + /// current main (the table is the primary lookup path). CIDs are computed + /// from raw object bytes exactly as the production pin path does + /// (`Cid::from_git_object_bytes`), not from the git oid hash. + /// Returns each seeded path's canonical CID for use in `GET /ipfs/{cid}`. + pub async fn seed_pinned_cids_for_repo( + &self, + owner_did: &str, + repo_name: &str, + repo_id: &str, + oids: &std::collections::HashMap, + ) -> std::collections::HashMap { + let slug = owner_did.replace([':', '/'], "_"); + let bare = self.repos_dir.join(&slug).join(format!("{repo_name}.git")); + let mut cids = std::collections::HashMap::new(); + for (path, oid_hex) in oids { + if path == "HEAD" { + continue; + } + let (_ty, raw) = crate::git::store::read_object(&bare, oid_hex) + .expect("read seeded object") + .expect("seeded object exists in bare repo"); + let preliminary = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + let cid = preliminary + .parse::>() + .expect("parse production pin cid") + .to_string(); + self.db + .record_pinned_cid_with_source(oid_hex, &cid, repo_id) + .await + .expect("seed pinned_cids row"); + let roundtrip = self + .db + .oids_for_cid(&cid) + .await + .expect("oids_for_cid after seed"); + assert!( + !roundtrip.is_empty(), + "pinned_cids seed for {path} did not round-trip cid {cid}" + ); + cids.insert(path.clone(), cid); + } + cids + } + + /// Create a real bare git repository on disk at the exact path + /// `RepoStore::acquire` reads (`//.git`), with + /// one commit on `main` containing `files` (`(path, contents)`). Returns the + /// blob OID of each path so callers can assert a withheld OID never appears + /// in served bytes (U8). Shells out to `git`, mirroring + /// `test_support`'s served-content seam. + /// `object_format` is `"sha1"` for the git smart-HTTP surfaces and + /// `"sha256"` for the content-addressed `/ipfs/{cid}` surface (whose CIDs + /// are the sha2-256 object ids). + pub fn seed_bare_repo( + &self, + owner_did: &str, + name: &str, + files: &[(&str, &str)], + object_format: &str, + ) -> std::collections::HashMap { + let run = |args: &[&str], cwd: &std::path::Path| { + // Test-harness-only fixture seeding, feature-gated out of the production + // binary; runs local git against a tempdir inside a test's own lifetime. + // allow-unbounded-git: test-only seeding, never on a request path holding a permit + let out = std::process::Command::new("git") + .args(args) + .current_dir(cwd) + .output() // allow-unbounded-git: test-harness fixture seeding only + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Build a source work tree, then bare-clone it into the served path. + let src = self.repos_dir.join(format!("src-{name}")); + std::fs::create_dir_all(&src).expect("create src dir"); + let fmt_arg = format!("--object-format={object_format}"); + run(&["init", "-q", "-b", "main", &fmt_arg], &src); + run(&["config", "user.email", "t@t"], &src); + run(&["config", "user.name", "t"], &src); + for (path, contents) in files { + let full = src.join(path); + if let Some(parent) = full.parent() { + std::fs::create_dir_all(parent).expect("create file parent dir"); + } + std::fs::write(&full, contents).expect("write seed file"); + run(&["add", path], &src); + } + run(&["commit", "-q", "-m", "seed"], &src); + + let mut oids = std::collections::HashMap::new(); + for (path, _) in files { + let oid = run(&["rev-parse", &format!("HEAD:{path}")], &src); + oids.insert((*path).to_string(), oid); + } + // The HEAD commit oid, used as the `want` when driving upload-pack. + oids.insert("HEAD".to_string(), run(&["rev-parse", "HEAD"], &src)); + + let slug = owner_did.replace([':', '/'], "_"); + let bare = self.repos_dir.join(&slug).join(format!("{name}.git")); + std::fs::create_dir_all(bare.parent().unwrap()).expect("create bare parent"); + run( + &[ + "clone", + "--bare", + "-q", + src.to_str().unwrap(), + bare.to_str().unwrap(), + ], + &self.repos_dir, + ); + + oids + } +} + +#[cfg(test)] +mod tests { + use super::{join_or_abort, ServeTeardown}; + use std::time::Duration; + + /// A wedged serve task (never returns within the window) must be ABORTED and + /// reaped on timeout, not detached. The previous teardown moved the + /// `JoinHandle` into `timeout` and dropped it on elapse, which detaches the + /// still-running task and leaks the test DB across `DROP DATABASE` (#194 F2). + /// Load-bearing: neutralize the abort in `join_or_abort` (return without + /// abort+await) and this flips RED — `handle.is_finished()` stays false + /// because the task was left running. + #[tokio::test] + async fn join_or_abort_aborts_a_wedged_task_on_timeout() { + let mut handle: tokio::task::JoinHandle> = tokio::spawn(async { + loop { + tokio::time::sleep(Duration::from_secs(3600)).await; + } + }); + assert!( + matches!( + join_or_abort(&mut handle, Duration::from_millis(50)).await, + ServeTeardown::TimedOutAborted + ), + "a task that never returns must be reported timed-out, not graceful" + ); + assert!( + handle.is_finished(), + "the wedged serve task must be aborted and reaped, not detached" + ); + } + + /// A task that returns within the window is reported graceful, carrying its + /// `io::Result` — the normal teardown path. + #[tokio::test] + async fn join_or_abort_reports_graceful_when_task_exits_in_time() { + let mut handle: tokio::task::JoinHandle> = + tokio::spawn(async { Ok(()) }); + assert!( + matches!( + join_or_abort(&mut handle, Duration::from_secs(5)).await, + ServeTeardown::Graceful(Ok(())) + ), + "a prompt clean exit must be reported graceful" + ); + } +} diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 9e927e237..66926209c 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -54,6 +54,7 @@ pub(crate) async fn test_state(pool: PgPool) -> AppState { /// `config::tests::enforce_owner_push_is_declared_true_independent_of_the_environment`, /// which reads the declaration off the parser. Splitting the two keeps a parser /// question and an authorization question from sharing one failure mode. +#[allow(dead_code)] pub(crate) async fn test_state_with( pool: PgPool, configure: impl FnOnce(&mut crate::config::Config), @@ -163,6 +164,7 @@ pub(crate) async fn app(pool: PgPool) -> Router { /// Sets `Content-Type: application/json` — the API is JSON throughout, and /// without it axum's `Json` extractor returns 415 before the handler runs /// (which would make any JSON-body authz assertion a false pass). +#[allow(dead_code)] pub(crate) fn signed_request_as(did: &str, method: Method, uri: &str, body: Body) -> Request { Request::builder() .method(method) @@ -180,6 +182,7 @@ pub(crate) fn signed_request_as(did: &str, method: Method, uri: &str, body: Body /// test pass for the wrong reason.) The accepted sockets are parked in the /// spawned task, which dies with the test's runtime, so the peer never sees a /// close mid-test. +#[allow(dead_code)] pub(crate) async fn silent_http_endpoint() -> String { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let endpoint = format!("http://{}", listener.local_addr().unwrap()); diff --git a/crates/gitlawb-node/tests/deny_harness.rs b/crates/gitlawb-node/tests/deny_harness.rs new file mode 100644 index 000000000..5da981abd --- /dev/null +++ b/crates/gitlawb-node/tests/deny_harness.rs @@ -0,0 +1,2881 @@ +//! Real-node deny harness: boots a real gitlawb-node over a bound TCP socket +//! and drives trust-boundary DENY paths through a real reqwest client, +//! asserting both the refusal status and that no withheld data leaks +//! (INV-1/INV-2/INV-8). Requires `--features test-harness`. +//! +//! Each `#[sqlx::test]` gets an ephemeral per-test database; `spawn_node` runs +//! the schema migrations and serves the real router on `127.0.0.1:0`. Every +//! test ends with `node.shutdown().await` so the serve task is joined and the +//! pool closed before sqlx's `DROP DATABASE` cleanup runs (except the Drop +//! regression test at the bottom, which deliberately skips it). + +mod support; + +use std::process::Command; + +use support::assert::assert_denied; +use support::probe::{probes_for, Expect, Fixture, Probe, Signer}; +use support::routes::{deny_bearing_routes, GateClass}; +use support::signing::signed_request; + +use gitlawb_core::identity::Keypair; +use gitlawb_node::test_harness::spawn_node; + +/// A reqwest client with a bounded timeout so a wedged node route fails the test +/// fast rather than hanging until the CI job timeout. +fn bounded_client() -> reqwest::Client { + reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .expect("client builds") +} + +// ── U2: the signing client produces signatures require_signature accepts ───── + +/// A validly signed receive-pack request clears `require_signature` (it does not +/// get a 401): it proceeds past the signature layer and is denied later for a +/// different reason (the repo does not exist). This proves the signing client +/// is producing signatures the real verifier accepts over the socket. +#[sqlx::test] +async fn signed_receive_pack_clears_signature_layer(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = bounded_client(); + let kp = Keypair::generate(); + + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + "/alice/repo/git-receive-pack", + b"0000".to_vec(), + &kp, + ) + .send() + .await + .expect("request sends"); + + assert_ne!( + resp.status().as_u16(), + 401, + "a valid signature must clear require_signature; got 401" + ); + + node.shutdown().await; +} + +/// Tampering the body after signing invalidates the content-digest, so the +/// server rejects with 400 `content_digest_mismatch` (distinct from the 401 a +/// missing/invalid signature gets). Proves the signature actually covers the +/// body: the digest is signed and the server re-checks it against the bytes it +/// received. +#[sqlx::test] +async fn tampered_body_after_signing_is_rejected(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = bounded_client(); + let kp = Keypair::generate(); + + // Sign one body, then replace it before sending. + let mut req = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + "/alice/repo/git-receive-pack", + b"0000".to_vec(), + &kp, + ) + .build() + .expect("build request"); + *req.body_mut() = Some(reqwest::Body::from(b"tampered".to_vec())); + + let resp = client.execute(req).await.expect("request sends"); + assert_eq!( + resp.status().as_u16(), + 400, + "a body that no longer matches its content-digest must be rejected" + ); + + node.shutdown().await; +} + +// ── U5(a): INV-8 — an unsigned push is denied and leaks nothing ────────────── + +/// An unauthenticated git-receive-pack (no signature headers) is rejected with +/// 401 before any handler runs, and the denial body carries no repo internals. +#[sqlx::test] +async fn unsigned_receive_pack_is_denied(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = bounded_client(); + + let url = format!("{}/alice/repo/git-receive-pack", node.base_url); + let resp = client + .post(&url) + .header("content-type", "application/x-git-receive-pack-request") + .body(b"0000".to_vec()) + .send() + .await + .expect("request sends"); + + // No repo was seeded, so there are no OIDs to leak; the assertion still + // enforces the 4xx-and-not-empty-200 INV-8 shape. + assert_denied(resp, 401, &[]).await; + + node.shutdown().await; +} + +/// Build a minimal git-receive-pack request body: one ref-update pkt-line for +/// `refs/heads/` (dummy 40-hex old/new SHAs — the branch-protection gate +/// only reads the ref NAME) plus a flush. Enough to reach the ref-update parse +/// and the protection gate, not a real pack. +fn receive_pack_update_body(branch: &str) -> Vec { + let old = "0".repeat(40); + let new = "1".repeat(40); + let line = format!("{old} {new} refs/heads/{branch}\0report-status\n"); + let pkt = format!("{:04x}{line}", line.len() + 4); + let mut body = pkt.into_bytes(); + body.extend_from_slice(b"0000"); + body +} + +/// #195 (F3): a signed NON-OWNER pushing to a PROTECTED branch is forbidden (403) +/// by the branch-protection gate (`repos.rs`), and the owner pushing to the same +/// branch is NOT blocked (control). git_receive_pack's registry row only drives the +/// unsigned-401 signature path, so without this probe inverting the 403 leaves the +/// sweep AND the completeness scan green. Drives the 403 so the gate can't rot. +#[sqlx::test] +async fn signed_stranger_protected_branch_push_is_forbidden(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + // #195 (F1): a bounded timeout so a wedged git-receive-pack fails the suite + // rather than hanging it until CI kills the job. + let client = support::bounded_client(); + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let stranger = Keypair::generate(); + + let repo_id = node.seed_repo(&owner_did, "protrepo", true).await; + node.seed_protected_branch(&repo_id, "main", &owner_did) + .await; + + let path = format!("/{owner_did}/protrepo/git-receive-pack"); + let body = receive_pack_update_body("main"); + + // Signed non-owner -> 403 branch protection; the denial leaks no repo internals. + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body.clone(), + &stranger, + ) + .send() + .await + .expect("request sends"); + assert_eq!( + resp.status().as_u16(), + 403, + "a signed non-owner push to a protected branch must be forbidden (403)" + ); + assert_denied(resp, 403, &[repo_id.as_str()]).await; + + // Owner control: the owner is NOT blocked by branch protection (it may fail + // later on the dummy pack, but must not be the 403 the stranger got). + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body, + &owner, + ) + .send() + .await + .expect("request sends"); + assert_ne!( + resp.status().as_u16(), + 403, + "the owner must not be blocked by their own branch protection (control)" + ); + + node.shutdown().await; +} + +// ── U6(b): INV-1 — a validly signed NON-owner push to an UNprotected branch is +// owner-gated (403) by enforce_owner_push, not by branch protection ───────── + +/// `enforce_owner_push` (on by default) rejects a signed non-owner before +/// branch protection runs, on an unprotected branch where the protected-branch +/// gate does not apply. Without this probe the registry sweep and completeness +/// scan stay green while the owner-push wiring rots, because git_receive_pack's +/// registry row only drives the unsigned-401 signature path. Drives the 403 so +/// the gate cannot regress silently. +#[sqlx::test] +async fn signed_stranger_push_to_unprotected_branch_is_forbidden(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = support::bounded_client(); + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let stranger = Keypair::generate(); + + // Unprotected repo: no branch protection, so the 403 can only come from + // enforce_owner_push, not from the protected-branch gate. + let repo_id = node.seed_repo(&owner_did, "pushrepo", true).await; + + let path = format!("/{owner_did}/pushrepo/git-receive-pack"); + let body = receive_pack_update_body("main"); + + // Signed non-owner -> 403 from enforce_owner_push; leaks no repo internals. + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body.clone(), + &stranger, + ) + .send() + .await + .expect("request sends"); + assert_eq!( + resp.status().as_u16(), + 403, + "a signed non-owner push to an unprotected branch must be forbidden (403) by enforce_owner_push" + ); + assert_denied(resp, 403, &[repo_id.as_str()]).await; + + // Owner control: the owner is NOT blocked by enforce_owner_push (it may fail + // later on the dummy pack, but must not be the 403 the stranger got). + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body, + &owner, + ) + .send() + .await + .expect("request sends"); + assert_ne!( + resp.status().as_u16(), + 403, + "the owner must not be blocked by their own enforce_owner_push gate (control)" + ); + + node.shutdown().await; +} + +// ── U5(b): INV-8/INV-2 — anonymous /ipfs/{cid} of a withheld blob is denied ── + +/// A public repo with a `/secret/**` withhold rule (readers = one allowed DID). +/// An anonymous content-addressed read of the withheld blob's CID is denied +/// (404) and leaks neither the secret bytes nor its OID; the sibling public +/// blob's CID is served anonymously, proving the withhold is blob-scoped. +#[sqlx::test] +async fn anon_ipfs_read_of_withheld_blob_is_denied(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = bounded_client(); + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let stranger = Keypair::generate(); + + let repo_id = node.seed_repo(&owner_did, "u5b-repo", true).await; + // sha256 object format: the /ipfs CID is the sha2-256 object id. + let oids = node.seed_bare_repo( + &owner_did, + "u5b-repo", + &[ + ("public/a.txt", "public bytes U5b"), + ("secret/b.txt", "TOPSECRET-U5b"), + ], + "sha256", + ); + let cids = node + .seed_pinned_cids_for_repo(&owner_did, "u5b-repo", &repo_id, &oids) + .await; + node.withhold_path( + &repo_id, + "/secret/**", + &[reader.did().to_string()], + &owner_did, + ) + .await; + + let secret_oid = oids["secret/b.txt"].clone(); + let secret_cid = cids["secret/b.txt"].clone(); + let public_cid = cids["public/a.txt"].clone(); + + // Public blob first (blob-scoped withhold must not poison sibling reads). + let resp = client + .get(format!("{}/ipfs/{public_cid}", node.base_url)) + .send() + .await + .expect("request sends"); + let public_status = resp.status().as_u16(); + let public_body = resp.text().await.unwrap_or_default(); + assert_eq!( + public_status, 200, + "public blob CID must be served to anon; body={public_body:?}" + ); + assert!( + public_body.contains("public bytes U5b"), + "the public blob content is returned" + ); + + // Signed non-reader on the withheld blob's CID: denied, no leak. + let secret_path = format!("/ipfs/{secret_cid}"); + let resp = signed_request( + &client, + reqwest::Method::GET, + &node.base_url, + &secret_path, + vec![], + &stranger, + ) + .send() + .await + .expect("request sends"); + assert_denied( + resp, + 404, + &["TOPSECRET-U5b", &secret_oid, &secret_oid[..12]], + ) + .await; + + // Allowlisted reader can fetch the withheld blob. + let resp = signed_request( + &client, + reqwest::Method::GET, + &node.base_url, + &secret_path, + vec![], + &reader, + ) + .send() + .await + .expect("request sends"); + let reader_status = resp.status().as_u16(); + let reader_body = resp.text().await.unwrap_or_default(); + assert_eq!( + reader_status, 200, + "allowlisted reader must be served the withheld blob; body={reader_body:?}" + ); + assert!( + reader_body.contains("TOPSECRET-U5b"), + "the withheld blob content is returned to the reader" + ); + + // Anonymous read of the withheld blob's CID: denied, no leak of content or OID. + let resp = support::anon_get(&client, &node.base_url, &secret_path) + .send() + .await + .expect("request sends"); + assert_denied( + resp, + 404, + &["TOPSECRET-U5b", &secret_oid, &secret_oid[..12]], + ) + .await; + + node.shutdown().await; +} + +// ── U6: INV-1 — a validly signed NON-owner mutation is owner-gated (403) ────── + +/// The case the in-crate `oneshot` suite cannot reach over the full stack: a +/// fully valid RFC-9421 signature from a non-owner DID hits a mutation and is +/// rejected by `require_owner` with 403, not merely "not authenticated". The +/// non-owner sends no `x-ucan` header, so `require_ucan_chain` passes through +/// and the request reaches the owner gate. +#[sqlx::test] +async fn wrong_owner_visibility_put_is_forbidden(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = bounded_client(); + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let stranger = Keypair::generate(); + + node.seed_repo(&owner_did, "harness-repo", true).await; + + let path = format!("/api/v1/repos/{owner_did}/harness-repo/visibility"); + let body = br#"{"path_glob":"/","reader_dids":[]}"#.to_vec(); + + // Non-owner: valid signature, wrong identity -> 403 from the owner gate. + let resp = signed_request( + &client, + reqwest::Method::PUT, + &node.base_url, + &path, + body.clone(), + &stranger, + ) + .header("content-type", "application/json") + .send() + .await + .expect("request sends"); + assert_denied(resp, 403, &[]).await; + + // Reachability proof: the same request signed by the OWNER is not 403 (it + // reaches the handler). Without this, a 403 produced by an earlier layer or + // a mis-seeded repo would masquerade as a passing INV-1 case. + let resp = signed_request( + &client, + reqwest::Method::PUT, + &node.base_url, + &path, + body, + &owner, + ) + .header("content-type", "application/json") + .send() + .await + .expect("request sends"); + assert!( + resp.status().is_success(), + "owner's signed visibility PUT must reach the handler, got {}", + resp.status() + ); + + node.shutdown().await; +} + +// ── U7: INV-2 — a read over a withheld path is denied and leaks nothing ─────── + +/// A public repo with a path-scoped withhold rule on `/secret/**`. An anonymous +/// blob read of the withheld path is denied (404 RepoNotFound) and the body +/// carries neither the secret content nor its blob OID; a read of a sibling +/// public path succeeds, proving the gate is path-scoped, not a blanket refusal. +#[sqlx::test] +async fn withheld_path_blob_read_is_denied(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = bounded_client(); + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + + let repo_id = node.seed_repo(&owner_did, "u7-repo", true).await; + let oids = node.seed_bare_repo( + &owner_did, + "u7-repo", + &[ + ("public.txt", "hello public"), + ("secret/data.txt", "TOPSECRET-CONTENT-U7"), + ], + "sha1", + ); + node.withhold_path(&repo_id, "/secret/**", &[], &owner_did) + .await; + + let secret_oid = oids["secret/data.txt"].clone(); + let short_oid = &secret_oid[..12]; + + // Withheld path: denied, and neither the content nor the OID (full or short) + // may appear in the denial body. + let resp = client + .get(format!( + "{}/api/v1/repos/{owner_did}/u7-repo/blob/secret/data.txt", + node.base_url + )) + .send() + .await + .expect("request sends"); + assert_denied(resp, 404, &["TOPSECRET-CONTENT-U7", &secret_oid, short_oid]).await; + + // Sibling public path: served, proving the withhold is path-scoped. + let resp = client + .get(format!( + "{}/api/v1/repos/{owner_did}/u7-repo/blob/public.txt", + node.base_url + )) + .send() + .await + .expect("request sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "public sibling path must be served" + ); + assert!( + resp.text().await.unwrap().contains("hello public"), + "the public blob content is returned" + ); + + node.shutdown().await; +} + +// ── U3: get_pr_diff SECOND (per-path) gate — a diff touching a withheld path ── + +/// Build a git v0 receive-pack body creating `refs/heads/` at a fresh +/// commit that adds `files` on top of the existing `main`, force-updating main to +/// a deterministic base so `` shares history with main. Returns the body +/// bytes plus a map of each pushed file's blob OID (full sha1), so the caller can +/// withhold the OID of a pushed secret path (#195, F2). Shells out to the local +/// `git`. +fn build_branch_push_body( + server_main_tip: &str, + branch: &str, + files: &[(&str, &str)], +) -> (Vec, std::collections::HashMap) { + use std::io::Write; + use std::process::{Command, Stdio}; + + let work = std::env::temp_dir().join(format!( + "gl-u3-prdiff-{}-{}", + std::process::id(), + server_main_tip + )); + let _ = std::fs::remove_dir_all(&work); + std::fs::create_dir_all(&work).unwrap(); + let run = |args: &[&str]| -> String { + let out = Command::new("git") + .args(args) + .current_dir(&work) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + // Pin sha1 to match the served fixture repo; init.defaultObjectFormat=sha256 breaks the ref update. + run(&["init", "-q", "-b", "main", "--object-format=sha1"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + std::fs::write(work.join("base.txt"), "prdiff base").unwrap(); + run(&["add", "base.txt"]); + run(&["commit", "-q", "-m", "base"]); + let new_main = run(&["rev-parse", "HEAD"]); + run(&["checkout", "-q", "-b", branch]); + for (p, c) in files { + let full = work.join(p); + if let Some(parent) = full.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(&full, c).unwrap(); + run(&["add", p]); + } + run(&["commit", "-q", "-m", "feature"]); + let feature_tip = run(&["rev-parse", "HEAD"]); + + // #195 (F2): capture each pushed file's blob OID (full sha1). seed_bare_repo + // never seeds these pushed paths, so this is the only place their OID is known; + // the PR-diff denial withholds the secret path's OID (full + [..12] short form). + let mut pushed_oids = std::collections::HashMap::new(); + for (p, _) in files { + pushed_oids.insert((*p).to_string(), run(&["rev-parse", &format!("HEAD:{p}")])); + } + + let child = Command::new("git") + .args(["pack-objects", "--stdout", "--revs"]) + .current_dir(&work) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .as_ref() + .unwrap() + .write_all(format!("{new_main}\n{feature_tip}\n").as_bytes()) + .unwrap(); + let pack = child.wait_with_output().unwrap().stdout; + + let zero = "0".repeat(server_main_tip.len()); + let l1 = format!("{server_main_tip} {new_main} refs/heads/main\0report-status\n"); + let l2 = format!("{zero} {feature_tip} refs/heads/{branch}\n"); + let mut body: Vec = Vec::new(); + body.extend(format!("{:04x}{l1}", l1.len() + 4).into_bytes()); + body.extend(format!("{:04x}{l2}", l2.len() + 4).into_bytes()); + body.extend_from_slice(b"0000"); + body.extend_from_slice(&pack); + let _ = std::fs::remove_dir_all(&work); + (body, pushed_oids) +} + +/// get_pr_diff has a SECOND, path-scoped gate after the repo-root read: it +/// iterates the diff's touched paths and 404s if any is withheld (pulls.rs +/// visibility_check loop). On a PUBLIC repo with a `/secret/**` withhold rule and a +/// PR whose diff touches `secret/x.txt`, an anon caller PASSES the public repo-root +/// gate but must be DENIED by the per-path gate (404), leaking neither the withheld +/// content nor its OID; the owner sees the full diff. This drives the per-path gate +/// the private-repo registry row cannot reach (there, the root gate fires first for +/// every non-owner), so reverting the visibility_check Deny-return turns this RED. +#[sqlx::test] +async fn get_pr_diff_withheld_path_is_denied(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + // #195 (F1): a bounded timeout so a wedged git subprocess push fails the suite + // rather than hanging it until CI kills the job. + let client = support::bounded_client(); + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + + let repo_id = node.seed_repo(&owner_did, "prdiff-repo", true).await; + let oids = node.seed_bare_repo( + &owner_did, + "prdiff-repo", + &[("public/a.txt", "public seed")], + "sha1", + ); + node.withhold_path(&repo_id, "/secret/**", &[], &owner_did) + .await; + + // Push a `feature` branch whose diff vs main touches the withheld secret path. + let (body, pushed_oids) = build_branch_push_body( + &oids["HEAD"], + "feature", + &[("secret/x.txt", "TOPSECRET-PRDIFF-PATH")], + ); + let push = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &format!("/{owner_did}/prdiff-repo/git-receive-pack"), + body, + &owner, + ) + .header("content-type", "application/x-git-receive-pack-request") + .send() + .await + .expect("push sends"); + assert_eq!(push.status().as_u16(), 200, "owner push must land"); + + // Open a PR main <- feature (as owner, a reader of the public repo). + let create = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &format!("/api/v1/repos/{owner_did}/prdiff-repo/pulls"), + br#"{"title":"prdiff","source_branch":"feature","target_branch":"main"}"#.to_vec(), + &owner, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create PR sends"); + assert_eq!(create.status().as_u16(), 201, "PR create must return 201"); + let pr: serde_json::Value = create.json().await.unwrap(); + let number = pr["number"].as_i64().expect("PR number"); + + // Anon PASSES the public repo-root gate, then the per-path gate DENIES the diff + // (it touches the withheld secret path): 404, leaking no content or OID. + // #195 (F2): withhold the pushed secret blob's OID in both forms (the full + // sha1 and `oid[..12]`), matching the private_blob_oid_short convention + // (probe.rs:200), NOT `git rev-parse --short` (variable-length). + let secret_oid_full = pushed_oids["secret/x.txt"].clone(); + let secret_oid_short = secret_oid_full[..12].to_string(); + let resp = client + .get(format!( + "{}/api/v1/repos/{owner_did}/prdiff-repo/pulls/{number}/diff", + node.base_url + )) + .send() + .await + .expect("anon diff read sends"); + let withheld = vec![ + "TOPSECRET-PRDIFF-PATH", + secret_oid_full.as_str(), + secret_oid_short.as_str(), + ]; + assert_denied(resp, 404, &withheld).await; + + // Owner sees the full diff (the per-path gate admits the reader). + let resp = signed_request( + &client, + reqwest::Method::GET, + &node.base_url, + &format!("/api/v1/repos/{owner_did}/prdiff-repo/pulls/{number}/diff"), + Vec::new(), + &owner, + ) + .send() + .await + .expect("owner diff read sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "owner must see the full PR diff (per-path gate admits the reader)" + ); + assert!( + resp.text().await.unwrap().contains("TOPSECRET-PRDIFF-PATH"), + "the owner's diff returns the touched file content" + ); + + node.shutdown().await; +} + +// ── U8: INV-2 — an anonymous clone/fetch excludes withheld subtree blobs ────── + +/// The replication/clone surface, the one `oneshot` cannot serve: a public repo +/// with a `/secret/**` withhold rule. A real anonymous `git clone` over +/// git-upload-pack must yield a packfile that omits the withheld blob's object +/// while keeping the sibling public blob. The assertion is packfile-aware: git +/// parsed the served pack, and `cat-file -e` checks object presence in it. +#[sqlx::test] +async fn anon_clone_excludes_withheld_subtree_blobs(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + + let repo_id = node.seed_repo(&owner_did, "u8-repo", true).await; + let oids = node.seed_bare_repo( + &owner_did, + "u8-repo", + &[ + ("public/a.txt", "public bytes U8"), + ("secret/b.txt", "TOPSECRET-U8"), + ], + "sha1", + ); + node.withhold_path(&repo_id, "/secret/**", &[], &owner_did) + .await; + + let secret_oid = oids["secret/b.txt"].clone(); + let public_oid = oids["public/a.txt"].clone(); + let head = oids["HEAD"].clone(); + + // Drive git-upload-pack directly (v0 stateless-RPC): a want for HEAD, a flush, + // then done. No side-band capability, so the response is a raw packfile after + // the NAK pkt-line. Vanilla `git clone` negotiates protocol v2 and deadlocks + // against the node's v0 server; the POST is the real replication surface and, + // via a bounded reqwest timeout, cannot wedge the suite. + let pkt = |s: &str| format!("{:04x}{}", s.len() + 4, s).into_bytes(); + let mut req_body: Vec = Vec::new(); + req_body.extend(pkt(&format!("want {head}\n"))); + req_body.extend_from_slice(b"0000"); // flush after wants + req_body.extend(pkt("done\n")); + + let client = bounded_client(); + let resp = client + .post(format!( + "{}/{owner_did}/u8-repo/git-upload-pack", + node.base_url + )) + .header("content-type", "application/x-git-upload-pack-request") + .body(req_body) + .send() + .await + .expect("upload-pack POST sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "anon upload-pack POST must serve a pack" + ); + let bytes = resp.bytes().await.expect("read pack response"); + + // The packfile starts at the "PACK" magic (after the NAK pkt-line). + let pack_start = bytes + .windows(4) + .position(|w| w == b"PACK") + .expect("response must contain a packfile"); + let pack = &bytes[pack_start..]; + + // Index the served pack and list its objects (packfile-aware: a raw byte scan + // could not see an OID inside the zlib-compressed stream). + let dir = std::env::temp_dir().join(format!("gl-u8-pack-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let pack_path = dir.join("deny.pack"); + std::fs::write(&pack_path, pack).unwrap(); + let idx = Command::new("git") + .args(["index-pack", pack_path.to_str().unwrap()]) + .output() + .expect("git index-pack runs"); + assert!( + idx.status.success(), + "the served pack must index cleanly: {}", + String::from_utf8_lossy(&idx.stderr) + ); + let verify = Command::new("git") + .args([ + "verify-pack", + "-v", + pack_path.with_extension("idx").to_str().unwrap(), + ]) + .output() + .expect("git verify-pack runs"); + let listing = String::from_utf8_lossy(&verify.stdout).to_string(); + let _ = std::fs::remove_dir_all(&dir); + + assert!( + !listing.contains(&secret_oid), + "withheld blob must be absent from the served pack; listing:\n{listing}" + ); + assert!( + listing.contains(&public_oid), + "public blob {public_oid} must be present (withhold is subtree-scoped); listing:\n{listing}" + ); + + node.shutdown().await; +} + +// ── U3: drive the whole deny-bearing route registry over the real stack ─────── + +/// Send one [`Probe`] against the running node, signing as the probe's identity. +/// Anon requests are unsigned; Owner/Stranger requests carry a real RFC-9421 +/// signature for the fixture's owner / stranger keypair. +async fn send_probe( + client: &reqwest::Client, + base_url: &str, + fixture: &Fixture, + probe: &Probe, +) -> reqwest::Response { + let rb = match probe.signer { + Signer::Anon => client + .request(probe.method.clone(), format!("{base_url}{}", probe.path)) + .body(probe.body.clone()), + Signer::Owner => signed_request( + client, + probe.method.clone(), + base_url, + &probe.path, + probe.body.clone(), + &fixture.owner, + ), + Signer::Stranger => signed_request( + client, + probe.method.clone(), + base_url, + &probe.path, + probe.body.clone(), + &fixture.stranger, + ), + // #195 (F1): each multi-principal arm signs with its own distinct fixture + // identity, so reverting one arm in the handler turns only that arm's twin + // RED. + Signer::Author => signed_request( + client, + probe.method.clone(), + base_url, + &probe.path, + probe.body.clone(), + &fixture.author, + ), + Signer::Creator => signed_request( + client, + probe.method.clone(), + base_url, + &probe.path, + probe.body.clone(), + &fixture.creator, + ), + Signer::Claimant => signed_request( + client, + probe.method.clone(), + base_url, + &probe.path, + probe.body.clone(), + &fixture.claimant, + ), + // #195 (F3): the assignee actor arm of complete_task / fail_task signs with + // the fixture's assignee identity (the one that claimed the seeded task), so + // reverting the assignee gate turns that arm's Not403 twin RED. + Signer::Assignee => signed_request( + client, + probe.method.clone(), + base_url, + &probe.path, + probe.body.clone(), + &fixture.assignee, + ), + }; + let rb = if probe.json { + rb.header("content-type", "application/json") + } else { + rb + }; + rb.send().await.unwrap_or_else(|e| { + panic!( + "probe failed to send: {} [{} {}]: {e}", + probe.label, probe.method, probe.path + ) + }) +} + +/// Walk every deny-bearing route (U1 registry), expand each into its hostile +/// probe plus positive twin (U2), and drive them against a real node: the +/// hostile request must return the exact deny status and leak nothing, and the +/// twin must reach the handler (owner-gate: not 403; read-gate: 2xx). This is +/// the runtime discharge of INV-1/INV-2/INV-8 across the whole registry, not one +/// hand-written case at a time. +/// +/// Terminal anti-vacuous-green invariant: exactly one hostile probe per row is +/// driven and the count equals the registry size, so a row that silently +/// produced no probe fails here instead of the sweep passing by testing nothing. +#[sqlx::test] +async fn deny_bearing_registry_denies_hostile_and_admits_authorized(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + // A bounded timeout so a wedged route fails the suite rather than hanging it. + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .unwrap(); + let fixture = Fixture::seed(&node).await; + + let rows = deny_bearing_routes(); + assert!( + !rows.is_empty(), + "deny-bearing route registry is empty — nothing to sweep" + ); + + // #195 (F1): every read-gate row is now probed by a signed non-reader too, so a + // dropped stranger probe must fail loudly rather than being absorbed into the + // total. Count the read-gate rows and the stranger hostiles separately. + let readgate_rows = rows + .iter() + .filter(|r| matches!(r.gate, GateClass::ReadGate)) + .count(); + + let mut hostile_driven = 0usize; + let mut readgate_stranger_driven = 0usize; + let mut twins_driven = 0usize; + + for row in rows { + let probes = probes_for(row, &fixture); + assert!( + !probes.is_empty(), + "row {} {} produced no probes", + row.method, + row.path + ); + for probe in &probes { + let ctx = format!("{} [{} {}]", probe.label, probe.method, probe.path); + let resp = send_probe(&client, &node.base_url, &fixture, probe).await; + let status = resp.status(); + match &probe.expect { + Expect::Deny(code) => { + hostile_driven += 1; + // A signed-stranger 404 is the read-gate signed-non-reader probe + // (owner-gate strangers are 403, signature hostiles are anon 401). + if probe.signer == Signer::Stranger && *code == 404 { + readgate_stranger_driven += 1; + } + assert_eq!( + status.as_u16(), + *code, + "hostile probe must deny with {code}, got {status}: {ctx}" + ); + // INV-8 shape guard: rechecks the status and that the body is + // neither an empty-200-as-success nor carrying withheld data. The + // probe's `withheld` tokens (the private repo's secret bytes and + // blob OID on read-gate probes) are passed through, so a 404 that + // spilled the private content fails here instead of being counted + // as a clean denial. + let tokens: Vec<&str> = probe.withheld.iter().map(String::as_str).collect(); + assert_denied(resp, *code, &tokens).await; + } + Expect::Not403 => { + twins_driven += 1; + assert_ne!( + status.as_u16(), + 403, + "owner-reachability twin must reach the handler (not 403): {ctx}" + ); + } + Expect::Ok2xx => { + twins_driven += 1; + assert!( + status.is_success(), + "read-reachability twin must be 2xx, got {status}: {ctx}" + ); + // A 2xx with an empty body is a denial rendered as success: + // the authorized read must actually return the resource. + let body = resp.bytes().await.unwrap_or_default(); + assert!( + !body.is_empty(), + "read-reachability twin returned an empty 2xx body (denial-as-success?): {ctx}" + ); + } + } + } + } + + // One hostile per row (a signed stranger for the 403 gates, including the + // #195 N2 status-gated bounty rows; anon for the signature and read-gate + // rows), plus the EXTRA signed-stranger 404 each read-gate row drives. + assert_eq!( + hostile_driven, + rows.len() + readgate_rows, + "expected one hostile per row ({} rows) plus one signed-stranger hostile per read-gate row ({readgate_rows}), drove {hostile_driven}", + rows.len() + ); + // #195 (F1): every read-gate row must be probed by a signed non-reader — a + // dropped stranger probe fails HERE rather than silently reducing coverage. + assert_eq!( + readgate_stranger_driven, readgate_rows, + "every read-gate row must drive a signed-non-reader 404 probe ({readgate_rows} rows), drove {readgate_stranger_driven}" + ); + assert!( + twins_driven >= 1, + "no positive twins were driven — the reachability proof is missing" + ); + + node.shutdown().await; +} + +// ── Additional INV-1 owner-gates over the real stack (fan-out of U6) ────────── + +/// The remaining high-blast-radius owner-gated mutations that previously had +/// only the source-level authz-table guard and no runtime deny test: branch +/// protection, webhook create/delete, and visibility removal. Each rejects a +/// validly-signed non-owner with 403, and the owner reaches the handler (not +/// 403), proving the 403 is the owner gate. All share `did_matches` as the root +/// gate (see the mutation check in the harness commit). +#[sqlx::test] +async fn additional_owner_gates_reject_non_owner(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = bounded_client(); + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let stranger = Keypair::generate(); + + node.seed_repo(&owner_did, "gated-repo", true).await; + let base = format!("/api/v1/repos/{owner_did}/gated-repo"); + + // (method, path, body, needs-json-content-type) + let cases: Vec<(reqwest::Method, String, Vec, bool)> = vec![ + ( + reqwest::Method::POST, + format!("{base}/branches/main/protect"), + Vec::new(), + false, + ), + ( + reqwest::Method::DELETE, + format!("{base}/branches/main/protect"), + Vec::new(), + false, + ), + ( + reqwest::Method::POST, + format!("{base}/hooks"), + br#"{"url":"https://example.com/hook","events":["*"]}"#.to_vec(), + true, + ), + ( + reqwest::Method::DELETE, + format!("{base}/hooks/deadbeef"), + Vec::new(), + false, + ), + ( + reqwest::Method::DELETE, + format!("{base}/visibility"), + br#"{"path_glob":"/"}"#.to_vec(), + true, + ), + ]; + + let send = |m: reqwest::Method, path: String, body: Vec, json: bool, kp: &Keypair| { + let mut rb = signed_request(&client, m, &node.base_url, &path, body, kp); + if json { + rb = rb.header("content-type", "application/json"); + } + rb.send() + }; + + for (method, path, body, json) in cases { + // Non-owner: valid signature, wrong identity -> 403 owner gate, no leak. + let resp = send(method.clone(), path.clone(), body.clone(), json, &stranger) + .await + .unwrap_or_else(|e| panic!("{method} {path} sends: {e}")); + assert_denied(resp, 403, &[]).await; + + // Owner reaches the handler: NOT 403 (proves the 403 above was the gate, + // not an earlier layer or a 415/404 masquerade). + let resp = send(method.clone(), path.clone(), body, json, &owner) + .await + .unwrap_or_else(|e| panic!("{method} {path} owner sends: {e}")); + assert_ne!( + resp.status().as_u16(), + 403, + "owner must reach {method} {path} (got 403)" + ); + } + + node.shutdown().await; +} + +// ── Teardown: shutdown() joins the serve task and closes the pool ───────────── + +/// `shutdown()` must join the detached serve task and close the pool before +/// returning. RED condition: with drop-only teardown the pool clones held by +/// the still-running server are open when the test future returns, so +/// `#[sqlx::test]`'s `DROP DATABASE` cleanup fails (sqlx prints and ignores +/// it), leaking per-test databases and flaking under parallel execution. The +/// `observer` clone shares the pool's inner, so `is_closed()` here is exactly +/// the state sqlx cleanup will see. +#[sqlx::test] +async fn shutdown_joins_server_and_closes_pool(pool: sqlx::PgPool) { + let observer = pool.clone(); + + let node = spawn_node(pool).await; + let base_url = node.base_url.clone(); + let client = bounded_client(); + + // One real request so at least one server connection (and its keep-alive) + // has existed; graceful shutdown must close it, not wait on it. + client + .get(format!("{base_url}/health")) + .send() + .await + .expect("probe request sends"); + + node.shutdown().await; + + assert!( + observer.is_closed(), + "shutdown() must close the pool before the test returns, or sqlx's \ + DROP DATABASE cleanup races the server's open connections" + ); + // The serve task was joined, not merely signalled: a fresh request to the + // old address must fail to connect. + let after = client.get(format!("{base_url}/health")).send().await; + assert!( + after.is_err(), + "the server must be gone after shutdown(); got {:?} instead", + after.map(|r| r.status()) + ); +} + +// ── Teardown: Drop without shutdown() must still release the database ───────── + +/// A test that returns WITHOUT calling `shutdown()` (early return, forgotten +/// teardown) must not leave the detached serve task holding pool clones open +/// against sqlx's cleanup: `#[sqlx::test]` issues a plain `DROP DATABASE` (no +/// FORCE) the moment the test future returns, a still-connected session makes +/// the drop fail, and sqlx only prints the error, silently leaking the +/// per-test database. `TestNode`'s `Drop` tears the server down two redundant +/// ways: the graceful watch signal, and an abort of the remaining serve +/// handle (which drops the task's future, and the pool clones inside its +/// router/state, on a subsequent scheduler tick without depending on the +/// graceful chain of watcher/connection tasks completing). This test fails if +/// BOTH are broken (verified by neutering them in turn: with signal and abort +/// both removed the probe sees the leaked sessions for the full bound; either +/// mechanism alone drains them by the first probe), so it fences the Drop +/// teardown contract as a whole rather than either mechanism individually. +/// +/// Two guards against a vacuous pass: the node pool is built with ambient +/// reaping disabled (`idle_timeout`/`max_lifetime` `None`; sqlx's default test +/// pool options reap idle connections after ~1s, which would release the +/// database by itself and mask a broken teardown), and the probe runs over a +/// standalone connection rather than a clone of the node pool, so observing +/// does not itself keep the pool alive. +#[sqlx::test] +async fn drop_without_shutdown_unblocks_database( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, +) { + use sqlx::{ConnectOptions as _, Connection as _}; + + let pool = pool_opts + .idle_timeout(None) + .max_lifetime(None) + .connect_with(connect_opts.clone()) + .await + .expect("node pool connects"); + + // Inner scope so the node (and the client's keep-alive connection) drop + // exactly as they would when a test body returns, with no shutdown() call. + { + let node = spawn_node(pool).await; + let client = bounded_client(); + let resp = client + .get(format!("{}/health", node.base_url)) + .send() + .await + .expect("probe request sends"); + assert!( + resp.status().is_success(), + "the server must be live before the drop" + ); + } + + // Standalone observer: count OTHER sessions connected to the per-test + // database. Zero means the only session left is the probe itself, i.e. the + // database is droppable the instant the probe disconnects, which is + // exactly the state sqlx's cleanup needs. Bounded poll: the abort takes + // effect on a scheduler tick, not synchronously in Drop. + let mut probe = connect_opts + .connect() + .await + .expect("standalone probe connects"); + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + loop { + let others: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_stat_activity \ + WHERE datname = current_database() AND pid <> pg_backend_pid()", + ) + .fetch_one(&mut probe) + .await + .expect("session count query runs"); + if others == 0 { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "dropped-without-shutdown node still holds {others} session(s) \ + against the per-test database after 5s; sqlx's DROP DATABASE \ + cleanup would fail and leak the database" + ); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + probe.close().await.expect("probe connection closes"); +} + +/// Isolates the `handle.abort()` half of `TestNode`'s `Drop` teardown. The +/// existing `drop_without_shutdown_unblocks_database` fences both redundant +/// mechanisms together and stays GREEN if only the abort is removed (the +/// graceful watch signal alone drains the serve task). This test suppresses the +/// graceful signal (`force_abort_only_teardown`) so ONLY the abort runs, making +/// the abort line load-bearing: with the abort removed from `Drop` neither +/// mechanism runs, the serve task keeps its pool clones open, and the probe sees +/// the leaked sessions for the full bound (RED); with the abort present the +/// aborted task's future is dropped on a scheduler tick and the sessions drain +/// (GREEN). Same vacuous-pass guards as the sibling test: node pool built with +/// reaping disabled, and a standalone observer connection. +#[sqlx::test] +async fn drop_with_broken_graceful_chain_still_unblocks_via_abort( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, +) { + use sqlx::{ConnectOptions as _, Connection as _}; + + let pool = pool_opts + .idle_timeout(None) + .max_lifetime(None) + .connect_with(connect_opts.clone()) + .await + .expect("node pool connects"); + + // Inner scope so the node drops as a test body's would, with no shutdown(). + { + let mut node = spawn_node(pool).await; + // Break the graceful chain: Drop will not send the watch signal, so + // only handle.abort() can tear the serve task down. + node.force_abort_only_teardown(); + let client = bounded_client(); + let resp = client + .get(format!("{}/health", node.base_url)) + .send() + .await + .expect("probe request sends"); + assert!( + resp.status().is_success(), + "the server must be live before the drop" + ); + } + + // Standalone observer: with only the abort running, the serve task's future + // (and its pool clones) must still be dropped, leaving zero other sessions. + let mut probe = connect_opts + .connect() + .await + .expect("standalone probe connects"); + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + loop { + let others: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_stat_activity \ + WHERE datname = current_database() AND pid <> pg_backend_pid()", + ) + .fetch_one(&mut probe) + .await + .expect("session count query runs"); + if others == 0 { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "abort-only teardown still holds {others} session(s) against the \ + per-test database after 5s; handle.abort() is not releasing the \ + serve task's pool clones and sqlx's DROP DATABASE cleanup would fail" + ); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + probe.close().await.expect("probe connection closes"); +} + +// ── U4: completeness cross-check (no DB) — the registry cannot silently drift ── +// +// The runtime sweep (U3) only proves the routes it is HANDED. This guard, a pure +// source scrape, keeps that hand-off honest against `server.rs`: no registry row +// points at a route that no longer mounts (stale row), and no owner-gated mount +// escapes the registry (orphan). It complements — does not duplicate — the +// in-crate `authz_guard` egress guard (`src/api/mod.rs`), which proves every +// repo-scoped API handler is *gated*; this proves the deny-bearing ones are +// *driven*, and it reaches the non-API git mounts `authz_guard` never sees. +mod completeness { + use std::collections::HashSet; + + use super::deny_bearing_routes; + use crate::support::routes::{GateClass, Principal}; + + /// The CLOSED set of structural reasons a read-gate handler may sit out of the + /// runtime 404 sweep (#195, U4/R4). This replaces the free-text `reason` + /// strings the old allowlist carried: a "deferred / needs seeding" prose excuse + /// no longer type-checks. Every `READ_GATE_NOT_DRIVEN` entry must name one of + /// these variants, so a reviewer sees a STRUCTURAL class (why the read is not a + /// drivable 404-deny GET), never a soft deferral that hides an undriven private + /// read. A drivable GET (the nine U3 reads) cannot be parked here because none + /// of these classes fits it — and the driven-not-relabeled assertion below + /// makes that a hard failure rather than a judgment call. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum NotDrivenReason { + /// A create/write path that read-gates the caller before mutating, not a + /// 404-deny GET. Its owner/author behaviour is covered by the mutation + /// authz guard in `src/api/mod.rs`, not this GET-read sweep. + ReadGatingMutation, + /// A git smart-HTTP read (`info/refs`, `git-upload-pack`): its + /// withheld-subtree 404/exclusion is driven by the U7/U8 real-clone cases, + /// not the API GET sweep. + GitSmartHttpRead, + /// A content-addressed read (`/ipfs/{cid}`): its 404 deny + no-leak is + /// driven by the U5b/U8 anon_ipfs/clone cases in this file. + ContentAddressedRead, + /// A global list-FILTER: returns 200 with unreadable rows removed, never a + /// 404, so it is not a deny-bearing read at all. + GlobalListFilter, + /// A repo-scoped read whose deny is already driven as a NON-ReadGate + /// registry row — the handler read-gates for reach, but the deny the + /// sweep must prove is the other gate's status, not a 404. + DrivenByOtherGate, + /// A repo-scoped read whose repo selector arrives as a QUERY parameter + /// (`?repo=`), not a mounted path segment — a `deny_bearing_routes()` row + /// can only name a mounted path, so the sweep cannot express its 404. + QueryScopedRead, + /// The owner-2xx twin needs a live external dependency the harness lacks: + /// `get_encrypted_blob` reads through `ipfs_pin::cat` with no harness stub, + /// so it cannot be driven as a 2xx twin here. An honest exclusion, not a + /// deferral (#195, KTD-5). + ExternalDependencyUnavailable, + } + + /// Read-gate handlers NOT driven as a `ReadGate` row, each carrying a CLOSED + /// structural reason ([`NotDrivenReason`]) — never free-text. This is the + /// enforced form of the prose that used to live in routes.rs; a handler leaving + /// the sweep must be moved here under a structural class, never with a soft + /// "deferred / needs seeding" excuse (which no longer type-checks). + /// + /// Single source of truth: both the marker-scan guard + /// (`every_read_gate_handler_is_driven_or_explicitly_allowlisted`) and the + /// route-table cross-check (`mounted_repo_gets_are_driven_excused_or_declared_public`) + /// read this, so the two cannot drift. + /// + /// get_issue / get_pr / get_pr_diff / list_issue_comments / list_reviews / + /// list_comments / get_cert / get_bounty / get_tree are DRIVEN as real ReadGate + /// rows in deny_bearing_routes() (#195, U3); the driven-not-relabeled assertion + /// bars any of them from being quietly re-parked here. + const READ_GATE_NOT_DRIVEN: &[(&str, NotDrivenReason)] = &[ + // Owner-2xx twin needs a live IPFS backend (`ipfs_pin::cat`) the harness has + // no stub for — an honest external-dependency exclusion (#195, KTD-5). + ( + "get_encrypted_blob", + NotDrivenReason::ExternalDependencyUnavailable, + ), + // Read-gates but is a mutation, not a 404-deny GET. + ("create_review", NotDrivenReason::ReadGatingMutation), + ("create_comment", NotDrivenReason::ReadGatingMutation), + ("create_pr", NotDrivenReason::ReadGatingMutation), + ("create_issue", NotDrivenReason::ReadGatingMutation), + ("create_issue_comment", NotDrivenReason::ReadGatingMutation), + ("create_bounty", NotDrivenReason::ReadGatingMutation), + ("claim_bounty", NotDrivenReason::ReadGatingMutation), + ("fork_repo", NotDrivenReason::ReadGatingMutation), + ("star_repo", NotDrivenReason::ReadGatingMutation), + ("unstar_repo", NotDrivenReason::ReadGatingMutation), + // merge_pr authorize_repo_reads before the owner check, but it is a POST + // merge mutation, not a 404-deny GET — its deny is already driven as the + // OwnerGate row in deny_bearing_routes(). + ("merge_pr", NotDrivenReason::ReadGatingMutation), + // Mutations that gained the authorize_repo_read quarantine pre-check on + // main after the table froze: each read-gates for reach, then denies + // under the gate its registry row already drives (Owner / MultiPrincipal + // / SignerSelf). None is a 404-deny GET. + ("add_label", NotDrivenReason::ReadGatingMutation), + ("remove_label", NotDrivenReason::ReadGatingMutation), + ("approve_bounty", NotDrivenReason::ReadGatingMutation), + ("cancel_bounty", NotDrivenReason::ReadGatingMutation), + ("dispute_bounty", NotDrivenReason::ReadGatingMutation), + ("submit_bounty", NotDrivenReason::ReadGatingMutation), + ("close_issue", NotDrivenReason::ReadGatingMutation), + ("close_pr", NotDrivenReason::ReadGatingMutation), + ("create_webhook", NotDrivenReason::ReadGatingMutation), + ("delete_webhook", NotDrivenReason::ReadGatingMutation), + ("set_visibility", NotDrivenReason::ReadGatingMutation), + ("remove_visibility", NotDrivenReason::ReadGatingMutation), + ("protect_branch", NotDrivenReason::ReadGatingMutation), + ("unprotect_branch", NotDrivenReason::ReadGatingMutation), + ("register_replica", NotDrivenReason::ReadGatingMutation), + ("unregister_replica", NotDrivenReason::ReadGatingMutation), + // Smart-HTTP push: read-gates the repo before accepting pack objects; its + // denies are driven by the dedicated signed_* tests. + ("git_receive_pack", NotDrivenReason::ReadGatingMutation), + // list_visibility is a GET, but its deny is the owner gate's 403, already + // driven by the OwnerGate row — not a 404-deny read. + ("list_visibility", NotDrivenReason::DrivenByOtherGate), + // list_anchors scopes its repo via ?repo= (a query param), which a + // mounted-path row cannot express — a suite coverage gap to drive + // separately, not a 404-deny GET on the path table. + ("list_anchors", NotDrivenReason::QueryScopedRead), + // Content-addressed read: driven by the U5b/U8 anon_ipfs/clone cases via + // the internal `gate_and_serve` helper (marker moved off `get_by_cid`). + ("gate_and_serve", NotDrivenReason::ContentAddressedRead), + // Git smart-HTTP reads: driven by the U7/U8 real-clone cases. + ("git_info_refs", NotDrivenReason::GitSmartHttpRead), + ("git_upload_pack", NotDrivenReason::GitSmartHttpRead), + // Global list-FILTER: 200 with unreadable rows removed, never a 404. + ("list_all_bounties", NotDrivenReason::GlobalListFilter), + ]; + + /// Every `.route("", ()…)` mount in `src`, as + /// (METHOD, path). Multiline-aware: most mounts put the path on the line after + /// `.route(`, so a per-line scan false-greens. Walks balanced parens from each + /// `.route(` so chained and multi-method (`put().delete().get()`) mounts are + /// all captured. + fn scrape_mounts(src: &str) -> Vec<(String, String)> { + let bytes = src.as_bytes(); + let mut out = Vec::new(); + let mut i = 0; + while let Some(rel) = src[i..].find(".route(") { + let open = i + rel + ".route(".len(); + let mut depth = 1i32; + let mut j = open; + while j < src.len() && depth > 0 { + match bytes[j] { + b'(' => depth += 1, + b')' => depth -= 1, + _ => {} + } + j += 1; + } + let call = &src[open..j.saturating_sub(1)]; + i = j; + // The path is the first string literal in the call. + let Some(qs) = call.find('"') else { continue }; + let Some(qe) = call[qs + 1..].find('"') else { + continue; + }; + let path = &call[qs + 1..qs + 1 + qe]; + for m in ["get", "post", "put", "delete", "patch"] { + let needle = format!("{m}("); + let mut k = 0; + while let Some(mrel) = call[k..].find(&needle) { + let at = k + mrel; + // Reject a match that is the tail of a longer ident (e.g. the + // `get(` inside `budget(`): the char before must be a boundary. + let boundary = call[..at] + .chars() + .last() + .map(|c| !(c.is_alphanumeric() || c == '_')) + .unwrap_or(true); + if boundary { + out.push((m.to_uppercase(), path.to_string())); + } + k = at + needle.len(); + } + } + } + out + } + + /// Names of `fn`s in `src` whose body contains any `marker`. The body is the + /// slice from a fn's `(` to the next top-level fn declaration — the same + /// boundary set `authz_guard::fn_body` uses — so a marker can't leak across + /// into the next handler. + fn handlers_with_marker(src: &str, markers: &[&str]) -> HashSet { + let decls = [ + "\npub async fn ", + "\npub(crate) async fn ", + "\nasync fn ", + "\npub(crate) fn ", + "\npub fn ", + "\nfn ", + ]; + // Ordered start offsets of every fn declaration. + let mut starts: Vec = Vec::new(); + for d in decls { + let mut k = 0; + while let Some(r) = src[k..].find(d) { + starts.push(k + r + 1); // +1: skip the leading '\n' + k = k + r + d.len(); + } + } + starts.sort_unstable(); + let mut out = HashSet::new(); + for (idx, &s) in starts.iter().enumerate() { + let end = starts.get(idx + 1).copied().unwrap_or(src.len()); + let seg = &src[s..end]; + // The name is the ident after this decl's `fn ` (handles `pub(crate)`, + // whose own `(` would otherwise be mistaken for the arg list). + let Some(fnpos) = seg.find("fn ") else { + continue; + }; + let name: String = seg[fnpos + 3..] + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect(); + if name.is_empty() { + continue; + } + if markers.iter().any(|m| seg.contains(m)) { + out.insert(name); + } + } + out + } + + /// Replaces the CONTENTS of comments and string/char literals with spaces, + /// preserving byte length and delimiters. A `fn`-shaped or marker-shaped token + /// inside a literal must not read as a real decl boundary or gate marker: + /// api/mod.rs's `resolver_body` fixture (`fn helper() { … did_matches … }` in a + /// test string) claimed a 20KB segment and swept the marker tables that + /// followed it, false-orphaning as `helper`. Interior newlines are masked too, + /// so `"\nfn "` inside a string cannot fake a declaration start. This is also + /// what makes [`strip_test_modules`]' brace matching sound: a `}` inside a + /// `///` doc comment (authz_guard has one) otherwise closes the mod early and + /// leaves the tail — and its fixture `fn`s — in the scan. + fn mask_noncode(src: &str) -> String { + let b = src.as_bytes(); + let n = b.len(); + let mut out = b.to_vec(); + let mut i = 0; + while i < n { + if b[i] == b'/' && i + 1 < n && b[i + 1] == b'/' { + let start = i; + while i < n && b[i] != b'\n' { + i += 1; + } + out[start..i].fill(b' '); + continue; + } + if b[i] == b'/' && i + 1 < n && b[i + 1] == b'*' { + let start = i; + i += 2; + let mut depth = 1i32; + while i + 1 < n && depth > 0 { + if b[i] == b'/' && b[i + 1] == b'*' { + depth += 1; + i += 2; + } else if b[i] == b'*' && b[i + 1] == b'/' { + depth -= 1; + i += 2; + } else { + i += 1; + } + } + out[start..i.min(n)].fill(b' '); + continue; + } + // Raw string: `r"`, `r#"`, `r##"…` or the `br…` byte forms. + let raw = if b[i] == b'r' { + Some(i + 1) + } else if b[i] == b'b' && i + 1 < n && b[i + 1] == b'r' { + Some(i + 2) + } else { + None + }; + if let Some(mut p) = raw { + let mut hashes = 0usize; + while p < n && b[p] == b'#' { + hashes += 1; + p += 1; + } + if p < n && b[p] == b'"' { + let inner = p + 1; + let mut j = inner; + while j < n { + if b[j] == b'"' { + let mut k = 0; + while k < hashes && j + 1 + k < n && b[j + 1 + k] == b'#' { + k += 1; + } + if k == hashes { + break; + } + } + j += 1; + } + out[inner..j.min(n)].fill(b' '); + i = (j + 1 + hashes).min(n); + continue; + } + } + // Cooked string or byte string: `"` or `b"`. + let cooked = if b[i] == b'"' { + Some(i + 1) + } else if b[i] == b'b' && i + 1 < n && b[i + 1] == b'"' { + Some(i + 2) + } else { + None + }; + if let Some(inner) = cooked { + let mut j = inner; + while j < n && b[j] != b'"' { + if b[j] == b'\\' { + j += 1; + } + j += 1; + } + out[inner..j.min(n)].fill(b' '); + i = (j + 1).min(n); + continue; + } + // Char literal `'x'` / `'\e'` (a bare `'a` is a lifetime — skip it). + if b[i] == b'\'' { + if i + 3 < n && b[i + 1] == b'\\' && b[i + 3] == b'\'' { + out[i + 1] = b' '; + out[i + 2] = b' '; + i += 4; + continue; + } + if i + 2 < n && b[i + 2] == b'\'' { + out[i + 1] = b' '; + i += 3; + continue; + } + } + i += 1; + } + String::from_utf8(out).expect("masking writes only ASCII spaces") + } + + /// The source a handler scan should see: string/comment contents masked, then + /// `#[cfg(test)] mod … { … }` blocks removed. + fn scan_src(src: &str) -> String { + strip_test_modules(&mask_noncode(src)) + } + + /// Removes `#[cfg(test)] mod { … }` blocks (attribute + `mod` header + + /// the whole braced body) before the handler scans run. Unit-test modules are + /// never mounted handlers. Run it on [`mask_noncode`] output: its brace + /// matching is only sound once literal and comment braces are gone. + fn strip_test_modules(src: &str) -> String { + let mut out = String::with_capacity(src.len()); + let mut rest = src; + // Line-anchored: a `#[cfg(test)]` inside a comment or string literal must + // not strip the next real `mod` and hide real handlers from the scan. + while let Some(attr) = rest.find("\n#[cfg(test)]").map(|p| p + 1) { + let after = &rest[attr + "#[cfg(test)]".len()..]; + // Only strip when the attribute applies directly to a `mod`; a + // `#[cfg(test)]` on a bare fn leaves real decls reachable. + let Some(mod_kw) = after.find("mod ") else { + break; + }; + if !after[..mod_kw].trim().is_empty() { + break; + } + out.push_str(&rest[..attr]); + let Some(brace) = after[mod_kw..].find('{') else { + rest = after; + break; + }; + let open = mod_kw + brace; + let bytes = after.as_bytes(); + let mut depth = 0i32; + let mut j = open; + while j < after.len() { + match bytes[j] { + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 { + break; + } + } + _ => {} + } + j += 1; + } + rest = if j < after.len() { &after[j + 1..] } else { "" }; + } + out.push_str(rest); + out + } + + /// PREDICATE rather than a substring — used for the inline owner-gate + /// did_matches idiom (F3), whose "first arg is the caller, second is + /// owner_did" shape a plain substring cannot express. Same fn-segmentation as + /// `handlers_with_marker` so a match cannot leak across into the next handler. + fn handlers_matching(src: &str, pred: fn(&str) -> bool) -> HashSet { + let decls = [ + "\npub async fn ", + "\npub(crate) async fn ", + "\nasync fn ", + "\npub(crate) fn ", + "\npub fn ", + "\nfn ", + ]; + let mut starts: Vec = Vec::new(); + for d in decls { + let mut k = 0; + while let Some(r) = src[k..].find(d) { + starts.push(k + r + 1); + k = k + r + d.len(); + } + } + starts.sort_unstable(); + let mut out = HashSet::new(); + for (idx, &s) in starts.iter().enumerate() { + let end = starts.get(idx + 1).copied().unwrap_or(src.len()); + let seg = &src[s..end]; + let Some(fnpos) = seg.find("fn ") else { + continue; + }; + let name: String = seg[fnpos + 3..] + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect(); + if name.is_empty() { + continue; + } + if pred(seg) { + out.insert(name); + } + } + out + } + + fn short_handler(h: &str) -> &str { + h.rsplit("::").next().unwrap_or(h) + } + + /// Like [`handlers_with_marker`] but ignores markers that appear only inside a + /// full-line `//` comment. The owner-gate markers never show up in prose, but + /// the read-gate markers (`authorize_repo_read(` / `visibility_check(`) do: + /// several docstrings and a test comment name them (repos.rs's + /// `owner_push_rejection` / `dedupe_canonical_repos` regions, the fork + /// docstring), and a raw `contains` would misattribute those to the nearest + /// preceding fn and force a phantom non-handler onto the read-gate allowlist. + /// Mirrors `authz_guard::fn_body`'s comment-stripping so the scan sees code, + /// not prose. + fn handlers_with_code_marker(src: &str, markers: &[&str]) -> HashSet { + let stripped: String = src + .lines() + .map(|l| { + if l.trim_start().starts_with("//") { + "" + } else { + l + } + }) + .collect::>() + .join("\n"); + handlers_with_marker(&stripped, markers) + } + + /// True when `body` contains an INLINE owner-gate `did_matches` call: one + /// whose FIRST arg is the caller (`caller` / `&caller` / `&auth.0`) AND whose + /// SECOND arg is `&record.owner_did`. That two-arg shape is the discriminator + /// (F3): `require_owner`/`require_repo_owner` are the named markers, but + /// protect_branch / unprotect_branch owner-gate with this raw idiom instead, so + /// a plain marker misses them and a future owner-only handler using the same + /// pattern could slip past the registry while U4 stayed green. + /// + /// It must NOT fire on the signer-self / author forms that share the helper: + /// register_replica's `did_matches(replica_did, &record.owner_did)` (first arg + /// is the replica, not the caller — signer-self by design), close_pr's + /// `did_matches(&auth.0, &pr.author_did)`, and the bounty/task self forms + /// (`&bounty.creator_did`, delegator/assignee dids). Whitespace and newlines + /// inside the arg list are tolerated so a reformatted call still matches. + fn has_owner_did_matches(body: &str) -> bool { + let bytes = body.as_bytes(); + let mut i = 0; + while let Some(rel) = body[i..].find("did_matches(") { + let open = i + rel + "did_matches(".len(); + // Walk to the matching close paren, splitting the top-level args on the + // first depth-0 comma (there are exactly two args). + let mut depth = 1i32; + let mut comma: Option = None; + let mut j = open; + while j < body.len() && depth > 0 { + match bytes[j] { + b'(' => depth += 1, + b')' => depth -= 1, + b',' if depth == 1 && comma.is_none() => comma = Some(j), + _ => {} + } + j += 1; + } + i = j; + let close = j.saturating_sub(1); + let Some(comma) = comma else { continue }; + let arg1 = body[open..comma].split_whitespace().collect::(); + let arg2 = body[comma + 1..close] + .split_whitespace() + .collect::(); + let caller_first = arg1 == "caller" || arg1 == "&caller" || arg1 == "&auth.0"; + let owner_second = arg2 == "&record.owner_did"; + if caller_first && owner_second { + return true; + } + } + false + } + + /// The CLOSED set of caller-self REST gates deliberately NOT driven as a + /// registry row. INTENTIONALLY EMPTY (#195, F3/R4): every caller-self + /// `did_matches(caller, X)` gate in `src/api` is driven by a real-node + /// hostile+control probe (a `SignerSelfGate` field-binding row or a + /// `MultiPrincipalGate` assignee-arm row), so no caller-self REST gate is + /// un-drivable. A new classifier-marked handler that is not a driven row must + /// fail `signer_self_gates_are_all_driven`; it is not parked here. + const SIGNER_SELF_NOT_DRIVEN: &[&str] = &[]; + + /// True when `body` contains a CALLER-SELF `did_matches` call: one whose FIRST + /// arg is the caller (`caller` / `&caller` / `&auth.0`) AND whose SECOND arg is + /// NOT `&record.owner_did`. Sibling to [`has_owner_did_matches`] (the owner form + /// is guarded there); this is the field-binding / actor idiom the task and + /// register handlers use: create_task (`&body.delegator_did`), claim_task + /// (`&body.assignee_did`), complete_task / fail_task (the stored `assignee_did`, + /// spelled across lines), register (`agent_did.as_str()`). The arg1 set matches + /// its owner sibling's (NOT just `&auth.0`), so a `let caller = &auth.0; + /// did_matches(caller, X)` gate cannot escape. The paren-walk is multi-line-robust + /// because complete_task / fail_task spell the call across lines. + fn has_signer_self_did_matches(body: &str) -> bool { + let bytes = body.as_bytes(); + let mut i = 0; + while let Some(rel) = body[i..].find("did_matches(") { + let open = i + rel + "did_matches(".len(); + let mut depth = 1i32; + let mut comma: Option = None; + let mut j = open; + while j < body.len() && depth > 0 { + match bytes[j] { + b'(' => depth += 1, + b')' => depth -= 1, + b',' if depth == 1 && comma.is_none() => comma = Some(j), + _ => {} + } + j += 1; + } + i = j; + let close = j.saturating_sub(1); + let Some(comma) = comma else { continue }; + let arg1 = body[open..comma].split_whitespace().collect::(); + let arg2 = body[comma + 1..close] + .split_whitespace() + .collect::(); + let caller_first = arg1 == "caller" || arg1 == "&caller" || arg1 == "&auth.0"; + let owner_second = arg2 == "&record.owner_did"; + if caller_first && !owner_second { + return true; + } + } + false + } + + /// True when `seg` carries the inline claimant gate idiom: a local `is_claimant` + /// boolean tied to `claimant_did` before a Forbidden return. submit_bounty uses + /// this shape instead of a bare `did_matches(&auth.0, claimant)` call. + fn has_claimant_inline_gate(seg: &str) -> bool { + seg.contains("is_claimant") + && seg.contains("claimant_did") + && (seg.contains("Forbidden") || seg.contains("!is_claimant")) + } + + #[test] + fn deny_registry_is_not_stale_and_owner_gates_are_all_driven() { + let server = include_str!("../src/server.rs"); + let mounts: HashSet<(String, String)> = scrape_mounts(server).into_iter().collect(); + + // Scraper-integrity floor: if the parser silently stopped finding mounts, + // the anti-stale check below would pass vacuously. The tree has ~95 mounts. + assert!( + mounts.len() >= 90, + "mount scrape found only {} routes — the parser likely broke (floor 90)", + mounts.len() + ); + + // ANTI-STALE: every deny-bearing row must still point at a live mount, so a + // renamed/removed route can't leave a row that the sweep drives against a + // 404 and calls a passing deny. + for row in deny_bearing_routes() { + assert!( + mounts.contains(&(row.method.to_string(), row.path.to_string())), + "stale registry row: {} {} is not mounted in server.rs (renamed or removed?)", + row.method, + row.path + ); + } + + // ORPHAN GUARD: every handler that owner-gates must be a registry owner-gate + // row, so a newly added owner-gated mutation cannot escape the runtime + // sweep. The api dir is read at test time (like authz_guard's completeness + // scan) so a brand-new module is covered too. Two owner-gate shapes are + // recognized: + // (1) the named markers `require_repo_owner(` / `require_owner(`, and + // (2) the INLINE `did_matches(caller, &record.owner_did)` idiom (F3) that + // protect_branch / unprotect_branch use instead of a named helper. + // Only the SECOND-arg-is-owner_did form of did_matches is treated as an + // owner gate; the signer-self / author forms (register_replica's + // did_matches(replica_did, …), close_pr's did_matches(&auth.0, &author_did)) + // are deliberately NOT matched, so they don't false-orphan. + // A handler satisfies the owner marker if it is a plain OwnerGate row OR a + // MultiPrincipalGate row that declares the Owner arm (#195, F1: close_pr / + // close_issue owner-gate via require_repo_owner but are owner-OR-author, so + // they register as MultiPrincipalGate — their owner arm is still driven). + let registry_owner_handlers: HashSet<&str> = deny_bearing_routes() + .iter() + .filter(|r| { + r.gate == GateClass::OwnerGate + || (r.gate == GateClass::MultiPrincipalGate + && r.principals.contains(&Principal::Owner)) + }) + .map(|r| short_handler(r.handler)) + .collect(); + + let api_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/src/api"); + let mut owner_marked: HashSet = HashSet::new(); + for entry in std::fs::read_dir(api_dir).expect("read api dir") { + let path = entry.expect("dir entry").path(); + if path.extension().is_some_and(|e| e == "rs") { + let src = scan_src(&std::fs::read_to_string(&path).expect("read api file")); + owner_marked.extend(handlers_with_marker( + &src, + &["require_repo_owner(", "require_owner("], + )); + // Fold in the inline did_matches-owner idiom (F3). + owner_marked.extend(handlers_matching(&src, has_owner_did_matches)); + } + } + // The gate helpers' own definitions carry the owner-gate idiom (the named + // marker string, and visibility::require_owner's body is itself a + // `did_matches(caller, &record.owner_did)` check); they are helpers, not + // mounted handlers, so drop them. + owner_marked.remove("require_owner"); + owner_marked.remove("require_repo_owner"); + // git_receive_pack also carries the inline did_matches-owner idiom, but for + // a SECONDARY gate: its branch-protection push check (repos.rs), not a + // whole-handler owner gate. Its 403 is DRIVEN by the dedicated + // `signed_stranger_protected_branch_push_is_forbidden` test (#195, F3) — a + // signed non-owner push to a protected branch, with an owner control — so + // inverting the gate goes RED there. It is therefore not an owner-gate + // orphan (the OwnerGate registry rows can't carry the protected-branch + // substrate); drop it so the owner-orphan assert does not misflag a route + // whose owner gate is already driven. + owner_marked.remove("git_receive_pack"); + // create_task (tasks.rs) carries a CONDITIONAL owner gate (#496): only when + // the caller-supplied repo_id names a hosted, non-quarantined repo. Its + // primary gate is the delegator signer-self binding, already driven as the + // SignerSelfGate row, and the repo-owner arm is driven by the dedicated + // `create_task_rejects_foreign_repo_id` handler test — same driven- + // elsewhere class as git_receive_pack, not an orphan. + owner_marked.remove("create_task"); + // Non-vacuous floor: if the marker scan silently found nothing (a parser + // regression), the orphan loop below would pass by checking zero handlers. + // The tree has 10 owner-marker handlers today; 6 trips only on a real break. + assert!( + owner_marked.len() >= 6, + "owner-gate marker scan found only {} handlers — the scan likely broke", + owner_marked.len() + ); + + for h in &owner_marked { + assert!( + registry_owner_handlers.contains(h.as_str()), + "handler `{h}` carries an owner-gate marker but is not an owner-gate \ + row in deny_bearing_routes() — add it so the runtime sweep drives its 403" + ); + } + } + + /// CALLER-SELF ORPHAN GUARD (#195, F3/R4, INV-21b). Sibling to the owner-gate + /// and read-gate orphan guards above: every handler carrying a caller-self + /// `did_matches(caller, X)` gate (X != `&record.owner_did`) must be a DRIVEN + /// registry row (ANY gate class: a `SignerSelfGate` field-binding row or a + /// `MultiPrincipalGate` assignee-arm row) OR listed in the (empty) + /// `SIGNER_SELF_NOT_DRIVEN`. Derived from a SOURCE scan of the `src/api` `.rs` + /// files at test time (KTD-2): a caller-self gate spelled with the inline + /// `did_matches(caller|&caller|&auth.0, X)` idiom in an existing OR new api file + /// is caught and forced to be a driven row. It does NOT catch two shapes (the + /// same limits the owner/read sibling guards have): a check hidden behind a + /// helper (the owner side tracks `require_owner` / `require_repo_owner` as markers + /// for exactly this reason -- add a marker if a caller-self helper is introduced), + /// and a gate in a new SUBDIRECTORY module (the scan is flat, not recursive). The + /// parallel GraphQL mutation gates (`src/graphql/mutation.rs`) are out of this + /// scan's scope, fenced separately (#219). + #[test] + fn signer_self_gates_are_all_driven() { + // A caller-self gate may be driven as a SignerSelfGate row (body-field bind) + // OR a MultiPrincipalGate row (the complete/fail assignee actor gate), so the + // required set is EVERY registry handler regardless of class. + let registry_handlers: HashSet<&str> = deny_bearing_routes() + .iter() + .map(|r| short_handler(r.handler)) + .collect(); + + let api_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/src/api"); + let mut signer_self_marked: HashSet = HashSet::new(); + for entry in std::fs::read_dir(api_dir).expect("read api dir") { + let path = entry.expect("dir entry").path(); + if path.extension().is_some_and(|e| e == "rs") { + let src = scan_src(&std::fs::read_to_string(&path).expect("read api file")); + signer_self_marked.extend(handlers_matching(&src, has_signer_self_did_matches)); + } + } + // Drop the gate helpers' own definitions (mirror of the owner scan dropping + // require_owner / require_repo_owner): none of these is a mounted handler. + signer_self_marked.remove("did_matches"); + signer_self_marked.remove("require_owner"); + signer_self_marked.remove("require_repo_owner"); + // task_claimable (tasks.rs) is a `pub(crate)` eligibility predicate, not a + // mounted handler: its `did_matches(caller, trim_assignee_did(assignee))` + // is the assignee arm that `claim_task`'s driven SignerSelfGate row already + // exercises through the real route. Same helper class as the removals + // above, not a SIGNER_SELF_NOT_DRIVEN REST gate. + signer_self_marked.remove("task_claimable"); + + // Non-vacuous floor PINNED to the exact caller-self handler count (close_pr, + // close_issue, dispute/submit/approve/cancel_bounty, create/claim/complete/ + // fail_task, register = 11). Pinned, not loose, so a PARTIAL classifier + // regression (dropping even one) trips here rather than being absorbed by the + // runtime sweep; bump it when the caller-self surface legitimately changes. + assert!( + signer_self_marked.len() >= 11, + "caller-self marker scan found only {} handlers (expected 11); the scan broke or the caller-self surface changed without bumping this floor", + signer_self_marked.len() + ); + + for h in &signer_self_marked { + assert!( + registry_handlers.contains(h.as_str()) + || SIGNER_SELF_NOT_DRIVEN.contains(&h.as_str()), + "handler `{h}` carries a caller-self did_matches gate but is neither a \ + driven registry row nor in SIGNER_SELF_NOT_DRIVEN; drive its 403" + ); + } + + // Staleness: every SIGNER_SELF_NOT_DRIVEN entry must still be a real + // caller-self handler. Vacuous while the list is empty, but it keeps a future + // exemption honest (the mirror of the owner/read staleness checks). + for name in SIGNER_SELF_NOT_DRIVEN { + assert!( + signer_self_marked.contains(*name), + "SIGNER_SELF_NOT_DRIVEN lists `{name}`, which no longer carries a \ + caller-self did_matches gate; update the list" + ); + } + } + + /// MULTI-PRINCIPAL INLINE ORPHAN GUARD (#195, F1). submit_bounty and + /// dispute_bounty spell their claimant arm with a local `is_claimant` boolean + /// instead of a bare `did_matches(&auth.0, claimant)` call, so the signer-self + /// classifier cannot catch them. This scan is the twin for that idiom: every + /// handler carrying it must be a `MultiPrincipalGate` registry row. + #[test] + fn multi_principal_inline_claimant_gates_are_all_driven() { + let registry_mp: HashSet<&str> = deny_bearing_routes() + .iter() + .filter(|r| r.gate == GateClass::MultiPrincipalGate) + .map(|r| short_handler(r.handler)) + .collect(); + + let api_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/src/api"); + let mut inline_mp: HashSet = HashSet::new(); + for entry in std::fs::read_dir(api_dir).expect("read api dir") { + let path = entry.expect("dir entry").path(); + if path.extension().is_some_and(|e| e == "rs") { + let src = scan_src(&std::fs::read_to_string(&path).expect("read api file")); + inline_mp.extend(handlers_matching(&src, has_claimant_inline_gate)); + } + } + + assert!( + inline_mp.len() >= 2, + "inline claimant gate scan found only {} handlers (expected submit_bounty + dispute_bounty)", + inline_mp.len() + ); + + for h in &inline_mp { + assert!( + registry_mp.contains(h.as_str()), + "handler `{h}` carries an inline claimant gate but is not a \ + MultiPrincipalGate row in deny_bearing_routes(); add it so the sweep \ + drives its 403" + ); + } + } + + /// READ-GATE ORPHAN GUARD (F1). The owner-gate orphan guard above stops an + /// owner-gated mount from escaping the sweep; this is its read-gate twin. Every + /// handler carrying a read-gate marker (`authorize_repo_read(` / + /// `visibility_check(` — the two markers repos.rs/issues.rs/pulls.rs/ + /// bounties.rs/certs.rs/labels.rs/changelog.rs/events.rs/encrypted.rs/stars.rs/ + /// visibility.rs all use, verified there is no third) must be EITHER a + /// `ReadGate` row in `deny_bearing_routes()` (driven for its 404) OR an explicit + /// entry in `READ_GATE_NOT_DRIVEN` with a reason. That enumerated allowlist + /// REPLACES the old free-text "DEFERRED / EXCLUDED" prose in routes.rs: it is + /// now enforced code, so removing or bypassing the read gate on any of these + /// endpoints (or adding a brand-new read-gated handler) trips this instead of + /// leaving the real-node sweep silently green. The api dir is read at test time, + /// like the owner scan, so a new module is covered too. + #[test] + fn every_read_gate_handler_is_driven_or_explicitly_allowlisted() { + // The allowlist is the module-level `READ_GATE_NOT_DRIVEN` (single source of + // truth, shared with the route-table cross-check). Its entries carry a closed + // `NotDrivenReason`, so a soft "deferred" excuse cannot be added. + + let driven_read_handlers: HashSet<&str> = deny_bearing_routes() + .iter() + .filter(|r| r.gate == GateClass::ReadGate) + .map(|r| short_handler(r.handler)) + .collect(); + let allowlisted: HashSet<&str> = READ_GATE_NOT_DRIVEN.iter().map(|(h, _)| *h).collect(); + + let api_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/src/api"); + let mut read_marked: HashSet = HashSet::new(); + for entry in std::fs::read_dir(api_dir).expect("read api dir") { + let path = entry.expect("dir entry").path(); + // Skip api/mod.rs: it holds the gate HELPERS themselves + // (`authorize_repo_read`, `visibility_check`, `require_repo_owner`, …), + // not mounted route handlers — every mounted read handler lives in a + // sibling module (repos.rs, issues.rs, …). Scanning mod.rs would attach + // the marker to helper definitions and force phantom allowlist entries. + if path.file_name().is_some_and(|f| f == "mod.rs") { + continue; + } + if path.extension().is_some_and(|e| e == "rs") { + let src = scan_src(&std::fs::read_to_string(&path).expect("read api file")); + // Comment-stripped scan: the read-gate markers appear in several + // docstrings, and a raw scan would allowlist phantom non-handlers. + read_marked.extend(handlers_with_code_marker( + &src, + &["authorize_repo_read(", "visibility_check("], + )); + } + } + // The read-gate helper `visibility_check` is re-declared/aliased in + // visibility.rs's own use-path; drop the helper name if the scan attaches + // the marker to it (it is a helper, not a mounted handler) — the mirror of + // the owner scan dropping require_owner / require_repo_owner. + read_marked.remove("authorize_repo_read"); + read_marked.remove("visibility_check"); + + // Non-vacuous floor: the tree has 41 read-gate-marked handlers today. If the + // marker scan silently found far fewer, the orphan loop below would pass by + // checking almost nothing; 30 trips only on a real parser break. + assert!( + read_marked.len() >= 30, + "read-gate marker scan found only {} handlers — the scan likely broke (floor 30)", + read_marked.len() + ); + + for h in &read_marked { + let name = h.as_str(); + assert!( + driven_read_handlers.contains(name) || allowlisted.contains(name), + "handler `{name}` carries a read-gate marker but is neither a ReadGate \ + row in deny_bearing_routes() nor in READ_GATE_NOT_DRIVEN — drive its \ + 404 or add it to the allowlist with a reason" + ); + } + + // Staleness: every allowlist entry must still be a real read-gate handler, + // so a rename/removal can't leave a dead exemption that masks a future gap. + for (name, _reason) in READ_GATE_NOT_DRIVEN { + assert!( + read_marked.contains(*name), + "READ_GATE_NOT_DRIVEN lists `{name}`, which no longer carries a \ + read-gate marker (renamed, removed, or gate dropped?) — update the list" + ); + } + + // DRIVEN-NOT-RELABELED (#195, U4/R5). The nine reads U3 drove must each be a + // driven `ReadGate` row AND absent from `READ_GATE_NOT_DRIVEN`, so none can + // be quietly re-parked under a mislabeled structural reason. A structural + // class (mutation / git / content-addressed / list-filter / external-dep) + // does not fit any of these nine, but a reviewer could still mistype one in; + // this makes that a hard failure rather than a judgment call. Together with + // the closed `NotDrivenReason` enum, a drivable private read cannot leave the + // sweep: either the class does not compile, or this assertion fires. + const U3_DRIVEN_READS: &[&str] = &[ + "get_issue", + "get_pr", + "get_pr_diff", + "list_issue_comments", + "list_reviews", + "list_comments", + "get_cert", + "get_bounty", + "get_tree", + ]; + for name in U3_DRIVEN_READS { + assert!( + driven_read_handlers.contains(name), + "U3 drove `{name}` as a private read, but it is no longer a ReadGate \ + row in deny_bearing_routes() — a driven read was dropped or renamed" + ); + assert!( + !allowlisted.contains(name), + "`{name}` is a U3-driven private read but was re-parked in \ + READ_GATE_NOT_DRIVEN — a drivable 404-deny GET may not be excused \ + under a structural reason; drive it, do not relabel it" + ); + } + } + + /// The repo-scoped GET mounts the [`scrape_mounts`] route-table walk does NOT + /// treat as read-gated — genuinely public repo listings that fetch the repo and + /// return metadata WITHOUT an `authorize_repo_read` / `visibility_check` call + /// (verified in their handlers). They carry no read gate, so the cross-check + /// below must not demand they be driven or structurally excused. Keyed by path + /// (the unit `scrape_mounts` yields) so a new gate added to one of these later + /// still surfaces via the marker scan; this list only says "no gate today". + /// + /// Currently EMPTY: `/hooks`, `/branches/protected`, and `/replicas` were listed + /// here as ungated public GETs, but #113 added an `authorize_repo_read` gate to + /// all three. They are now DRIVEN as real `ReadGate` rows in + /// `deny_bearing_routes()` (list_webhooks also as an OwnerGate row for its owner + /// layer), so they are classified via the driven-path set, not this list. The + /// mechanism is kept (not deleted) so the next genuinely-public repo-GET has a + /// home — the marker scan is the source of truth that keeps it honest. + const PUBLIC_REPO_GETS: &[&str] = &[]; + + /// True for a mounted path that is scoped to a single repo — the API form + /// `/api/v1/repos/{owner}/{repo}/…` and the bare git form `/{owner}/{repo}/…`. + /// These are the paths whose GET, if read-gated, must be driven or excused; a + /// collection/global path (`/api/v1/repos`, `/api/v1/bounties`, `/api/v1/agents` + /// …) is out of scope for the repo-read cross-check. + fn is_repo_scoped_path(path: &str) -> bool { + if let Some(rest) = path.strip_prefix("/api/v1/repos/") { + // `.../{owner}/{repo}/…` — at least owner + repo + a trailing segment, + // and NOT the `/api/v1/repos/federated` or bare `/api/v1/repos` forms + // (those have no `{owner}` placeholder). + return rest.starts_with("{owner}/{repo}/"); + } + // Bare git form: `/{owner}/{repo}/info/refs` etc. It is repo-scoped when the + // first two segments are the owner/repo placeholders. + let segs: Vec<&str> = path.trim_start_matches('/').split('/').collect(); + segs.len() >= 3 && segs[0] == "{owner}" && segs[1] == "{repo}" + } + + /// Extract the handler name from a single `.route("path", …get(handler)…)` mount + /// body: the last `::`-tail ident inside the FIRST routing-method call for the + /// requested HTTP method. Returns `None` if the method is not mounted on this + /// route. Reuses the same balanced-paren mount slice `scrape_mounts` walks. + fn handler_for_mount(call: &str, method: &str) -> Option { + let needle = format!("{}(", method.to_lowercase()); + let mut k = 0; + while let Some(rel) = call[k..].find(&needle) { + let at = k + rel; + let boundary = call[..at] + .chars() + .last() + .map(|c| !(c.is_alphanumeric() || c == '_')) + .unwrap_or(true); + if boundary { + // The handler is the first path-expression after `get(`; take up to + // the closing `)` and keep the final `::`-segment ident. + let inner_start = at + needle.len(); + let inner = &call[inner_start..]; + let end = inner.find([')', ',']).unwrap_or(inner.len()); + let expr = inner[..end].trim(); + let name: String = expr + .rsplit("::") + .next() + .unwrap_or(expr) + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect(); + if !name.is_empty() { + return Some(name); + } + } + k = at + needle.len(); + } + None + } + + /// Like [`scrape_mounts`] but also captures the handler name, so a mounted + /// (METHOD, path) can be resolved to its handler for the read-gate cross-check. + /// Same balanced-paren walk as `scrape_mounts`. + fn scrape_mounts_with_handler(src: &str) -> Vec<(String, String, String)> { + let bytes = src.as_bytes(); + let mut out = Vec::new(); + let mut i = 0; + while let Some(rel) = src[i..].find(".route(") { + let open = i + rel + ".route(".len(); + let mut depth = 1i32; + let mut j = open; + while j < src.len() && depth > 0 { + match bytes[j] { + b'(' => depth += 1, + b')' => depth -= 1, + _ => {} + } + j += 1; + } + let call = &src[open..j.saturating_sub(1)]; + i = j; + let Some(qs) = call.find('"') else { continue }; + let Some(qe) = call[qs + 1..].find('"') else { + continue; + }; + let path = &call[qs + 1..qs + 1 + qe]; + for m in ["get", "post", "put", "delete", "patch"] { + let needle = format!("{m}("); + let mut k = 0; + while let Some(mrel) = call[k..].find(&needle) { + let at = k + mrel; + let boundary = call[..at] + .chars() + .last() + .map(|c| !(c.is_alphanumeric() || c == '_')) + .unwrap_or(true); + if boundary { + if let Some(h) = handler_for_mount(call, m) { + out.push((m.to_uppercase(), path.to_string(), h)); + } + } + k = at + needle.len(); + } + } + } + out + } + + /// SCRAPE-MOUNTS CROSS-CHECK (#195, U4/R5, INV-21b). The read-gate orphan guard + /// above derives its required set from a SOURCE-MARKER scan of `src/api`; this + /// derives it instead from the MOUNTED ROUTE TABLE (`server.rs`). Every mounted + /// repo-scoped GET must be EITHER a driven `ReadGate` row, OR a deny-bearing GET + /// of another class already in the registry (`list_visibility` is an owner-gated + /// GET), OR structurally excused in `READ_GATE_NOT_DRIVEN`, OR a declared public + /// repo-GET (`PUBLIC_REPO_GETS`). A mounted repo-GET that is none of these fails + /// HERE — which catches a read that gates via a helper the marker scan does not + /// recognize: it would still be mounted, so it must be classified rather than + /// slipping past silently. + #[test] + fn mounted_repo_gets_are_driven_excused_or_declared_public() { + let server = include_str!("../src/server.rs"); + let mounts = scrape_mounts_with_handler(server); + + // Integrity floor: the handler-aware walk must find roughly as many mounts + // as the plain scrape; if it collapsed, the cross-check would pass vacuously. + assert!( + mounts.len() >= 90, + "handler-aware mount scrape found only {} routes — the parser likely broke (floor 90)", + mounts.len() + ); + + // Every path that is a driven ReadGate row, and every deny-bearing GET path + // of any class (so the owner-gated `list_visibility` GET is accounted for). + let driven_read_paths: HashSet<&str> = deny_bearing_routes() + .iter() + .filter(|r| r.gate == GateClass::ReadGate) + .map(|r| r.path) + .collect(); + let deny_bearing_get_paths: HashSet<&str> = deny_bearing_routes() + .iter() + .filter(|r| r.method == "GET") + .map(|r| r.path) + .collect(); + let excused_handlers: HashSet<&str> = + READ_GATE_NOT_DRIVEN.iter().map(|(h, _)| *h).collect(); + let public_paths: HashSet<&str> = PUBLIC_REPO_GETS.iter().copied().collect(); + + let mut repo_gets_seen = 0usize; + for (method, path, handler) in &mounts { + if method != "GET" || !is_repo_scoped_path(path) { + continue; + } + repo_gets_seen += 1; + let classified = driven_read_paths.contains(path.as_str()) + || deny_bearing_get_paths.contains(path.as_str()) + || excused_handlers.contains(handler.as_str()) + || public_paths.contains(path.as_str()); + assert!( + classified, + "mounted repo-scoped GET `{path}` (handler `{handler}`) is neither a \ + driven ReadGate row, a deny-bearing GET row, structurally excused in \ + READ_GATE_NOT_DRIVEN, nor a declared public repo-GET — classify it \ + (it may gate via a helper the source-marker scan does not recognize)" + ); + } + + // Non-vacuous floor: the tree mounts well over a dozen repo-scoped GETs; if + // the filter found almost none, the loop above proved nothing. + assert!( + repo_gets_seen >= 15, + "found only {repo_gets_seen} mounted repo-scoped GETs — the path filter \ + or the scrape likely broke (floor 15)" + ); + } + + // ── F3 unit tests: the inline owner-gate did_matches discriminator ─────────── + // + // These pin `has_owner_did_matches` against the EXACT real bodies (positive and + // negative), so the two-arg discriminator can't silently regress into matching + // the signer-self / author forms that share the helper. + + #[test] + fn has_owner_did_matches_catches_the_protect_branch_owner_idiom() { + // protect.rs:28 form: first arg the caller, second `&record.owner_did`. + let body = "let caller = &auth.0;\n\ + if !crate::api::did_matches(caller, &record.owner_did) {\n\ + return Err(AppError::Forbidden(\"only the owner\".into()));\n\ + }"; + assert!( + has_owner_did_matches(body), + "the protect_branch owner idiom must be recognized" + ); + } + + #[test] + fn has_owner_did_matches_catches_the_auth0_owner_form() { + // The `&auth.0` first-arg spelling (repos.rs's branch-protection check), + // tolerating a newline before the second arg. + let body = + "if x\n && !crate::api::did_matches(&auth.0,\n &record.owner_did)\n {"; + assert!( + has_owner_did_matches(body), + "the &auth.0 / owner_did form must be recognized across a newline" + ); + } + + #[test] + fn has_owner_did_matches_ignores_register_replica_signer_self() { + // replicas.rs:52: FIRST arg is `replica_did`, NOT the caller — signer-self + // by design. The second arg is owner_did, so a naive "second arg is + // owner_did" check would wrongly flag it. It must NOT match. + let body = "if crate::api::did_matches(replica_did, &record.owner_did) {\n\ + // the signer is registering itself as a replica\n\ + }"; + assert!( + !has_owner_did_matches(body), + "register_replica's signer-self did_matches must NOT be treated as an owner gate" + ); + } + + #[test] + fn has_owner_did_matches_ignores_close_pr_author_form() { + // pulls.rs:277: caller-first but SECOND arg is `&pr.author_did`, not + // owner_did — the owner-or-author close gate. Must NOT match as owner-only. + let body = "let is_author = crate::api::did_matches(&auth.0, &pr.author_did);"; + assert!( + !has_owner_did_matches(body), + "close_pr's author did_matches must NOT be treated as an owner gate" + ); + } + + #[test] + fn has_owner_did_matches_ignores_bounty_and_task_self_forms() { + // bounties.rs / tasks.rs: caller-first, but the second arg is a + // creator/delegator/assignee did, never owner_did. + for body in [ + "if !crate::api::did_matches(&auth.0, &bounty.creator_did) {", + "if !crate::api::did_matches(&auth.0, &body.delegator_did) {", + "if !crate::api::did_matches(&auth.0, &body.assignee_did) {", + ] { + assert!( + !has_owner_did_matches(body), + "self/author did_matches form must NOT be treated as an owner gate: {body}" + ); + } + } + + // ── F3 unit tests: the caller-self did_matches discriminator ───────────────── + + #[test] + fn has_signer_self_did_matches_fires_on_caller_self_and_ignores_owner_form() { + // Fires: caller-spelled first arg (bound via `let caller = &auth.0;`), a + // non-owner second arg, the field-binding form the task handlers use. Proves + // the arg1 set is honored (a `&auth.0`-only classifier would miss this). + let self_form = "let caller = &auth.0;\n\ + if !crate::api::did_matches(caller, &x.foo_did) {\n\ + return Err(forbidden(\"...\"));\n\ + }"; + assert!( + has_signer_self_did_matches(self_form), + "the caller-self field-binding did_matches must be recognized" + ); + // The `&auth.0` spelling with a non-owner second arg fires too. + let auth0_form = "if !crate::api::did_matches(&auth.0, &body.assignee_did) {"; + assert!( + has_signer_self_did_matches(auth0_form), + "the &auth.0 caller-self form must be recognized" + ); + // Does NOT fire on the owner form (that is has_owner_did_matches' job), so a + // caller-self classifier that matched it would double-count. + let owner_form = "if !crate::api::did_matches(caller, &record.owner_did) {"; + assert!( + !has_signer_self_did_matches(owner_form), + "the owner form must NOT be treated as a caller-self gate (no double-count)" + ); + } + + #[test] + fn has_claimant_inline_gate_fires_on_submit_and_dispute() { + let submit = "let is_claimant = bounty\n.claimant_did\n.as_deref()\n\ + .is_some_and(|c| did_matches(&auth.0, c));\n\ + if !is_claimant {\n return Err(AppError::Forbidden(\"only the claimant\".into())); }"; + assert!(has_claimant_inline_gate(submit)); + let dispute = "let is_creator = did_matches(&auth.0, &bounty.creator_did);\n\ + let is_claimant = bounty.claimant_did.as_deref().is_some_and(...);\n\ + if !is_creator && !is_claimant {\n return Err(AppError::Forbidden(...)); }"; + assert!(has_claimant_inline_gate(dispute)); + } + + #[test] + fn has_claimant_inline_gate_ignores_read_only_claimant_use() { + let body = "if let Some(ref agent_did) = bounty.claimant_did {\n\ + let current = state.db.get_trust_score(agent_did);\n\ + }"; + assert!( + !has_claimant_inline_gate(body), + "post-gate claimant_did reads must not trip the inline claimant classifier" + ); + } + + // ── Shutdown-completeness guard: every spawn_node test must shutdown() ──────── + + /// Blank the CONTENT of comments and double-quoted / raw string literals in `src` + /// (length-preserving, newlines kept) so the brace slicer and the marker scans see + /// CODE only. Without it this guard carries two vacuity holes of exactly the class + /// it exists to close (INV-21 on the guard itself): a `{` inside a format string + /// unbalances the body slice and can swallow a later test's `shutdown()`, and a + /// `// node.shutdown()` comment satisfies the presence check while the test still + /// returns through `Drop`. Handles `//` and nested `/* */` comments, `"..."` (with + /// `\` escapes), and hash-delimited raw strings (`r#"..."#`, `br#"..."#`). Char + /// literals and lifetimes are left as code: they carry neither braces-of-interest + /// nor the shutdown marker, and telling a `'a` lifetime from a `'a'` char would add + /// hazard for no coverage. + fn code_only(src: &str) -> String { + fn blank(out: &mut [u8], from: usize, to: usize) { + for c in &mut out[from..to] { + if *c != b'\n' { + *c = b' '; + } + } + } + let b = src.as_bytes(); + let n = b.len(); + let mut out = b.to_vec(); + let mut i = 0; + while i < n { + // Line comment: `//` … EOL. + if b[i] == b'/' && i + 1 < n && b[i + 1] == b'/' { + let start = i; + while i < n && b[i] != b'\n' { + i += 1; + } + blank(&mut out, start, i); + continue; + } + // Block comment `/* … */`, nesting allowed (Rust nests them). + if b[i] == b'/' && i + 1 < n && b[i + 1] == b'*' { + let start = i; + let mut depth = 1i32; + i += 2; + while i < n && depth > 0 { + if b[i] == b'/' && i + 1 < n && b[i + 1] == b'*' { + depth += 1; + i += 2; + } else if b[i] == b'*' && i + 1 < n && b[i + 1] == b'/' { + depth -= 1; + i += 2; + } else { + i += 1; + } + } + blank(&mut out, start, i.min(n)); + continue; + } + // Raw string `r"…"` / `r#"…"#` (also the b-prefixed byte form, entered at + // the `r`). A raw ident `r#foo` has no `"` after the hashes and falls through. + if b[i] == b'r' && i + 1 < n && (b[i + 1] == b'"' || b[i + 1] == b'#') { + let mut h = i + 1; + let mut hashes = 0usize; + while h < n && b[h] == b'#' { + hashes += 1; + h += 1; + } + if h < n && b[h] == b'"' { + let content = h + 1; + let mut k = content; + let mut close = None; + while k < n { + if b[k] == b'"' { + let mut extra = 0; + while k + 1 + extra < n && extra < hashes && b[k + 1 + extra] == b'#' { + extra += 1; + } + if extra == hashes { + close = Some(k); + break; + } + } + k += 1; + } + blank(&mut out, content, close.unwrap_or(n)); + i = close.map(|c| c + 1 + hashes).unwrap_or(n); + continue; + } + } + // Normal string `"…"`, honoring `\` escapes (so `\"` does not close it). + if b[i] == b'"' { + let content = i + 1; + let mut k = content; + while k < n { + if b[k] == b'\\' { + k += 2; + continue; + } + if b[k] == b'"' { + break; + } + k += 1; + } + blank(&mut out, content, k.min(n)); + i = if k >= n { n } else { k + 1 }; + continue; + } + i += 1; + } + String::from_utf8(out).expect("code_only blanks whole literal spans, so utf-8 survives") + } + + /// The spawn_node tests that fail the shutdown contract: they spawn a node, are not + /// allowlisted, and their (code-only) body carries no `node.shutdown(` call. Pure so + /// the adversarial unit tests below can drive synthetic sources; the `#[test]` guard + /// runs it over this file's real, sanitized source. + fn spawn_node_tests_missing_shutdown( + tests: &[(String, String)], + allow: &[&str], + ) -> Vec { + tests + .iter() + .filter(|(name, body)| { + body.contains("spawn_node") + && !allow.contains(&name.as_str()) + && !body.contains("node.shutdown(") + }) + .map(|(name, _)| name.clone()) + .collect() + } + + /// `(name, body)` for every real `#[sqlx::test]` in `src`. `src` is expected to be + /// [`code_only`]-sanitized so string/comment content cannot unbalance the body + /// slice. The attribute is only counted when nothing but whitespace precedes it on + /// its line (belt-and-suspenders with sanitization, which already blanks any + /// backtick mention inside a doc comment, e.g. the module header naming the + /// attribute in prose, so it is not mistaken for a test). The needle is spelled in + /// two pieces so this + /// scanner does not match its own source when it scrapes this file. The body is + /// brace-matched from the signature's opening `{` to its close, so a marker cannot + /// leak across into the next item. + fn sqlx_test_bodies(src: &str) -> Vec<(String, String)> { + // Split so the literal attribute never appears contiguously in this fn's own + // source (belt-and-suspenders with the whitespace-before-on-line check below). + let attr = concat!("#[sqlx", "::test]"); + let bytes = src.as_bytes(); + let mut out = Vec::new(); + let mut search = 0; + while let Some(rel) = src[search..].find(attr) { + let at = search + rel; + search = at + attr.len(); + // Real attribute only: the text between the line start and the attribute + // must be blank (rejects `//! … #[sqlx::test] …` and any string mention). + let line_start = src[..at].rfind('\n').map(|n| n + 1).unwrap_or(0); + if !src[line_start..at].trim().is_empty() { + continue; + } + let Some(fnrel) = src[search..].find("fn ") else { + break; + }; + let name_start = search + fnrel + "fn ".len(); + let name: String = src[name_start..] + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect(); + if name.is_empty() { + continue; + } + let Some(brel) = src[name_start..].find('{') else { + break; + }; + let open = name_start + brel; + let mut depth = 0i32; + let mut j = open; + while j < src.len() { + match bytes[j] { + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 { + j += 1; + break; + } + } + _ => {} + } + j += 1; + } + out.push((name, src[open..j].to_string())); + search = j; + } + out + } + + /// Every `#[sqlx::test]` that spawns a node must end it with + /// `node.shutdown().await`, so the serve task's pool clones are released before + /// `#[sqlx::test]`'s synchronous `DROP DATABASE` cleanup fires. A test that returns + /// via `TestNode`'s async `Drop` instead only *signals* teardown (it drains on a + /// later scheduler tick — the Drop-regression tests poll up to 5s for it), so it + /// races the cleanup and can leak the per-test database / flake under parallelism. + /// The module header states this contract in prose; this makes it load-bearing — + /// removing any one `shutdown()` turns this RED. + /// + /// The two Drop-regression tests are the CLOSED allowlist: they deliberately + /// return WITHOUT `shutdown()` to exercise the Drop teardown, so they must NOT call + /// it. The allowlist is checked both ways — an allowlisted name that no longer + /// exists (rename/removal) or that grows a `shutdown()` call fails here — so it + /// cannot silently rot into covering a test it no longer describes. + #[test] + fn every_spawn_node_test_shuts_the_node_down() { + // The ONLY tests allowed to return without shutdown(): they test Drop itself. + const DROP_REGRESSION_ALLOW: &[&str] = &[ + "drop_without_shutdown_unblocks_database", + "drop_with_broken_graceful_chain_still_unblocks_via_abort", + ]; + + let src = code_only(include_str!("deny_harness.rs")); + let tests = sqlx_test_bodies(&src); + + // Non-vacuous floor: if the scraper silently found nothing, every loop below + // would pass by checking zero tests. The file has 14 sqlx::test fns today. + assert!( + tests.len() >= 12, + "sqlx::test scrape found only {} tests — the parser likely broke (floor 12)", + tests.len() + ); + + // Each allowlisted name must resolve to a real spawn_node test that (correctly) + // does NOT call shutdown — so the allowlist cannot cover a renamed/deleted test + // or one that later grew a shutdown() call and no longer exercises Drop. + for allowed in DROP_REGRESSION_ALLOW { + let (_, body) = tests + .iter() + .find(|(name, _)| name == allowed) + .unwrap_or_else(|| { + panic!( + "allowlisted Drop-regression test `{allowed}` not found — renamed or \ + removed? update DROP_REGRESSION_ALLOW" + ) + }); + assert!( + body.contains("spawn_node"), + "allowlisted test `{allowed}` no longer spawns a node — it does not belong \ + on the shutdown allowlist" + ); + assert!( + !body.contains("node.shutdown("), + "allowlisted Drop-regression test `{allowed}` now calls shutdown() — it no \ + longer exercises Drop-only teardown; remove it from DROP_REGRESSION_ALLOW" + ); + } + + // Every other spawn_node test must shut its node down. + let missing = spawn_node_tests_missing_shutdown(&tests, DROP_REGRESSION_ALLOW); + assert!( + missing.is_empty(), + "these tests spawn a node but never call node.shutdown().await — they return \ + via TestNode::Drop, racing sqlx's DROP DATABASE cleanup (add the awaited \ + shutdown, or add to DROP_REGRESSION_ALLOW if they deliberately test Drop): \ + {missing:?}" + ); + } + + // ── Adversarial unit tests for the guard itself (INV-21 on the guard) ──────── + // These drive synthetic sources so the two vacuity holes an external review + // flagged are proven CLOSED by execution: a commented-out shutdown, and an + // unbalanced brace inside a string that would otherwise swallow a sibling's + // shutdown. Each passes (false green) against a raw substring/brace scan; + // sanitizing via `code_only` first is what makes them fail as they must. + + #[test] + fn shutdown_guard_flags_a_commented_out_shutdown() { + // The ONLY node.shutdown( occurrence is a comment: the node is never torn + // down, so the test must be flagged despite the textual match. + let synth = concat!( + "#[sqlx", + "::test]\n", + "async fn foo(pool: PgPool) {\n", + " let node = spawn_node(pool).await;\n", + " // node.shutdown().await; (deleted, left as a comment)\n", + "}\n" + ); + let tests = sqlx_test_bodies(&code_only(synth)); + assert_eq!( + spawn_node_tests_missing_shutdown(&tests, &[]), + vec!["foo".to_string()], + "a commented-out shutdown must NOT satisfy the guard" + ); + } + + #[test] + fn shutdown_guard_is_not_fooled_by_a_brace_inside_a_string() { + // `a` has no shutdown and an unbalanced `{` inside a string; `b` follows and + // does shut down. A brace-blind slicer would extend `a` into `b`, borrow b's + // shutdown, and clear `a` — sanitization keeps the two bodies separate. + let synth = concat!( + "#[sqlx", + "::test]\n", + "async fn a(pool: PgPool) {\n", + " let node = spawn_node(pool).await;\n", + " let _ = \"oops {\";\n", + "}\n", + "#[sqlx", + "::test]\n", + "async fn b(pool: PgPool) {\n", + " let node = spawn_node(pool).await;\n", + " node.shutdown().await;\n", + "}\n" + ); + let tests = sqlx_test_bodies(&code_only(synth)); + let missing = spawn_node_tests_missing_shutdown(&tests, &[]); + assert!( + missing.contains(&"a".to_string()), + "test `a` (no shutdown; unbalanced brace in a string) must be flagged, not \ + masked by test `b`'s shutdown: got {missing:?}" + ); + assert!( + !missing.contains(&"b".to_string()), + "test `b` shuts down and must not be flagged: {missing:?}" + ); + } + + #[test] + fn shutdown_guard_passes_clean_and_exempts_only_via_allowlist() { + let synth = concat!( + "#[sqlx", + "::test]\n", + "async fn good(pool: PgPool) {\n", + " let node = spawn_node(pool).await;\n", + " node.shutdown().await;\n", + "}\n", + "#[sqlx", + "::test]\n", + "async fn drop_probe(pool: PgPool) {\n", + " let node = spawn_node(pool).await;\n", + " // deliberately no shutdown: exercises Drop\n", + "}\n" + ); + let tests = sqlx_test_bodies(&code_only(synth)); + // Clean test + allowlisted Drop test -> no offenders. + assert!( + spawn_node_tests_missing_shutdown(&tests, &["drop_probe"]).is_empty(), + "a clean test plus an allowlisted Drop test must produce no offenders" + ); + // Without the allowlist the Drop test IS an offender, proving the exemption + // comes from the allowlist, not from an accident of the scan. + assert_eq!( + spawn_node_tests_missing_shutdown(&tests, &[]), + vec!["drop_probe".to_string()], + "un-allowlisted, the no-shutdown test must be flagged" + ); + } +} diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs index af844a5ec..8fd6e48ef 100644 --- a/crates/gitlawb-node/tests/inv22_gates.rs +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -715,11 +715,11 @@ fn inv26_step3_live_and_drain_share_apply_request_effects() { } /// #26 Split PR 1 step 4 — the periodic queue-lifecycle purge is -/// wired in `main.rs` and the contract is enforced by the `idx_receive_pack_requests_completed_at` +/// wired in `lib.rs` and the contract is enforced by the `idx_receive_pack_requests_completed_at` /// partial index from v41. /// /// Assertions: -/// 1. `main.rs` calls `purge_request_queue` once at boot (well, on +/// 1. `lib.rs` calls `purge_request_queue` once at boot (well, on /// the spawn-task interval) and the spawn function exists. /// 2. The DB helpers `purge_completed_receive_pack_requests` and /// `purge_completed_pending_ref_transitions` exist in `db/mod.rs`. @@ -729,14 +729,14 @@ fn inv26_step3_live_and_drain_share_apply_request_effects() { /// introduce the state, but the invariant holds for the future). #[test] fn inv26_step4_queue_lifecycle_purge_is_wired() { - let main_src = src("main.rs"); + let boot_src = src("lib.rs"); assert!( - main_src.contains("spawn_queue_lifecycle_sweep"), - "main.rs must spawn the periodic queue-lifecycle purge" + boot_src.contains("spawn_queue_lifecycle_sweep"), + "lib.rs must spawn the periodic queue-lifecycle purge" ); assert!( - main_src.contains("purge_request_queue"), - "main.rs must call purge_request_queue on the periodic sweep" + boot_src.contains("purge_request_queue"), + "lib.rs must call purge_request_queue on the periodic sweep" ); let db_src = src("db/mod.rs"); diff --git a/crates/gitlawb-node/tests/support/assert.rs b/crates/gitlawb-node/tests/support/assert.rs new file mode 100644 index 000000000..eaf1697ea --- /dev/null +++ b/crates/gitlawb-node/tests/support/assert.rs @@ -0,0 +1,235 @@ +//! INV-8 deny assertion. A denial must be an actual refusal (4xx) whose body +//! and headers carry none of the data it is withholding, and must never be a 2xx +//! (an empty-200 rendered as success is the denial-as-success bug this guards). +//! +//! The check is split into a pure `check_denied` (unit-tested below with the +//! self-check scenarios) and an async `assert_denied` wrapper that reads a real +//! `reqwest::Response`. + +/// Pure deny check. Returns `Err(reason)` when the response is not a clean +/// denial. `withheld` are exact tokens (OIDs, paths, private slugs) that must +/// not appear anywhere in the body; register short-OID prefixes and encoded +/// path forms as separate tokens at the call site (a raw substring scan cannot +/// normalize them). +pub fn check_denied( + status: u16, + body: &str, + expected: u16, + withheld: &[&str], +) -> Result<(), String> { + // The harness only ever expects denials; a non-4xx expectation is a + // programming error in the test, not a node behavior under test. + if !(400..500).contains(&expected) { + return Err(format!( + "test bug: assert_denied expected status {expected} is not a 4xx" + )); + } + // INV-8: a denial rendered as success is the headline bug. Catch it + // explicitly so the failure message names it, rather than only failing the + // status-equality check below. + if (200..300).contains(&status) { + return Err(format!( + "denial rendered as success: got {status}, expected denial {expected}. \ + body={body:?}" + )); + } + if status != expected { + return Err(format!( + "expected denial status {expected}, got {status}. body={body:?}" + )); + } + for token in withheld { + if token.is_empty() { + continue; + } + if body.contains(token) { + return Err(format!( + "withheld token {token:?} leaked in denial body: {body:?}" + )); + } + } + Ok(()) +} + +/// Like [`check_denied`] but also scans serialized response headers for withheld +/// tokens. Call sites that only have a body should keep using `check_denied`. +pub fn check_denied_with_headers( + status: u16, + body: &str, + header_text: &str, + expected: u16, + withheld: &[&str], +) -> Result<(), String> { + check_denied(status, body, expected, withheld)?; + for token in withheld { + if token.is_empty() { + continue; + } + if header_text.contains(token) { + return Err(format!( + "withheld token {token:?} leaked in denial headers: {header_text:?}" + )); + } + } + Ok(()) +} + +/// Replace each withheld token in `s` with `[REDACTED]` so a test-failure +/// witness does not itself leak the secret it caught. +fn redact_withheld(s: &str, withheld: &[&str]) -> String { + let mut result = s.to_string(); + for token in withheld { + if !token.is_empty() { + result = result.replace(token, "[REDACTED]"); + } + } + result +} + +/// Drive a real response through the deny check and panic on failure. Reads the +/// full body once. +pub async fn assert_denied(resp: reqwest::Response, expected: u16, withheld: &[&str]) { + let status = resp.status().as_u16(); + let header_text = resp + .headers() + .iter() + .filter_map(|(name, value)| { + value + .to_str() + .ok() + .map(|v| format!("{}: {}", name.as_str(), v)) + }) + .collect::>() + .join("\n"); + // A body-read failure must NOT fold to "" — an empty body passes the withheld + // scan vacuously, so an unreadable denial (a mid-body reset/truncation) would be + // certified leak-free without ever inspecting the bytes. Fail loud instead. A + // legitimately empty body is `Ok("")`, not `Err`, so the honest empty-401 denials + // still pass. + let body = resp + .text() + .await + .expect("read denial body: an unreadable denial cannot be certified leak-free"); + if let Err(reason) = check_denied_with_headers(status, &body, &header_text, expected, withheld) + { + // Panic with a fixed string: CodeQL's taint analysis traces withheld + // tokens from check_denied through redact_withheld into the panic sink, + // even though redact_withheld strips them at runtime. A fixed message + // keeps no tainted data on the sink path. The test name identifies + // which probe failed; check_denied's unit tests cover the error detail. + let _ = redact_withheld(&reason, withheld); + panic!("deny assertion failed: status/body/header leak check"); + } +} + +#[cfg(test)] +mod tests { + use super::{check_denied, check_denied_with_headers, redact_withheld}; + + #[test] + fn clean_403_with_no_leak_passes() { + let r = check_denied( + 403, + r#"{"error":"only the repo owner can do this"}"#, + 403, + &["a1b2c3"], + ); + assert!(r.is_ok(), "{r:?}"); + } + + #[test] + fn empty_200_is_flagged_as_denial_rendered_as_success() { + let r = check_denied(200, "", 403, &[]); + assert!(r.is_err()); + assert!(r.unwrap_err().contains("rendered as success")); + } + + #[test] + fn leaking_403_fails_and_names_the_token() { + let withheld = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; + let body = format!("not found near object {withheld}"); + let r = check_denied(403, &body, 403, &[withheld]); + assert!(r.is_err()); + assert!(r.unwrap_err().contains(withheld)); + } + + #[test] + fn wrong_status_fails() { + // A 404 when the owner gate's 403 was expected must not pass: it means + // an earlier layer denied, so the gate under test was never exercised. + let r = check_denied(404, "not found", 403, &[]); + assert!(r.is_err()); + assert!(r.unwrap_err().contains("expected denial status 403")); + } + + #[test] + fn non_4xx_expected_is_a_test_bug() { + // Guards the harness itself: expecting a non-denial status is a test + // authoring error, not a node behavior, and must be rejected. + let r = check_denied(403, "denied", 200, &[]); + assert!(r.is_err()); + assert!(r.unwrap_err().contains("not a 4xx")); + } + + #[test] + fn empty_withheld_token_never_false_positives() { + // An empty token must be skipped, never treated as "contained in every + // body" — otherwise every denial would spuriously fail as a leak. + let r = check_denied(403, "any body", 403, &[""]); + assert!(r.is_ok(), "{r:?}"); + } + + #[test] + fn leaking_header_fails_and_names_the_token() { + let withheld = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; + let headers = format!("X-Debug: object {withheld}"); + let r = check_denied_with_headers( + 404, + r#"{"error":"repository not found"}"#, + &headers, + 404, + &[withheld], + ); + assert!(r.is_err()); + let err = r.unwrap_err(); + assert!(err.contains("leaked in denial headers")); + assert!(err.contains(withheld)); + } + + #[test] + fn clean_headers_with_clean_body_passes() { + let withheld = "a1b2c3"; + let r = check_denied_with_headers( + 404, + r#"{"error":"repository not found"}"#, + "Content-Type: application/json", + 404, + &[withheld], + ); + assert!(r.is_ok(), "{r:?}"); + } + + #[test] + fn redact_withheld_replaces_secrets() { + let withheld = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; + let msg = format!("withheld token {withheld:?} leaked in body: near {withheld}"); + let redacted = redact_withheld(&msg, &[withheld]); + assert!( + !redacted.contains(withheld), + "token must be redacted: {redacted}" + ); + assert!( + redacted.contains("[REDACTED]"), + "redaction marker must appear: {redacted}" + ); + } + + #[test] + fn redact_withheld_leaves_clean_strings() { + let result = redact_withheld( + "no secrets here", + &["deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"], + ); + assert_eq!(result, "no secrets here"); + } +} diff --git a/crates/gitlawb-node/tests/support/mod.rs b/crates/gitlawb-node/tests/support/mod.rs new file mode 100644 index 000000000..865449e2d --- /dev/null +++ b/crates/gitlawb-node/tests/support/mod.rs @@ -0,0 +1,33 @@ +//! Shared support for the real-node deny harness: the deny assertion (U4), the +//! RFC-9421 signing client (U2), and (via `gitlawb_node::test_harness`) the +//! node spawner (U3). + +pub mod assert; +// U2 fixture state matrix + per-gate-class probe generators; driven by U3. +#[allow(dead_code)] +pub mod probe; +// U1 deny-bearing route registry; fields consumed by U2/U3 as they land. +#[allow(dead_code)] +pub mod routes; +pub mod signing; + +/// A reqwest client with a bounded request timeout, so a wedged git subprocess or +/// route fails the suite rather than hanging it until CI kills the job (#195, F1). +/// The two real-node probes that drive git subprocesses use this; the inline +/// twins at the upload-pack and registry-sweep sites use the same pattern. +#[allow(dead_code)] +pub fn bounded_client() -> reqwest::Client { + reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .unwrap() +} + +/// Build an anonymous (unsigned) GET request for `path` under `base_url`. +/// Extracted from inline call sites so URL construction goes through a +/// function boundary, matching the `signed_request` pattern in `support::signing`. +#[allow(dead_code)] +pub fn anon_get(client: &reqwest::Client, base_url: &str, path: &str) -> reqwest::RequestBuilder { + let url = format!("{base_url}{path}"); + client.get(url) +} diff --git a/crates/gitlawb-node/tests/support/probe.rs b/crates/gitlawb-node/tests/support/probe.rs new file mode 100644 index 000000000..c4befea9d --- /dev/null +++ b/crates/gitlawb-node/tests/support/probe.rs @@ -0,0 +1,1438 @@ +//! U2: fixture state matrix + per-gate-class probe generators. +//! +//! The fixture is a STATE MATRIX, not one repo: owner-gate rows need a repo the +//! signed stranger can REACH (so the request hits the owner gate, not an +//! existence-hiding 404), and read-gate rows need a repo the caller CANNOT see. +//! A single repo cannot be both, so we seed a public repo (owner-gate substrate) +//! and a private repo with content (read-gate substrate). +//! +//! Each deny-bearing row expands to probes: the hostile request (asserting the +//! exact deny) plus a positive twin (owner-reachability for owner-gate, +//! reader/sibling-public read for read-gate) so a 403/404 for the wrong reason +//! cannot false-pass. + +use gitlawb_core::identity::Keypair; +use gitlawb_node::test_harness::TestNode; + +use super::routes::{AnonRead, GateClass, IdSource, Principal, Reach, Row}; + +/// A seeded two-repo state matrix plus the owner and stranger identities. +pub struct Fixture { + pub owner: Keypair, + pub stranger: Keypair, + /// The PR/issue author — distinct from owner and stranger — so the + /// owner-OR-author close gates (#195, F1) drive their author arm with an + /// identity that is NOT the owner (the original bug seeded author == owner). + pub author: Keypair, + /// The bounty creator, distinct from every other identity. + pub creator: Keypair, + /// The bounty claimant, distinct from every other identity. + pub claimant: Keypair, + /// The task assignee, distinct from every other identity (#195, F3). The + /// complete/fail actor gates drive their Assignee arm as this identity, so it + /// must differ from owner and stranger or the arm cannot false-pass on an + /// owner fallback. + pub assignee: Keypair, + pub owner_did: String, + pub author_did: String, + pub creator_did: String, + pub claimant_did: String, + pub assignee_did: String, + /// The id (UUID) of the seeded disputable bounty, filled into the + /// `dispute_bounty` row's `{id}` placeholder (IdSource::BountyId). + pub bounty_id: String, + /// #195 (N2): one bounty per status-gated bounty mutation row. submit/approve/ + /// cancel check the bounty's STATUS before their 403 auth gate, so the stranger + /// hostile only exercises the gate against a bounty in the gate-reaching + /// status, and each row's authorized twin CONSUMES that status; the rows + /// cannot share a bounty. This one stays "claimed" (create + claim) for the + /// submit_bounty row (IdSource::ClaimedBountyId). + pub claimed_bounty_id: String, + /// Driven to "submitted" (create + claim + submit) for the approve_bounty row + /// (IdSource::SubmittedBountyId). + pub submitted_bounty_id: String, + /// Left "open" (create only) for the cancel_bounty row (IdSource::OpenBountyId). + pub open_bounty_id: String, + /// #195 (F3): a task seeded "pending" (delegated by the owner) for the + /// claim_task caller-self row (IdSource::ClaimableTaskId). + pub claimable_task_id: String, + /// A task seeded "claimed" by the assignee for the complete_task actor row + /// (IdSource::CompletableTaskId). + pub completable_task_id: String, + /// A second assignee-claimed task for the fail_task actor row + /// (IdSource::FailableTaskId). + pub failable_task_id: String, + /// The id (UUID) of the seeded issue (authored by `author`), filled into the + /// `close_issue` row's `{id}` placeholder (IdSource::IssueId). + pub issue_id: String, + /// The id (UUID) of an issue seeded in the PRIVATE repo (marker in its title), + /// filled into the get_issue / list_issue_comments read-gate rows + /// (IdSource::PrivIssueId). Distinct from `issue_id` (the public close issue). + pub priv_issue_id: String, + /// The id (UUID) of a bounty seeded against the PRIVATE repo (marker in its + /// title), filled into the get_bounty read-gate row (IdSource::PrivBountyId). + pub priv_bounty_id: String, + /// The id (UUID) of a ref-certificate issued by a real owner push to the + /// private repo, filled into the get_cert read-gate row (IdSource::CertId). + pub cert_id: String, + /// Per-read secret markers seeded into the private sub-entities (issue title, + /// PR title, PR diff, bounty title). Each read's own marker is added to + /// that read's withheld set so a 404 leaking THAT read's private content fails + /// (#195, R3/KTD-4). A union is used so every seeded marker is withheld from + /// every read-gate hostile — a per-read withheld token, never a status-only + /// 404 check. + pub priv_markers: Vec, + /// Public repo: owner-gate rows run against this so the stranger reaches the + /// owner gate rather than a hidden-repo 404. + pub public_repo: String, + /// The public repo's id, needed to seed the PR the close-gate rows require. + pub public_repo_id: String, + /// Private repo (is_public=false) with seeded content: read-gate rows run + /// against this so an anon caller gets the existence-hiding 404 and the owner + /// twin gets 2xx. + pub private_repo: String, + /// The private repo's internal id (UUID). A read-gate denial body must never + /// echo it: a 404 that serializes the private record's internal id has leaked + /// repository metadata just as much as the blob content (#195, F4). + pub private_repo_id: String, + /// A seeded blob path present in both repos, for `{*path}` reads. + pub content_path: String, + /// The DISTINCTIVE secret bytes seeded into the private repo's blob. A + /// read-gate denial body must never echo this (INV-8: the sweep claims the + /// denial "leaks nothing", so it must actually check for the private content). + pub private_secret: String, + /// The private blob's full sha1 object id, and its short prefix. Neither may + /// appear in a read-gate denial body — a 404 that names the withheld object's + /// OID has leaked its existence. + pub private_blob_oid: String, + pub private_blob_oid_short: String, +} + +impl Fixture { + pub async fn seed(node: &TestNode) -> Fixture { + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let stranger = Keypair::generate(); + // #195 (F1): distinct identities for each authorizing arm of the + // multi-principal gates. All four must differ from owner and stranger and + // from each other, or an arm silently collapses onto another and reverting + // it stays invisible (the original author == owner bug). + let author = Keypair::generate(); + let author_did = author.did().to_string(); + let creator = Keypair::generate(); + let creator_did = creator.did().to_string(); + let claimant = Keypair::generate(); + let claimant_did = claimant.did().to_string(); + // #195 (F3): a distinct assignee identity (!= owner, != stranger) for the + // complete/fail actor gates, so their Assignee arm cannot false-pass on an + // owner fallback. + let assignee = Keypair::generate(); + let assignee_did = assignee.did().to_string(); + let content_path = "public/a.txt".to_string(); + + let pub_repo = "prober-pub"; + let public_repo_id = node.seed_repo(&owner_did, pub_repo, true).await; + node.seed_bare_repo( + &owner_did, + pub_repo, + &[("public/a.txt", "pub content")], + "sha1", + ); + // The author-or-owner close gates load the PR/issue before they run, so + // the close rows need the entity to exist. Seed both authored by `author` + // (NOT the owner) so the author arm is granted by a non-owner identity and + // reverting either arm is observable. The PR seeds via the DB (author_did + // is a real column). The issue is seeded over HTTP as `author` (below): + // close_issue deserializes the WHOLE git-JSON IssueRecord before reading + // its author, so a bare `{"author":..}` DB-seed fails to deserialize and + // the author falls back to None — the author arm would then 403. + node.seed_pr(&public_repo_id, 1, &author_did).await; + let issue_id = seed_authored_issue(node, &owner_did, pub_repo, &author).await; + + // Seed a DISPUTABLE bounty (status "claimed", creator + claimant set) so the + // dispute_bounty creator/claimant arms reach their auth check. dispute_bounty + // gates on creator/claimant, NOT the repo, so the bounty is created on the + // public repo (both signers can read it). Seeded over HTTP through the real + // handlers — create (creator) then claim (claimant) — because the DB seam is + // private to TestNode and a test-only seed method would be a src/ change. + let bounty_id = seed_bounty_at_stage( + node, + &owner_did, + pub_repo, + &creator, + &claimant, + "prober dispute bounty", + BountyStage::Claimed, + ) + .await; + + // #195 (N2): submit/approve/cancel check the bounty's STATUS before their + // 403 auth gate, so each row gets its OWN bounty in the gate-reaching + // status (claimed / submitted / open); the authorized twins consume that + // status, so the rows cannot share a bounty with each other or with the + // disputable one above. + let claimed_bounty_id = seed_bounty_at_stage( + node, + &owner_did, + pub_repo, + &creator, + &claimant, + "prober submit bounty", + BountyStage::Claimed, + ) + .await; + let submitted_bounty_id = seed_bounty_at_stage( + node, + &owner_did, + pub_repo, + &creator, + &claimant, + "prober approve bounty", + BountyStage::Submitted, + ) + .await; + let open_bounty_id = seed_bounty_at_stage( + node, + &owner_did, + pub_repo, + &creator, + &claimant, + "prober cancel bounty", + BountyStage::Open, + ) + .await; + + let priv_repo = "prober-priv"; + // Seed the private blob with an UNMISTAKABLE marker (not "priv content"), + // so a read-gate denial body that echoes the content is caught as a leak + // rather than blending into generic prose. The OID is captured so the OID + // (full + short) is withheld too — a 404 naming the object's id has leaked + // its existence just as much as its bytes. + let private_secret = "TOPSECRET-PRIVREAD-U3".to_string(); + let private_repo_id = node.seed_repo(&owner_did, priv_repo, false).await; + let priv_oids = node.seed_bare_repo( + &owner_did, + priv_repo, + &[("public/a.txt", &private_secret)], + "sha1", + ); + let private_blob_oid = priv_oids["public/a.txt"].clone(); + let private_blob_oid_short = private_blob_oid[..12].to_string(); + // The private repo's current `main` commit tip, needed as the `old` oid of + // the ref-update that force-points main at the deterministic push below. + let private_main_tip = priv_oids["HEAD"].clone(); + + // #195 (F2/U3): seed the private sub-entities the deferred reads return, each + // carrying its OWN distinctive marker so the per-read leak assertion is + // load-bearing. All are seeded as the OWNER (the only reader of the private + // repo), so anon/stranger get the existence-hiding 404 and the owner twin + // gets a 2xx that actually returns the entity. + // + // A real owner push to the private repo advances a `feature` branch with a + // marker file (so get_pr_diff's branch_diff_names(main, feature) is + // NON-EMPTY) AND issues a ref-certificate (get_cert). Do it first so the PR's + // source branch exists before create_pr and the cert id is captured. + let pr_diff_marker = "TOPSECRET-PRDIFF-U3".to_string(); + let cert_id = seed_private_push_and_cert( + node, + &owner, + &owner_did, + priv_repo, + &private_main_tip, + &private_secret, + &private_blob_oid, + &pr_diff_marker, + ) + .await; + + let issue_marker = "TOPSECRET-ISSUE-U3".to_string(); + let priv_issue_id = + seed_private_issue(node, &owner, &owner_did, priv_repo, &issue_marker).await; + + let pr_marker = "TOPSECRET-PRTITLE-U3".to_string(); + seed_private_pr(node, &owner, &owner_did, priv_repo, &pr_marker).await; + + let bounty_marker = "TOPSECRET-BOUNTY-U3".to_string(); + let priv_bounty_id = + seed_private_bounty(node, &owner, &owner_did, priv_repo, &bounty_marker).await; + + let priv_markers = vec![issue_marker, pr_marker, pr_diff_marker, bounty_marker]; + + // #195 (F3): seed the caller-self task entities through the real task write + // routes (behind add_auth_layers only, no icaptcha). The claimable task + // stays "pending" so its claim control (the owner) can claim it; the + // completable and failable tasks are claimed by the assignee, so + // complete_task / fail_task reach their assignee actor gate against a + // "claimed" task. + let claimable_task_id = seed_pending_task(node, &owner, None).await; + // completable/failable tasks sit on the PUBLIC repo: post-#327 + // get_visible_task hides a repo-less task from any caller who is not the + // delegator/assignee, so a stranger hostile would 404 before the assignee + // gate. A public-repo task keeps the stranger visible so the hostile + // reaches the assignee 403 the rows drive. + let completable_task_id = + seed_claimed_task(node, &owner, &assignee, Some(&public_repo_id)).await; + let failable_task_id = + seed_claimed_task(node, &owner, &assignee, Some(&public_repo_id)).await; + + Fixture { + owner, + stranger, + author, + creator, + claimant, + assignee, + owner_did, + author_did, + creator_did, + claimant_did, + assignee_did, + bounty_id, + claimed_bounty_id, + submitted_bounty_id, + open_bounty_id, + claimable_task_id, + completable_task_id, + failable_task_id, + issue_id, + priv_issue_id, + priv_bounty_id, + cert_id, + priv_markers, + public_repo: pub_repo.to_string(), + public_repo_id, + private_repo: priv_repo.to_string(), + private_repo_id, + content_path, + private_secret, + private_blob_oid, + private_blob_oid_short, + } + } +} + +/// Seed an issue authored by `author` through the real `create_issue` handler +/// and return its minted UUID id. Written as a full git-JSON `IssueRecord` (the +/// handler serializes the whole record), so `close_issue`'s deserialize-then-read +/// author path finds the author — which a bare-field DB seed cannot satisfy. +async fn seed_authored_issue( + node: &TestNode, + owner_did: &str, + repo: &str, + author: &Keypair, +) -> String { + use super::signing::signed_request; + + let client = super::bounded_client(); + let path = format!("/api/v1/repos/{owner_did}/{repo}/issues"); + let body = br#"{"title":"prober close issue"}"#.to_vec(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body, + author, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create issue sends"); + assert_eq!( + resp.status().as_u16(), + 201, + "seeding: issue create must return 201 (author can read the public repo)" + ); + let created: serde_json::Value = resp.json().await.expect("issue create returns JSON"); + created["id"] + .as_str() + .expect("created issue carries an id") + .to_string() +} + +/// Create a "pending" task delegated by `delegator` through the real `create_task` +/// handler and return its minted id. `create_task` binds `delegator_did` to the +/// signer, so the body names the signer's DID. Task write routes sit behind +/// `add_auth_layers` only (no icaptcha), so a signed create reaches the handler. +async fn seed_pending_task(node: &TestNode, delegator: &Keypair, repo_id: Option<&str>) -> String { + use super::signing::signed_request; + + let client = super::bounded_client(); + let delegator_did = delegator.did().to_string(); + let repo_field = repo_id + .map(|id| format!(r#","repo_id":"{id}""#)) + .unwrap_or_default(); + let body = format!( + r#"{{"kind":"review","capability":"read","delegator_did":"{delegator_did}"{repo_field}}}"# + ) + .into_bytes(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + "/api/v1/tasks", + body, + delegator, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create task sends"); + assert_eq!( + resp.status().as_u16(), + 201, + "seeding: task create must return 201 (delegator_did binds to the signer)" + ); + let created: serde_json::Value = resp.json().await.expect("task create returns JSON"); + created["id"] + .as_str() + .expect("created task carries an id") + .to_string() +} + +/// Create a "pending" task (delegated by `owner`), then claim it as `assignee` +/// (`claim_task` binds `assignee_did` to the signer), leaving it "claimed" so +/// `complete_task` / `fail_task` reach their assignee actor gate. Returns the id. +async fn seed_claimed_task( + node: &TestNode, + owner: &Keypair, + assignee: &Keypair, + repo_id: Option<&str>, +) -> String { + use super::signing::signed_request; + + let id = seed_pending_task(node, owner, repo_id).await; + + let client = super::bounded_client(); + let assignee_did = assignee.did().to_string(); + let claim_path = format!("/api/v1/tasks/{id}/claim"); + let body = format!(r#"{{"assignee_did":"{assignee_did}"}}"#).into_bytes(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &claim_path, + body, + assignee, + ) + .header("content-type", "application/json") + .send() + .await + .expect("claim task sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "seeding: task claim must return 200 (assignee_did binds to the signer)" + ); + id +} + +/// How far along its lifecycle a seeded bounty is driven, expressed as the real +/// API transitions that get it there. Each status-gated bounty row needs the +/// status its handler's pre-auth check demands (#195, N2): cancel needs "open", +/// submit (and dispute) need "claimed", approve needs "submitted". +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BountyStage { + /// `create_bounty` only: status "open". + Open, + /// create + claim: status "claimed" (creator + claimant set). + Claimed, + /// create + claim + submit: status "submitted". + Submitted, +} + +/// Seed a bounty on the (public) repo, drive it to `stage`, and return its id. +/// +/// Driven through the real HTTP handlers rather than the DB: `create_bounty` +/// (creator) mints a UUID id we read back from the response, `claim_bounty` +/// (claimant) sets the claimant, `submit_bounty` (claimant) records a PR id. +/// All read-gate the (public) repo, which the creator/claimant can read, and the +/// claim/submit transitions pass their own auth gates as the claimant. A Claimed +/// bounty's deadline has not been exceeded, so a creator/claimant dispute returns +/// 400 (not 403) after passing auth, which is still a Not403 twin, while a +/// stranger is denied 403 at the auth check first. +async fn seed_bounty_at_stage( + node: &TestNode, + owner_did: &str, + repo: &str, + creator: &Keypair, + claimant: &Keypair, + title: &str, + stage: BountyStage, +) -> String { + use super::signing::signed_request; + + let client = super::bounded_client(); + + // create_bounty (creator) -> 201 with the minted BountyRecord (carries the id). + let create_path = format!("/api/v1/repos/{owner_did}/{repo}/bounties"); + let body = format!(r#"{{"title":"{title}","amount":1}}"#).into_bytes(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &create_path, + body, + creator, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create bounty sends"); + assert_eq!( + resp.status().as_u16(), + 201, + "seeding: bounty create must return 201 (creator can read the public repo)" + ); + let created: serde_json::Value = resp.json().await.expect("bounty create returns JSON"); + let bounty_id = created["id"] + .as_str() + .expect("created bounty carries an id") + .to_string(); + let creator_did = creator.did().to_string(); + assert_eq!( + created["creator_did"].as_str(), + Some(creator_did.as_str()), + "seeded bounty creator must be the fixture creator" + ); + if stage == BountyStage::Open { + return bounty_id; + } + + // claim_bounty (claimant) -> 200, status becomes "claimed", claimant recorded. + let claim_path = format!("/api/v1/bounties/{bounty_id}/claim"); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &claim_path, + br#"{}"#.to_vec(), + claimant, + ) + .header("content-type", "application/json") + .send() + .await + .expect("claim bounty sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "seeding: bounty claim must return 200 (claimant can read the public repo)" + ); + if stage == BountyStage::Claimed { + return bounty_id; + } + + // submit_bounty (claimant) -> 200, status becomes "submitted". pr_id is a + // free-text column; the seed value only needs to clear serde. + let submit_path = format!("/api/v1/bounties/{bounty_id}/submit"); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &submit_path, + br#"{"pr_id":"1"}"#.to_vec(), + claimant, + ) + .header("content-type", "application/json") + .send() + .await + .expect("submit bounty sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "seeding: bounty submit must return 200 (the claimant passes the submit gate)" + ); + + bounty_id +} + +/// Seed an issue in the PRIVATE repo (as the owner, the only reader) whose title +/// carries `marker`, and return its id. get_issue returns the whole IssueRecord, +/// so a 404 that echoes the title has leaked private content; the marker is added +/// to the withheld set. +async fn seed_private_issue( + node: &TestNode, + owner: &Keypair, + owner_did: &str, + repo: &str, + marker: &str, +) -> String { + use super::signing::signed_request; + + let client = super::bounded_client(); + let path = format!("/api/v1/repos/{owner_did}/{repo}/issues"); + let body = format!(r#"{{"title":"{marker}"}}"#).into_bytes(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body, + owner, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create private issue sends"); + assert_eq!( + resp.status().as_u16(), + 201, + "seeding: private issue create must return 201 (owner reads its own private repo)" + ); + let created: serde_json::Value = resp.json().await.expect("issue create returns JSON"); + created["id"] + .as_str() + .expect("created private issue carries an id") + .to_string() +} + +/// Seed a PR in the PRIVATE repo (as the owner) with `marker` in its title and +/// `feature` as its source branch (which the prior push created). get_pr returns +/// the whole PullRequest and get_pr_diff a real non-empty diff, so both the title +/// marker and the diff marker are withheld. Lands at PR number 1 (first PR). +async fn seed_private_pr( + node: &TestNode, + owner: &Keypair, + owner_did: &str, + repo: &str, + marker: &str, +) { + use super::signing::signed_request; + + let client = super::bounded_client(); + let path = format!("/api/v1/repos/{owner_did}/{repo}/pulls"); + let body = + format!(r#"{{"title":"{marker}","source_branch":"feature","target_branch":"main"}}"#) + .into_bytes(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body, + owner, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create private PR sends"); + assert_eq!( + resp.status().as_u16(), + 201, + "seeding: private PR create must return 201 (owner reads its own private repo)" + ); + let created: serde_json::Value = resp.json().await.expect("PR create returns JSON"); + assert_eq!( + created["number"].as_i64(), + Some(1), + "seeded private PR must be number 1 (the read-gate rows fill {{number}} = 1)" + ); +} + +/// Seed a bounty against the PRIVATE repo (as the owner) with `marker` in its +/// title, and return its id. get_bounty read-gates the bounty's own repo, so +/// anon/stranger get a 404 while the owner gets the bounty JSON (title marker +/// withheld from the 404). +async fn seed_private_bounty( + node: &TestNode, + owner: &Keypair, + owner_did: &str, + repo: &str, + marker: &str, +) -> String { + use super::signing::signed_request; + + let client = super::bounded_client(); + let path = format!("/api/v1/repos/{owner_did}/{repo}/bounties"); + let body = format!(r#"{{"title":"{marker}","amount":1}}"#).into_bytes(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body, + owner, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create private bounty sends"); + assert_eq!( + resp.status().as_u16(), + 201, + "seeding: private bounty create must return 201 (owner reads its own private repo)" + ); + let created: serde_json::Value = resp.json().await.expect("bounty create returns JSON"); + created["id"] + .as_str() + .expect("created private bounty carries an id") + .to_string() +} + +/// Point the PRIVATE repo's `main` at a deterministic commit and add a `feature` +/// branch (a marker file on top of it) via a real owner git-receive-pack push, +/// which also makes the node issue a ref-certificate. Returns the issued cert's +/// id, read back from the owner's list_certs. +/// +/// seed_bare_repo's `main` tip is non-deterministic (its commit dates are ambient), +/// so we cannot build a `feature` that descends from it without the parent objects. +/// Instead we push a full (non-thin) history: a fresh `main` commit and `feature` +/// on top, force-updating the served `main` (the ref-update carries the server's +/// current tip as its `old` oid; receive.denyNonFastForwards is off by default, so +/// the non-fast-forward update applies). After the push, `feature` descends from +/// the new `main`, so get_pr_diff's `git diff main...feature` has a merge base and +/// returns the marker file. The push is signed as the owner (require_signature) and, +/// on a non-protected branch of an owner-owned repo, lands cleanly and issues a cert. +#[allow(clippy::too_many_arguments)] +async fn seed_private_push_and_cert( + node: &TestNode, + owner: &Keypair, + owner_did: &str, + repo: &str, + server_main_tip: &str, + blob_contents: &str, + expected_blob_oid: &str, + marker_file_contents: &str, +) -> String { + use super::signing::signed_request; + use std::process::Command; + + // A unique per-call directory: parallel fixtures seeding within the same + // second produce the same server_main_tip, so a pid+tip path collides and + // one fixture's cleanup can delete another's in-progress tree. TempDir + // also removes the directory on the panic path. + let work_dir = tempfile::Builder::new() + .prefix("gl-u3-certpush-") + .tempdir() + .expect("create cert-push workdir"); + let work = work_dir.path().to_path_buf(); + let run = |args: &[&str], cwd: &std::path::Path| -> String { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Pin sha1 to match the served fixture repo; init.defaultObjectFormat=sha256 skews the OIDs. + run(&["init", "-q", "-b", "main", "--object-format=sha1"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + // Keep `public/a.txt` (the seeded blob) on the new main so the get_blob / + // get_tree owner twins still resolve it — the blob OID depends only on content, + // so preserving the bytes preserves `private_blob_oid`. + std::fs::create_dir_all(work.join("public")).expect("mk public dir"); + std::fs::write(work.join("public/a.txt"), blob_contents).expect("seed blob file"); + run(&["add", "public/a.txt"], &work); + run(&["commit", "-q", "-m", "u3 main"], &work); + let new_main = run(&["rev-parse", "HEAD"], &work); + let new_blob = run(&["rev-parse", "HEAD:public/a.txt"], &work); + assert_eq!( + new_blob, expected_blob_oid, + "the pushed public/a.txt blob OID must equal the fixture's private_blob_oid" + ); + + // feature = new main + a marker file. + run(&["checkout", "-q", "-b", "feature"], &work); + std::fs::write(work.join("feature.txt"), marker_file_contents).expect("seed marker file"); + run(&["add", "feature.txt"], &work); + run(&["commit", "-q", "-m", "feature commit"], &work); + let feature_tip = run(&["rev-parse", "HEAD"], &work); + + // Full (non-thin) pack of both branches' objects — the server keeps its old + // main objects but needs all of the new ones since feature does not descend + // from the server's current main. + let pack = { + let out = Command::new("git") + .args(["pack-objects", "--stdout", "--revs"]) + .current_dir(&work) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("spawn pack-objects"); + use std::io::Write; + out.stdin + .as_ref() + .expect("pack-objects stdin") + .write_all(format!("{new_main}\n{feature_tip}\n").as_bytes()) + .expect("write revs"); + let done = out.wait_with_output().expect("pack-objects completes"); + assert!(done.status.success(), "pack-objects must succeed"); + done.stdout + }; + + // git v0 receive-pack body: two ref-update pkt-lines — force-update main + // (old = the server's current tip) and create feature — the first carrying the + // capabilities after a NUL, a flush, then the packfile. + let zero = "0".repeat(server_main_tip.len()); + let l1 = format!("{server_main_tip} {new_main} refs/heads/main\0report-status\n"); + let l2 = format!("{zero} {feature_tip} refs/heads/feature\n"); + let mut body: Vec = Vec::new(); + body.extend(format!("{:04x}{l1}", l1.len() + 4).into_bytes()); + body.extend(format!("{:04x}{l2}", l2.len() + 4).into_bytes()); + body.extend_from_slice(b"0000"); + body.extend_from_slice(&pack); + + let client = super::bounded_client(); + let push_path = format!("/{owner_did}/{repo}/git-receive-pack"); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &push_path, + body, + owner, + ) + .header("content-type", "application/x-git-receive-pack-request") + .send() + .await + .expect("receive-pack push sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "seeding: owner push to a non-protected feature branch must return 200 so a cert issues" + ); + + // work_dir drops here and removes the tree, including on the panic path. + + // Read the issued cert id back from the owner's list_certs on the private repo. + let certs_path = format!("/api/v1/repos/{owner_did}/{repo}/certs"); + let resp = signed_request( + &client, + reqwest::Method::GET, + &node.base_url, + &certs_path, + Vec::new(), + owner, + ) + .send() + .await + .expect("list_certs sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "seeding: owner list_certs must return 200" + ); + let listed: serde_json::Value = resp.json().await.expect("list_certs returns JSON"); + listed["certificates"][0]["id"] + .as_str() + .expect("a ref-certificate must have been issued by the push") + .to_string() +} + +/// Who signs a probe request. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Signer { + Anon, + Owner, + Stranger, + /// The PR/issue author arm of a multi-principal close gate. + Author, + /// The bounty creator arm of dispute_bounty. + Creator, + /// The bounty claimant arm of dispute_bounty. + Claimant, + /// The task assignee arm of complete_task / fail_task (#195, F3). + Assignee, +} + +/// The signer identity that grants a given multi-principal arm. The runtime sweep +/// resolves each to its fixture keypair; the structural consistency test uses it +/// to confirm every declared arm has a twin signed by ITS identity. +pub fn signer_for_principal(p: Principal) -> Signer { + match p { + Principal::Owner => Signer::Owner, + Principal::Author => Signer::Author, + Principal::Creator => Signer::Creator, + Principal::Claimant => Signer::Claimant, + Principal::Assignee => Signer::Assignee, + } +} + +/// What a probe expects. +#[derive(Debug, Clone)] +pub enum Expect { + /// Hostile request: the exact deny status, body leaking none of `withheld`. + Deny(u16), + /// Owner-gate twin: the owner reaches the handler, so NOT 403. + Not403, + /// Read-gate twin: an authorized read returns a non-empty 2xx (an empty 2xx + /// would be a denial rendered as success). + Ok2xx, +} + +/// A single request to drive against the node. +#[derive(Debug, Clone)] +pub struct Probe { + pub label: String, + pub method: reqwest::Method, + /// Path used both for the URL and for RFC-9421 `@path` signing. + pub path: String, + pub body: Vec, + pub signer: Signer, + pub json: bool, + pub expect: Expect, + /// Private tokens (secret content, blob OID) that the DENY body must not echo. + /// Non-empty only for read-gate hostile probes, where a private repo/blob is + /// actually seeded; the owner-gate (403) and signature (401) hostile probes + /// fire on NO_ENTITY repos before any entity lookup, so there is genuinely + /// nothing seeded to leak and an empty list is the honest assertion there. + pub withheld: Vec, +} + +/// Substitute path-template placeholders from the fixture. `repo` is the public +/// repo for owner-gate rows and the private repo for read-gate rows. +/// +/// `id_source` selects what `{id}` resolves to: `Fixed` uses the static seed id +/// `"1"` (entities seeded at #1), while a per-row source such as `BountyId` +/// injects a UUID minted at seed time. This is the general id-threading hook U3 +/// reuses for the other id-keyed reads (a cert id, etc.): add an `IdSource` +/// variant + fixture field and map it here. +fn fill(path: &str, fixture: &Fixture, repo: &str, id_source: IdSource) -> String { + let id = match id_source { + IdSource::Fixed => "1", + IdSource::BountyId => fixture.bounty_id.as_str(), + IdSource::IssueId => fixture.issue_id.as_str(), + IdSource::PrivIssueId => fixture.priv_issue_id.as_str(), + IdSource::PrivBountyId => fixture.priv_bounty_id.as_str(), + IdSource::CertId => fixture.cert_id.as_str(), + // #195 (N2): the status-gated bounty mutations, each keyed to the bounty + // seeded in its gate-reaching status. + IdSource::ClaimedBountyId => fixture.claimed_bounty_id.as_str(), + IdSource::SubmittedBountyId => fixture.submitted_bounty_id.as_str(), + IdSource::OpenBountyId => fixture.open_bounty_id.as_str(), + // #195 (F3): the caller-self task rows, each keyed to a seeded task in the + // state its gate needs (pending for claim, claimed for complete/fail). + IdSource::ClaimableTaskId => fixture.claimable_task_id.as_str(), + IdSource::CompletableTaskId => fixture.completable_task_id.as_str(), + IdSource::FailableTaskId => fixture.failable_task_id.as_str(), + }; + path.replace("{owner}", &fixture.owner_did) + .replace("{repo}", repo) + .replace("{number}", "1") + .replace("{id}", id) + .replace("{label}", "bug") + .replace("{branch}", "main") + .replace("{*path}", &fixture.content_path) + .replace("{path}", &fixture.content_path) +} + +/// Expand one deny-bearing row into its hostile probe plus positive twin. +pub fn probes_for(row: &Row, fixture: &Fixture) -> Vec { + let method: reqwest::Method = row.method.parse().expect("valid method"); + let body = row.body.map(|b| b.as_bytes().to_vec()).unwrap_or_default(); + let json = row.body.is_some(); + + match row.gate { + GateClass::OwnerGate => { + let path = fill(row.path, fixture, &fixture.public_repo, row.id_source); + vec![ + Probe { + label: format!("{} owner-gate hostile", row.handler), + method: method.clone(), + path: path.clone(), + body: body.clone(), + signer: Signer::Stranger, + json, + expect: Expect::Deny(403), + // No entity is seeded on the public owner-gate substrate before + // the 403 fires, so there is nothing private to leak here. + withheld: Vec::new(), + }, + Probe { + label: format!("{} owner-reachability twin", row.handler), + method, + path, + body, + signer: Signer::Owner, + json, + expect: Expect::Not403, + withheld: Vec::new(), + }, + ] + } + GateClass::MultiPrincipalGate => { + // #195 (F1): a single stranger-403 hostile plus one Not403 twin PER + // declared arm. Each arm is signed by ITS distinct fixture identity, so + // reverting any single arm in the handler turns that arm's twin RED + // while the others (and the stranger 403) stay green. + let path = fill(row.path, fixture, &fixture.public_repo, row.id_source); + let mut v = vec![Probe { + label: format!("{} multi-principal hostile (stranger)", row.handler), + method: method.clone(), + path: path.clone(), + body: body.clone(), + signer: Signer::Stranger, + json, + expect: Expect::Deny(403), + // The gate fires before returning any entity; nothing private is + // seeded to leak on this substrate. + withheld: Vec::new(), + }]; + for arm in row.principals { + let signer = signer_for_principal(*arm); + v.push(Probe { + label: format!("{} arm twin ({:?})", row.handler, arm), + method: method.clone(), + path: path.clone(), + body: body.clone(), + signer, + json, + expect: Expect::Not403, + withheld: Vec::new(), + }); + } + v + } + GateClass::ReadGate => { + let path = fill(row.path, fixture, &fixture.private_repo, row.id_source); + // INV-8: the private repo's blob and its OID are actually seeded here, so + // the 404 body must echo neither the secret content nor the blob OID + // (full or short) nor the private repo's internal id (#195, F4). An empty + // withheld list would let a 404 that spilled the private content pass as + // a clean denial. + // #195 (R3/KTD-4): the fixed blob-scoped withheld set PLUS every seeded + // sub-entity marker. The new reads (get_issue/get_pr/get_pr_diff/get_cert + // /get_bounty and the list rows off them) return their OWN private + // content, whose markers are NOT in the blob set — a 404 that leaks an + // issue title or PR diff must fail, so the per-read markers are withheld + // from every read-gate hostile. A status-only 404 check would be vacuous. + let mut withheld = vec![ + fixture.private_secret.clone(), + fixture.private_blob_oid.clone(), + fixture.private_blob_oid_short.clone(), + fixture.private_repo_id.clone(), + ]; + withheld.extend(fixture.priv_markers.iter().cloned()); + // The anon leg: an `Option`-auth read existence-hides with a 404, but an + // auth-required read (`AnonRead::Deny401`, e.g. list_webhooks) rejects a + // headerless caller 401 BEFORE any lookup. The signed-non-reader 404 + // below is the existence-hiding gate either way; only this leg differs. + let anon_status = match row.anon_read { + AnonRead::Deny404 => 404, + AnonRead::Deny401 => 401, + }; + let mut v = vec![ + Probe { + label: format!("{} read-gate hostile (anon -> {anon_status})", row.handler), + method: method.clone(), + path: path.clone(), + body: body.clone(), + signer: Signer::Anon, + json, + expect: Expect::Deny(anon_status), + withheld: withheld.clone(), + }, + // #195 (F1): a signed NON-READER must also get the existence-hiding + // 404 — a signature is not access (INV-12). Without this probe, a + // regression that treats any valid signature as authorized would leak + // the private repo while the anon probe stayed green. + Probe { + label: format!("{} read-gate hostile (signed non-reader)", row.handler), + method: method.clone(), + path: path.clone(), + body: body.clone(), + signer: Signer::Stranger, + json, + expect: Expect::Deny(404), + withheld: withheld.clone(), + }, + ]; + // Positive twin: prove the 404 is the gate, not an absent entity. + let twin = match row.reach { + Reach::ReaderReads => Probe { + label: format!("{} read-reachability twin (owner)", row.handler), + method, + path, + body, + signer: Signer::Owner, + json, + expect: Expect::Ok2xx, + // The twin is a GRANTED read; it is expected to return the + // content, so nothing is withheld from it. + withheld: Vec::new(), + }, + Reach::SiblingPublic(sibling) => Probe { + label: format!("{} read-reachability twin (sibling public)", row.handler), + method, + path: fill(sibling, fixture, &fixture.private_repo, row.id_source), + body, + signer: Signer::Anon, + json, + expect: Expect::Ok2xx, + withheld: Vec::new(), + }, + Reach::None => panic!( + "read-gate row {} {} has no Reach twin (U1 consistency should have caught this)", + row.method, row.path + ), + }; + v.push(twin); + v + } + GateClass::SignatureRequired => { + let path = fill(row.path, fixture, &fixture.public_repo, row.id_source); + vec![Probe { + label: format!("{} signature-required hostile", row.handler), + method, + path, + body, + signer: Signer::Anon, + json, + expect: Expect::Deny(401), + // The 401 fires at the signature layer before any repo/entity is + // looked up (NO_ENTITY substrate), so nothing private is in scope. + withheld: Vec::new(), + }] + } + GateClass::SignerSelfGate => { + // #195 (F3, KTD-1): a body-field caller-self gate. The shared body runs + // through `fill()` so `{owner}` resolves to the real owner DID; hostile + // and control differ ONLY in signer. Hostile: a Stranger signs but the + // body names the owner -> `did_matches(stranger, owner) == false` -> 403. + // Control: the Owner signs and the body names the owner -> match -> + // Not403. (The repo arg to `fill` is inert here: task/register paths + // carry no `{repo}` placeholder, but the signature requires it.) + let path = fill(row.path, fixture, &fixture.public_repo, row.id_source); + let filled_body = row + .body + .map(|b| fill(b, fixture, &fixture.public_repo, row.id_source).into_bytes()) + .unwrap_or_default(); + vec![ + Probe { + label: format!("{} signer-self hostile (stranger)", row.handler), + method: method.clone(), + path: path.clone(), + body: filled_body.clone(), + signer: Signer::Stranger, + json, + expect: Expect::Deny(403), + // Each gate returns a fixed forbidden() message before serializing + // any entity, and nothing private is seeded on this substrate, so + // an empty withheld list is the honest assertion (KTD-4). + withheld: Vec::new(), + }, + Probe { + label: format!("{} signer-self control (owner)", row.handler), + method, + path, + body: filled_body, + signer: Signer::Owner, + json, + expect: Expect::Not403, + withheld: Vec::new(), + }, + ] + } + } +} + +/// A DB-less `Fixture` for the pure-generator unit tests (here and the structural +/// consistency test in `routes.rs`). No node is seeded, so the identities are +/// generated keypairs with placeholder repo/secret metadata; only the arm DIDs +/// and the injected `{id}` matter to `probes_for`. Shared so both test modules +/// build the same shape. +#[cfg(test)] +pub mod tests_support { + use super::*; + + pub fn fx() -> Fixture { + let author = Keypair::generate(); + let creator = Keypair::generate(); + let claimant = Keypair::generate(); + let assignee = Keypair::generate(); + Fixture { + author_did: author.did().to_string(), + creator_did: creator.did().to_string(), + claimant_did: claimant.did().to_string(), + assignee_did: assignee.did().to_string(), + author, + creator, + claimant, + assignee, + bounty_id: "bounty-uuid-1234".to_string(), + claimed_bounty_id: "claimed-bounty-uuid-1234".to_string(), + submitted_bounty_id: "submitted-bounty-uuid-1234".to_string(), + open_bounty_id: "open-bounty-uuid-1234".to_string(), + claimable_task_id: "claimable-task-uuid-1234".to_string(), + completable_task_id: "completable-task-uuid-1234".to_string(), + failable_task_id: "failable-task-uuid-1234".to_string(), + issue_id: "issue-uuid-1234".to_string(), + priv_issue_id: "priv-issue-uuid-1234".to_string(), + priv_bounty_id: "priv-bounty-uuid-1234".to_string(), + cert_id: "cert-uuid-1234".to_string(), + priv_markers: vec![ + "TOPSECRET-ISSUE-U3".to_string(), + "TOPSECRET-PRTITLE-U3".to_string(), + "TOPSECRET-PRDIFF-U3".to_string(), + "TOPSECRET-BOUNTY-U3".to_string(), + ], + owner: Keypair::generate(), + stranger: Keypair::generate(), + owner_did: "did:key:zOWNER".to_string(), + public_repo: "prober-pub".to_string(), + public_repo_id: "pub-id".to_string(), + private_repo: "prober-priv".to_string(), + private_repo_id: "priv-repo-uuid-0BADF00D".to_string(), + content_path: "public/a.txt".to_string(), + private_secret: "TOPSECRET-PRIVREAD-U3".to_string(), + private_blob_oid: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(), + private_blob_oid_short: "deadbeefdead".to_string(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::support::routes::{GateClass, Reach, Row}; + + use super::tests_support::fx; + + #[test] + fn owner_gate_row_yields_stranger_403_and_owner_twin() { + let row = Row { + method: "PUT", + path: "/api/v1/repos/{owner}/{repo}/visibility", + gate: GateClass::OwnerGate, + handler: "visibility::set_visibility", + body: Some(r#"{"path_glob":"/"}"#), + needs: &[], + reach: Reach::None, + principals: &[], + id_source: crate::support::routes::IdSource::Fixed, + anon_read: crate::support::routes::AnonRead::Deny404, + }; + let ps = probes_for(&row, &fx()); + assert_eq!(ps.len(), 2); + assert_eq!(ps[0].signer, Signer::Stranger); + assert!(ps[0].json); + assert!(matches!(ps[0].expect, Expect::Deny(403))); + assert!( + ps[0].path.contains("prober-pub"), + "owner-gate uses the public repo" + ); + assert_eq!(ps[1].signer, Signer::Owner); + assert!(matches!(ps[1].expect, Expect::Not403)); + // Owner-gate hostile fires on a NO_ENTITY public repo before any lookup, so + // there is nothing private seeded to leak: an empty withheld list is honest. + assert!( + ps[0].withheld.is_empty(), + "owner-gate hostile probe seeds no private entity, so withholds nothing" + ); + } + + #[test] + fn read_gate_row_yields_anon_and_stranger_404_and_positive_twin() { + let row = Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/blob/{*path}", + gate: GateClass::ReadGate, + handler: "repos::get_blob", + body: None, + needs: &[], + reach: Reach::ReaderReads, + principals: &[], + id_source: crate::support::routes::IdSource::Fixed, + anon_read: crate::support::routes::AnonRead::Deny404, + }; + let f = fx(); + let ps = probes_for(&row, &f); + // anon hostile, signed-stranger hostile (#195 F1), owner twin. + assert_eq!(ps.len(), 3); + assert_eq!(ps[0].signer, Signer::Anon); + assert!(matches!(ps[0].expect, Expect::Deny(404))); + // #195 (F1): the signed non-reader hostile must also expect a 404 no-leak. + assert_eq!(ps[1].signer, Signer::Stranger); + assert!(matches!(ps[1].expect, Expect::Deny(404))); + assert!( + !ps[1].withheld.is_empty(), + "the signed-stranger hostile must carry the same withheld tokens" + ); + assert!( + ps[0].path.contains("prober-priv"), + "read-gate uses the private repo" + ); + assert!(ps[0].path.contains("public/a.txt"), "{{*path}} substituted"); + // F2 belt-and-suspenders: the read-gate hostile probe MUST carry the private + // tokens, so a future refactor that drops them (reverting F2 to a vacuous + // empty-withheld 404 check) fails loudly here. + assert!( + !ps[0].withheld.is_empty(), + "read-gate hostile probe must carry withheld private tokens" + ); + assert!( + ps[0].withheld.contains(&f.private_secret), + "the seeded secret content must be a withheld token" + ); + assert!( + ps[0].withheld.contains(&f.private_blob_oid), + "the private blob OID must be a withheld token" + ); + assert!( + ps[0].withheld.contains(&f.private_blob_oid_short), + "the short blob OID prefix must be a withheld token" + ); + assert_eq!(ps[2].signer, Signer::Owner); + assert!(matches!(ps[2].expect, Expect::Ok2xx)); + // The granted twin returns the content, so it withholds nothing. + assert!( + ps[2].withheld.is_empty(), + "the granted read twin must not carry withheld tokens" + ); + } + + #[test] + fn signature_row_yields_unsigned_401_no_twin() { + let row = Row { + method: "POST", + path: "/{owner}/{repo}/git-receive-pack", + gate: GateClass::SignatureRequired, + handler: "repos::git_receive_pack", + body: None, + needs: &[], + reach: Reach::None, + principals: &[], + id_source: crate::support::routes::IdSource::Fixed, + anon_read: crate::support::routes::AnonRead::Deny404, + }; + let ps = probes_for(&row, &fx()); + assert_eq!(ps.len(), 1); + assert_eq!(ps[0].signer, Signer::Anon); + assert!(matches!(ps[0].expect, Expect::Deny(401))); + // The 401 fires before any entity lookup, so nothing private is in scope. + assert!( + ps[0].withheld.is_empty(), + "signature-required hostile probe withholds nothing (fires pre-lookup)" + ); + } + + #[test] + fn read_gate_sibling_public_twin_reads_the_sibling_path_anon() { + let row = Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/blob/secret/x.txt", + gate: GateClass::ReadGate, + handler: "repos::get_blob", + body: None, + needs: &[], + reach: Reach::SiblingPublic("/api/v1/repos/{owner}/{repo}/blob/{*path}"), + principals: &[], + id_source: crate::support::routes::IdSource::Fixed, + anon_read: crate::support::routes::AnonRead::Deny404, + }; + let ps = probes_for(&row, &fx()); + // anon hostile, signed-stranger hostile (#195 F1), sibling-public twin. + assert_eq!(ps.len(), 3); + // Hostile: anon on the withheld path -> 404. + assert_eq!(ps[0].signer, Signer::Anon); + assert!(matches!(ps[0].expect, Expect::Deny(404))); + assert!(ps[0].path.ends_with("/blob/secret/x.txt")); + // Signed non-reader on the withheld path -> 404 too. + assert_eq!(ps[1].signer, Signer::Stranger); + assert!(matches!(ps[1].expect, Expect::Deny(404))); + assert!(ps[1].path.ends_with("/blob/secret/x.txt")); + // Twin: anon on the sibling PUBLIC path -> non-empty 2xx (proves the 404 + // is path-scoped withholding, not a blanket refusal). + assert_eq!(ps[2].signer, Signer::Anon); + assert!(matches!(ps[2].expect, Expect::Ok2xx)); + assert!( + ps[2].path.contains("public/a.txt"), + "sibling template's {{*path}} is filled from the fixture content path" + ); + } + + // F2 RED proof at the check_denied level (no DB): feed a synthetic denial body + // that DOES contain the private secret through the SAME withheld tokens the + // read-gate probe carries. If the tokens are truly threaded and non-empty, the + // deny check must reject it as a leak. This is the fail case a real node's + // clean 404 avoids; it proves the sweep would actually catch a leaking 404. + #[test] + fn read_gate_withheld_tokens_reject_a_leaking_denial_body() { + use crate::support::assert::check_denied; + + let row = Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}", + gate: GateClass::ReadGate, + handler: "repos::get_repo", + body: None, + needs: &[], + reach: Reach::ReaderReads, + principals: &[], + id_source: crate::support::routes::IdSource::Fixed, + anon_read: crate::support::routes::AnonRead::Deny404, + }; + let f = fx(); + let ps = probes_for(&row, &f); + let tokens: Vec<&str> = ps[0].withheld.iter().map(String::as_str).collect(); + assert!( + !tokens.is_empty(), + "precondition: read-gate probe carries tokens" + ); + + // A hostile 404 that spilled the secret content: must be rejected as a leak. + let leaking = format!( + r#"{{"error":"blob {} at {}"}}"#, + f.private_secret, f.private_blob_oid + ); + let r = check_denied(404, &leaking, 404, &tokens); + assert!( + r.is_err(), + "a 404 body echoing the secret must be flagged: {r:?}" + ); + assert!( + r.unwrap_err().contains(&f.private_secret), + "the failure must name the leaked secret token" + ); + + // #195 (F4): a 404 that serialized the private repo's internal id (UUID) is + // a repository-metadata leak and must be flagged too — proving the UUID is a + // load-bearing withheld token, not just the blob content. + let leaking_meta = format!(r#"{{"error":"repo {} not found"}}"#, f.private_repo_id); + let rm = check_denied(404, &leaking_meta, 404, &tokens); + assert!( + rm.is_err(), + "a 404 body echoing the private repo id must be flagged: {rm:?}" + ); + assert!( + rm.unwrap_err().contains(&f.private_repo_id), + "the failure must name the leaked private repo id token" + ); + + // A clean 404 (no private tokens) with the same tokens passes: the tokens do + // not spuriously trip on an honest denial. + let clean = check_denied(404, r#"{"error":"repository not found"}"#, 404, &tokens); + assert!( + clean.is_ok(), + "a clean 404 must pass the withheld check: {clean:?}" + ); + } + + #[test] + #[should_panic(expected = "has no Reach twin")] + fn read_gate_row_without_a_reach_twin_panics() { + // A read-gate row with Reach::None is a registry bug the U1 consistency + // test rejects; if one slips through, the probe generator must fail loud + // rather than silently emit a hostile probe with no positive twin. + let row = Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/issues", + gate: GateClass::ReadGate, + handler: "issues::list_issues", + body: None, + needs: &[], + reach: Reach::None, + principals: &[], + id_source: crate::support::routes::IdSource::Fixed, + anon_read: crate::support::routes::AnonRead::Deny404, + }; + let _ = probes_for(&row, &fx()); + } +} diff --git a/crates/gitlawb-node/tests/support/routes.rs b/crates/gitlawb-node/tests/support/routes.rs new file mode 100644 index 000000000..0c463f961 --- /dev/null +++ b/crates/gitlawb-node/tests/support/routes.rs @@ -0,0 +1,1318 @@ +//! U1: the deny-bearing route set for the invariant deny-prober. +//! +//! Every route that carries a runtime deny to assert lives here: owner-gate +//! (403), multi-principal (403), read-gate (404), signature-required (401), and +//! caller-self / actor (403): `SignerSelfGate` for the request-body field +//! bindings (create_task delegator, claim_task assignee, register did) and +//! `MultiPrincipalGate` with the assignee arm for the complete/fail actor gates. +//! The completeness cross-check in `deny_harness.rs` derives the caller-self set +//! from `src/api` and requires each `did_matches(caller, X)` handler to be a +//! driven row, so a new one cannot silently escape the sweep. +//! +//! Scope: this doc claims the REST/api surface only. The parallel GraphQL +//! mutation caller-self gates (`src/graphql/mutation.rs`) are out of this +//! harness and fenced separately (#219). +//! +//! Each row is classified by READING its handler, never inferred from the route +//! name, and records the handler fn name so a reviewer can spot-check. Guessing +//! is how a false test enters the sweep (the register_replica lesson: it looks +//! like an owner-gate, it is signer-self; and list_visibility is a 403 +//! owner-gate despite being a GET). +//! +//! Rows are declarative (gate class + entities + reachability), not prebuilt +//! requests, so Flavor B (the cross-surface differential prober) can reuse them +//! (R9). + +/// The gate class of a deny-bearing route and the exact status its deny emits. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum GateClass { + /// `require_repo_owner` / `require_owner` / inline `did_matches(caller, owner)` + /// -> `AppError::Forbidden` == 403. Probed with a validly-signed non-owner. + OwnerGate, + /// A 403 gate with ONE OR MORE NAMED authorizing principals (owner-OR-author, + /// creator-OR-claimant, claimant-only, …). Probed with a validly-signed + /// stranger (403) plus one Not403 twin per declared arm (`Row.principals`), + /// so reverting ANY single arm turns that arm's twin RED (#195, F1). + /// `OwnerGate` is the owner-arm special case; a single-arm gate whose + /// principal is NOT the repo owner (submit_bounty's claimant-only check, + /// approve/cancel's creator-only check) belongs here, because the point is + /// that the arms are DECLARED and each one gets its own twin (#195, N2). + MultiPrincipalGate, + /// `authorize_repo_read` / `visibility_check` -> `RepoNotFound` == 404 + /// (existence-hiding). Probed with a non-reader against a private/withheld + /// target. A 404 alone is ambiguous (a missing entity also 404s), so every + /// read-gate row carries a `Reach` positive twin. + ReadGate, + /// `require_signature` layer -> 401 on the git write routes. Probed unsigned. + SignatureRequired, + /// A caller-self 403 gate that binds a DID field in the request BODY to the + /// authenticated signer (`did_matches(caller, &body._did)`): create_task + /// (delegator_did), claim_task (assignee_did), register (did). Probed with a + /// validly-signed stranger whose body names the OWNER (mismatch -> 403) plus an + /// owner control whose body names the owner (match -> Not403). Distinct from + /// `OwnerGate`: the bound identity is a body field, not the stored repo owner, + /// and the completeness classifier keys on exactly that (#195, F3, KTD-1). The + /// actor-vs-stored caller-self gates (complete_task/fail_task) are the + /// `MultiPrincipalGate` assignee-arm shape instead, not this class. + SignerSelfGate, +} + +impl GateClass { + pub fn expected_status(self) -> u16 { + match self { + GateClass::OwnerGate | GateClass::MultiPrincipalGate | GateClass::SignerSelfGate => 403, + GateClass::ReadGate => 404, + GateClass::SignatureRequired => 401, + } + } +} + +/// An authorizing principal (arm) of a multi-principal 403 gate. Each maps to a +/// distinct fixture identity in `probe.rs` so reverting one arm cannot silently +/// re-collapse onto another (the original bug seeded `author == owner`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Principal { + /// The repo owner (`require_repo_owner` / `did_matches(caller, owner)`). + Owner, + /// The PR/issue author (`did_matches(caller, author_did)`). + Author, + /// The bounty creator (`did_matches(caller, creator_did)`). + Creator, + /// The bounty claimant (`did_matches(caller, claimant_did)`). + Claimant, + /// The task assignee (`did_matches(caller, stored assignee_did)`): the actor + /// arm of complete_task / fail_task (#195, F3). Unlike the field-binding + /// `SignerSelfGate`, the identity is the task's STORED assignee, so the arm is + /// driven against a seeded claimed task, not a body field. + Assignee, +} + +/// How a read-gate row's positive twin proves the 404 is the gate and not a +/// merely-absent entity/repo. Owner-gate and signature rows use `None` +/// (the owner-gate twin is the same request re-signed as the owner, handled by +/// the probe generator from the class, not a path). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Reach { + /// Not a read-gate row; no read-twin. + None, + /// The authorized reader/owner issues the same request and gets 2xx. + ReaderReads, + /// A sibling PUBLIC path on the same repo returns 2xx-with-content, proving + /// the 404 is path-scoped withholding, not a blanket/absent 404. Mirrors the + /// existing U7/U8/anon_ipfs cases in `tests/deny_harness.rs`. + SiblingPublic(&'static str), +} + +/// How a row's `{id}` placeholder is filled. Most id-keyed paths use a fixed +/// seed id (`"1"`), but some entities are keyed by a UUID minted at seed time +/// (a bounty, a cert), so the fixture must inject the captured id per row. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IdSource { + /// `{id}` -> the fixed seed id `"1"` (static-template entities seeded at #1). + Fixed, + /// `{id}` -> the fixture's seeded disputable bounty id (UUID). + BountyId, + /// `{id}` -> the fixture's seeded issue id (UUID). The issue is stored as a + /// full git-JSON `IssueRecord` (authored by the fixture author), which a + /// bare `{"author":..}` seed cannot satisfy — close_issue deserializes the + /// whole record before reading its author, so the author arm needs the + /// complete record. + IssueId, + /// `{id}` -> the fixture's seeded PRIVATE-repo issue id (UUID). Distinct from + /// `IssueId` (the public-repo close-gate issue): the get_issue / + /// list_issue_comments read-gate rows run against the PRIVATE repo, so their + /// `{id}` must be an issue that actually exists there (#195, F2/U3). + PrivIssueId, + /// `{id}` -> the fixture's seeded PRIVATE-repo bounty id (UUID). get_bounty is + /// NOT repo-scoped in its path (`/api/v1/bounties/{id}`) but read-gates on the + /// bounty's own repo; the bounty is seeded against the private repo so anon / + /// stranger get the existence-hiding 404 and the owner gets 2xx (#195, U3). + PrivBountyId, + /// `{id}` -> the fixture's seeded ref-certificate id (UUID), issued by a real + /// owner push to the private repo. get_cert read-gates the private repo on "/" + /// then fetches the cert by id, so the owner-2xx twin needs a real cert whose + /// repo_id is the private repo's (#195, U3). + CertId, + /// `{id}` -> the fixture's bounty seeded in status "claimed" (#195, N2). + /// submit_bounty checks status BEFORE auth (bounties.rs:294), so the stranger + /// only reaches the claimant 403 gate on a bounty that is already "claimed"; + /// the claimant twin then consumes that status (claimed -> submitted), so the + /// row needs its own bounty. + ClaimedBountyId, + /// `{id}` -> the fixture's bounty driven to status "submitted" (#195, N2). + /// approve_bounty's status check fires before its creator gate + /// (bounties.rs:334), so only a "submitted" bounty reaches the 403; the + /// creator twin consumes it (submitted -> completed). + SubmittedBountyId, + /// `{id}` -> the fixture's bounty left in status "open" (#195, N2). + /// cancel_bounty's status check fires before its creator gate + /// (bounties.rs:381), so only an "open" bounty reaches the 403; the creator + /// twin consumes it (open -> cancelled). + OpenBountyId, + /// `{id}` -> the fixture's task seeded "pending" (#195, F3). The claim_task + /// caller-self gate fires before the DB claim, so the stranger 403 does not + /// consume it; the owner control then claims it (pending -> claimed). + ClaimableTaskId, + /// `{id}` -> the fixture's task seeded "claimed" by the assignee (#195, F3). + /// complete_task's assignee actor gate reaches it; the assignee twin then + /// finishes it (claimed -> completed). + CompletableTaskId, + /// `{id}` -> a second assignee-claimed task (#195, F3), so fail_task's twin + /// (claimed -> failed) does not race complete_task for one entity. + FailableTaskId, +} + +/// One deny-bearing route. `path` is a template with `{owner}`/`{repo}`/`{id}` +/// /`{number}`/`{*path}` placeholders the probe generator (U2) fills from the +/// fixture. `needs` names the seeded sub-entities the path requires (empty for +/// owner-gate rows: the gate fires before the entity lookup). +#[derive(Debug, Clone, Copy)] +pub struct Row { + pub method: &'static str, + pub path: &'static str, + pub gate: GateClass, + /// Handler fn name, recorded for review spot-check (per-row verification). + pub handler: &'static str, + /// Sample request body (JSON) or `None`. JSON bodies drive the + /// `Content-Type: application/json` the probe attaches to clear the extractor. + pub body: Option<&'static str>, + /// Seeded sub-entities the path template consumes. + pub needs: &'static [&'static str], + /// Positive-twin strategy for read-gate rows. + pub reach: Reach, + /// For `MultiPrincipalGate` rows, the authorizing arms this gate must drive + /// (one Not403 twin each). Empty for every other class — only meaningful for + /// `MultiPrincipalGate`, and the consistency test enforces that pairing. + pub principals: &'static [Principal], + /// How the `{id}` placeholder is filled for this row (see [`IdSource`]). + pub id_source: IdSource, + /// For `ReadGate` rows, the status an ANONYMOUS (headerless) caller gets on + /// the withheld target. Most read GETs take `Option` auth and existence-hide + /// with a 404 (`Deny404`), but a read GET that *requires* auth rejects a + /// headerless caller with 401 BEFORE any lookup (`Deny401`) — e.g. + /// `list_webhooks`, whose callback URLs are owner-secret config with no anon + /// form. The signed-non-reader 404 (existence-hiding) is unchanged either way; + /// only the anon leg differs. Meaningless on non-`ReadGate` rows, which set + /// `Deny404` as the not-applicable sentinel (mirrors `reach: Reach::None`). + /// A new auth-required read added as a `ReadGate` row that forgets `Deny401` + /// fails loudly (its anon probe gets 401, not the expected 404), so the field + /// is self-correcting rather than a silent default. + pub anon_read: AnonRead, +} + +/// The status class an anonymous caller gets from a `ReadGate` row's withheld +/// target — 404 existence-hiding (the default) or a 401 auth-required rejection. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AnonRead { + /// `Option`-auth read: anon gets the existence-hiding 404, same as a signed + /// non-reader. The default for every read GET that has an anonymous form. + Deny404, + /// Auth-required read: a headerless caller is rejected 401 before any lookup + /// (no anonymous form). The signed-non-reader still gets the 404. + Deny401, +} + +const NO_ENTITY: &[&str] = &[]; +const NO_PRINCIPAL: &[Principal] = &[]; + +/// The deny-bearing route set. Owner-gate and signature-required tranches are +/// fully verified against their handlers this session; the read-gate tranche is +/// populated as each handler's deny path (404-deny vs list-filter) is verified. +pub fn deny_bearing_routes() -> &'static [Row] { + &[ + // ── Owner-gate (403) — verified: each calls require_repo_owner / + // require_owner / inline did_matches against the repo owner. ────────── + Row { + method: "POST", + path: "/api/v1/repos/{owner}/{repo}/pulls/{number}/merge", + gate: GateClass::OwnerGate, + handler: "pulls::merge_pr", + body: None, + needs: &["pr_number"], + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // #195 (F1): close_pr / close_issue are owner-OR-author gates (pulls.rs:276, + // issues.rs:255). A plain OwnerGate tests only the owner arm, so reverting + // the author arm is invisible. MultiPrincipalGate drives BOTH arms. + Row { + method: "POST", + path: "/api/v1/repos/{owner}/{repo}/pulls/{number}/close", + gate: GateClass::MultiPrincipalGate, + handler: "pulls::close_pr", + body: None, + needs: &["pr_number"], + reach: Reach::None, + principals: &[Principal::Owner, Principal::Author], + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/v1/repos/{owner}/{repo}/issues/{id}/close", + gate: GateClass::MultiPrincipalGate, + handler: "issues::close_issue", + body: None, + needs: &["issue_id"], + reach: Reach::None, + principals: &[Principal::Owner, Principal::Author], + id_source: IdSource::IssueId, + anon_read: AnonRead::Deny404, + }, + // #195 (F1): dispute_bounty is creator-OR-claimant (bounties.rs:425). Not + // repo-scoped: it gates on the bounty's creator/claimant, so the row is + // id-keyed by the seeded disputable bounty's UUID (IdSource::BountyId). + Row { + method: "POST", + path: "/api/v1/bounties/{id}/dispute", + gate: GateClass::MultiPrincipalGate, + handler: "bounties::dispute_bounty", + body: None, + needs: &["bounty_id"], + reach: Reach::None, + principals: &[Principal::Creator, Principal::Claimant], + id_source: IdSource::BountyId, + anon_read: AnonRead::Deny404, + }, + // #195 (N2): submit/approve/cancel each 403 a non-claimant/non-creator, but + // their STATUS check fires before the auth check, so each row gets its own + // bounty seeded in the gate-reaching status (claimed / submitted / open). + // The sweep drives probes in emission order (hostile first, probes_for), + // so the stranger 403 fires while the bounty still holds that status; the + // arm twin then consumes it (submit: claimed -> submitted, approve: + // submitted -> completed, cancel: open -> cancelled), which is why the + // rows cannot share a bounty with each other or with dispute_bounty. + Row { + method: "POST", + path: "/api/v1/bounties/{id}/submit", + gate: GateClass::MultiPrincipalGate, + handler: "bounties::submit_bounty", + // pr_id is a free-text column (no FK); any string clears serde and + // the UPDATE. The gate fires before the value is used. + body: Some(r#"{"pr_id":"1"}"#), + needs: &["claimed_bounty_id"], + reach: Reach::None, + principals: &[Principal::Claimant], + id_source: IdSource::ClaimedBountyId, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/v1/bounties/{id}/approve", + gate: GateClass::MultiPrincipalGate, + handler: "bounties::approve_bounty", + // ApproveBountyRequest's only field (tx_hash) is Option, so an empty + // JSON object clears the Json extractor; a missing body would 4xx at + // the extractor, before the gate under test. + body: Some(r#"{}"#), + needs: &["submitted_bounty_id"], + reach: Reach::None, + principals: &[Principal::Creator], + id_source: IdSource::SubmittedBountyId, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/v1/bounties/{id}/cancel", + gate: GateClass::MultiPrincipalGate, + handler: "bounties::cancel_bounty", + body: None, + needs: &["open_bounty_id"], + reach: Reach::None, + principals: &[Principal::Creator], + id_source: IdSource::OpenBountyId, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/v1/repos/{owner}/{repo}/hooks", + gate: GateClass::OwnerGate, + handler: "webhooks::create_webhook", + body: Some(r#"{"url":"https://e.example/h","events":["*"]}"#), + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "DELETE", + path: "/api/v1/repos/{owner}/{repo}/hooks/{id}", + gate: GateClass::OwnerGate, + handler: "webhooks::delete_webhook", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // #113: the OWNER layer of list_webhooks (GET /hooks). On the PUBLIC repo a + // signed stranger passes authorize_repo_read (public) and is then denied 403 + // by require_repo_owner — this row drives that 403. The READ layer (404 on + // the private repo, anon 401) is the sibling ReadGate row; the two share the + // path, admitted by the (method, path, gate) dedup key. A GET owner-gate + // like list_visibility below. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/hooks", + gate: GateClass::OwnerGate, + handler: "webhooks::list_webhooks", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/v1/repos/{owner}/{repo}/labels", + gate: GateClass::OwnerGate, + handler: "labels::add_label", + body: Some(r#"{"label":"bug"}"#), + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "DELETE", + path: "/api/v1/repos/{owner}/{repo}/labels/{label}", + gate: GateClass::OwnerGate, + handler: "labels::remove_label", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/v1/repos/{owner}/{repo}/branches/{branch}/protect", + gate: GateClass::OwnerGate, + handler: "protect::protect_branch", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "DELETE", + path: "/api/v1/repos/{owner}/{repo}/branches/{branch}/protect", + gate: GateClass::OwnerGate, + handler: "protect::unprotect_branch", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "PUT", + path: "/api/v1/repos/{owner}/{repo}/visibility", + gate: GateClass::OwnerGate, + handler: "visibility::set_visibility", + // The twin really sets this rule on the shared prober-pub repo, so the + // glob must stay off "/": the visibility handlers now authorize_repo_read + // before the owner check, and a "/"-withholding rule would 404 every + // later stranger probe on this repo instead of reaching their gates. + body: Some(r#"{"path_glob":"/swept-scope","reader_dids":[]}"#), + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "DELETE", + path: "/api/v1/repos/{owner}/{repo}/visibility", + gate: GateClass::OwnerGate, + handler: "visibility::remove_visibility", + // Removes the rule the set_visibility twin created above. + body: Some(r#"{"path_glob":"/swept-scope"}"#), + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // list_visibility is a 403 owner-gate despite being a GET (calls + // require_owner); the /visibility mount chains put+delete+get, all gated. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/visibility", + gate: GateClass::OwnerGate, + handler: "visibility::list_visibility", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // ── Signature-required (401) — verified: git write route wrapped by the + // require_signature layer (add_auth_layers in server.rs). ───────────── + Row { + method: "POST", + path: "/{owner}/{repo}/git-receive-pack", + gate: GateClass::SignatureRequired, + handler: "repos::git_receive_pack", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // ── Read-gate (404) — verified: each handler calls + // authorize_repo_read / visibility_check on "/" which returns + // AppError::RepoNotFound (404) when a non-reader hits a private repo + // (api/mod.rs:44-62). The Reach twin is the owner issuing the same + // request against the private repo and getting 2xx, proving the 404 is + // the gate and not a merely-absent entity/repo. Every row here gates on + // the whole repo ("/"), so ReaderReads (owner re-read) is the twin; + // the fully-private fixture repo needs no per-row sub-entity seeding + // because these reads either return the seeded repo/blob or an empty + // 2xx list. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}", + gate: GateClass::ReadGate, + handler: "repos::get_repo", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // #195 (F2): repo-root reads that gate on "/" — driven as real ReadGate rows + // rather than source-only exemptions, so a runtime bypass that keeps the + // authorize_repo_read/visibility_check marker but leaks is caught. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/star", + gate: GateClass::ReadGate, + handler: "stars::get_star_status", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/icaptcha-proof", + gate: GateClass::ReadGate, + handler: "repos::get_icaptcha_proof", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/encrypted-blobs/replicate", + gate: GateClass::ReadGate, + handler: "encrypted::replicate_encrypted_blobs", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/commits", + gate: GateClass::ReadGate, + handler: "repos::list_commits", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/tree", + gate: GateClass::ReadGate, + handler: "repos::get_tree_root", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // Path-scoped read: get_blob gates on "/{path}" (repos.rs:390). The + // fully-private fixture repo denies any path to anon (404); the owner + // reads the seeded blob at content_path and gets 2xx bytes. (Path-scoped + // subtree WITHHOLDING on a *public* repo — the SiblingPublic twin — is + // already covered by the U7/U8 cases in deny_harness.rs.) + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/blob/{*path}", + gate: GateClass::ReadGate, + handler: "repos::get_blob", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/refs", + gate: GateClass::ReadGate, + handler: "repos::list_refs", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/issues", + gate: GateClass::ReadGate, + handler: "issues::list_issues", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/labels", + gate: GateClass::ReadGate, + handler: "labels::list_labels", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/certs", + gate: GateClass::ReadGate, + handler: "certs::list_certs", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/events", + gate: GateClass::ReadGate, + handler: "events::list_repo_events", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/pulls", + gate: GateClass::ReadGate, + handler: "pulls::list_prs", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/changelog", + gate: GateClass::ReadGate, + handler: "changelog::get_changelog", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/bounties", + gate: GateClass::ReadGate, + handler: "bounties::list_repo_bounties", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/withheld-paths", + gate: GateClass::ReadGate, + handler: "visibility::withheld_paths", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/encrypted-blobs", + gate: GateClass::ReadGate, + handler: "encrypted::list_encrypted_blobs", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // #195 (F2, U3): sub-entity reads that gate on "/" (the private repo) but + // require the entity to EXIST at the request path for the owner-2xx twin. + // Each carries its own distinctive marker (seeded in probe.rs) added to the + // per-read withheld set, so a 404 that echoes THAT read's private content + // (issue title, PR title, PR diff, bounty title) fails — a status- + // only 404 check would be vacuous. These were source-only exemptions in + // READ_GATE_NOT_DRIVEN before; they are now driven as real ReadGate rows. + // get_issue: id-keyed by the seeded PRIVATE issue (marker in its title). + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/issues/{id}", + gate: GateClass::ReadGate, + handler: "issues::get_issue", + body: None, + needs: &["priv_issue_id"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::PrivIssueId, + anon_read: AnonRead::Deny404, + }, + // list_issue_comments: parent issue must exist (private); child list may be + // empty — an empty `{"comments":[]}` is a non-empty 2xx body. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/issues/{id}/comments", + gate: GateClass::ReadGate, + handler: "issues::list_issue_comments", + body: None, + needs: &["priv_issue_id"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::PrivIssueId, + anon_read: AnonRead::Deny404, + }, + // get_pr: the seeded PRIVATE PR #1 (marker in its title). + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/pulls/{number}", + gate: GateClass::ReadGate, + handler: "pulls::get_pr", + body: None, + needs: &["priv_pr"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // get_pr_diff: the PRIVATE PR #1 has a real `feature` source branch with a + // marker file, so branch_diff_names(main, feature) is NON-EMPTY and the + // owner twin returns a real diff (a bail!/500 if the source ref were + // missing). The per-path visibility_check Deny-return is additionally + // driven by the hand-written get_pr_diff_withheld_path_is_denied test. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/pulls/{number}/diff", + gate: GateClass::ReadGate, + handler: "pulls::get_pr_diff", + body: None, + needs: &["priv_pr"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // list_reviews / list_comments: parent PR #1 must exist (private); child + // lists may be empty. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/pulls/{number}/reviews", + gate: GateClass::ReadGate, + handler: "pulls::list_reviews", + body: None, + needs: &["priv_pr"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/pulls/{number}/comments", + gate: GateClass::ReadGate, + handler: "pulls::list_comments", + body: None, + needs: &["priv_pr"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // get_cert: id-keyed by a real ref-certificate issued by an owner push to + // the private repo. No cert-content marker is seeded: get_cert read-gates + // the repo on "/" and 404s a non-reader BEFORE fetching the cert, so cert + // fields cannot reach the deny body — the repo-scoped tokens (private_repo_id, + // private_secret) plus the status check carry the no-leak assertion here. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/certs/{id}", + gate: GateClass::ReadGate, + handler: "certs::get_cert", + body: None, + needs: &["cert_id"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::CertId, + anon_read: AnonRead::Deny404, + }, + // get_bounty: NOT repo-scoped in its path, read-gates on the bounty's repo. + // Seeded against the PRIVATE repo (marker in its title), id-keyed. + Row { + method: "GET", + path: "/api/v1/bounties/{id}", + gate: GateClass::ReadGate, + handler: "bounties::get_bounty", + body: None, + needs: &["priv_bounty_id"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::PrivBountyId, + anon_read: AnonRead::Deny404, + }, + // get_tree (path-scoped): genuinely ADDITIONAL to get_tree_root (Q1). Root + // gates on "/"; get_tree gates on the REQUESTED subtree (N3, + // authorize_repo_read(&gate_path)), a distinct path-scoped Deny surface. The + // {*path} fills to the private content path, so a non-reader is denied the + // subtree listing and the owner gets a non-empty 2xx. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/tree/{*path}", + gate: GateClass::ReadGate, + handler: "repos::get_tree", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // #113: /replicas and /branches/protected gained an `authorize_repo_read` + // gate on "/" — a private repo's replica URLs / protected-branch list are + // now hidden (404) from a non-reader. Both take `Option` auth (anon and a + // signed non-reader alike get the existence-hiding 404), gate on the whole + // repo, and return an empty-or-seeded 2xx list to the owner, so `ReaderReads` + // (owner re-read) is the twin and no per-row sub-entity seeding is needed. + // Driven as real ReadGate rows rather than left in PUBLIC_REPO_GETS, which + // is why the completeness fence flagged them after the #113 merge. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/replicas", + gate: GateClass::ReadGate, + handler: "replicas::list_replicas", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/branches/protected", + gate: GateClass::ReadGate, + handler: "protect::list_protected_branches", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // #113: list_webhooks is DOUBLY gated — authorize_repo_read (404) THEN + // require_repo_owner (403) — and it rejects a headerless caller with 401 + // BEFORE the lookup (callback URLs are owner-secret config with no anon + // form). This is the READ layer: a signed non-reader of the private repo + // gets the existence-hiding 404, while an anon caller gets 401 (AnonRead:: + // Deny401). The OWNER layer (403 for a signed reader-non-owner) is the + // sibling OwnerGate row above, run against the public repo where the read + // gate passes and the owner gate is what fires. Two rows share this path + // (one ReadGate, one OwnerGate); the consistency dedup keys on + // (method, path, gate) to admit exactly this dual-gate case. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/hooks", + gate: GateClass::ReadGate, + handler: "webhooks::list_webhooks", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny401, + }, + // ── Caller-self / actor (403), #195 (F3). The task/register handlers carry + // genuine caller-self 403 gates a bypass would expose (impersonate a + // delegator/assignee, or forge a trust row for a DID you don't control). + // Before this change they were marker-only (INV-21 vacuous); now each is + // driven hostile-then-authorized. The field-binding shape (a body DID + // field must equal the caller) is `SignerSelfGate`; the actor-vs-stored + // shape (caller must equal the task's stored assignee) is + // `MultiPrincipalGate` with the Assignee arm. Register is DRIVEN, not + // excused: the harness runs icaptcha inert (spawn_node never inits the + // verifier), so a stranger-signed POST /api/register reaches the 403. + Row { + method: "POST", + path: "/api/v1/tasks", + gate: GateClass::SignerSelfGate, + handler: "tasks::create_task", + // delegator_did must equal the signer (tasks.rs:101). Stranger names the + // owner -> mismatch -> 403; owner names the owner -> match -> 201. + body: Some(r#"{"kind":"review","capability":"read","delegator_did":"{owner}"}"#), + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/v1/tasks/{id}/claim", + gate: GateClass::SignerSelfGate, + handler: "tasks::claim_task", + // assignee_did must equal the signer (tasks.rs:174); the gate fires + // before the DB claim, so the stranger 403 leaves the pending task for + // the owner control to claim. + body: Some(r#"{"assignee_did":"{owner}"}"#), + needs: &["claimable_task_id"], + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::ClaimableTaskId, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/register", + gate: GateClass::SignerSelfGate, + handler: "register::register", + // did must equal the signer (register.rs:55). The did must parse + // (register.rs:50) so the 403 is reached, so `{owner}` is a real DID. + // `message` is an unknown field serde ignores. + body: Some(r#"{"did":"{owner}","capabilities":[],"message":"probe"}"#), + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/v1/tasks/{id}/complete", + gate: GateClass::MultiPrincipalGate, + handler: "tasks::complete_task", + // Caller must equal the task's STORED assignee (tasks.rs:220). Driven + // against a claimed task: stranger -> 403, the assignee arm -> 200. + body: Some(r#"{"result":"ok"}"#), + needs: &["completable_task_id"], + reach: Reach::None, + principals: &[Principal::Assignee], + id_source: IdSource::CompletableTaskId, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/v1/tasks/{id}/fail", + gate: GateClass::MultiPrincipalGate, + handler: "tasks::fail_task", + // Caller must equal the task's STORED assignee (tasks.rs:273). Its own + // claimed task so the assignee twin (claimed -> failed) does not race + // complete_task's. + body: Some(r#"{"reason":"x"}"#), + needs: &["failable_task_id"], + reach: Reach::None, + principals: &[Principal::Assignee], + id_source: IdSource::FailableTaskId, + anon_read: AnonRead::Deny404, + }, + // The read-gate handlers NOT driven here (deferred GET reads, read-gating + // mutations, git smart-HTTP reads, the content-addressed read, and the + // global list-filters) are no longer tracked by free-text prose: they are + // ENFORCED in deny_harness.rs's `READ_GATE_NOT_DRIVEN` allowlist, which the + // `every_read_gate_handler_is_driven_or_explicitly_allowlisted` guard checks + // against a live scan of every read-gate marker in src/api. That is the + // single source of truth — adding/removing a driven read row here without + // updating the allowlist trips the guard. + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn registry_internal_consistency() { + let rows = deny_bearing_routes(); + + // No duplicate method+path+gate. The key is (method, path, gate), not + // (method, path): one endpoint can carry two distinct gate LAYERS that each + // need their own probe substrate — list_webhooks (GET /hooks) read-gates + // (404 on the private repo) THEN owner-gates (403 on the public repo), so it + // is one ReadGate row and one OwnerGate row on the same path. An accidental + // copy-paste of a row keeps the same gate, so this still catches the real + // double-registration bug; it only admits the deliberate dual-gate case. + let mut seen = std::collections::HashSet::new(); + for r in rows { + assert!( + seen.insert((r.method, r.path, r.gate)), + "duplicate deny-bearing row: {} {} {:?}", + r.method, + r.path, + r.gate + ); + } + + for r in rows { + // Every row records its handler fn (review spot-check anchor). + assert!( + !r.handler.is_empty(), + "row {} {} has no handler", + r.method, + r.path + ); + + match r.gate { + // Every read-gate row must carry a positive twin, or a 404 from a + // merely-absent entity is indistinguishable from the gate's 404. + GateClass::ReadGate => assert_ne!( + r.reach, + Reach::None, + "read-gate row {} {} needs a Reach positive twin", + r.method, + r.path + ), + // Owner-gate/signature rows use the class's own twin (owner re-sign) + // or none, never a read-path twin. + _ => assert_eq!( + r.reach, + Reach::None, + "non-read-gate row {} {} must not carry a read twin", + r.method, + r.path + ), + } + + // `principals` is meaningful ONLY on MultiPrincipalGate rows: it must + // be non-empty there (a gate with no declared arm drives nothing) and + // empty everywhere else (a stray arm set on a non-multi row is a bug). + match r.gate { + GateClass::MultiPrincipalGate => assert!( + !r.principals.is_empty(), + "multi-principal row {} {} declares no authorizing arms", + r.method, + r.path + ), + _ => assert!( + r.principals.is_empty(), + "non-multi-principal row {} {} must not declare arms", + r.method, + r.path + ), + } + } + } + + /// STRUCTURAL enforcement (#195, F1): every `MultiPrincipalGate` row's + /// generator output must carry exactly one Not403 twin PER declared arm plus + /// the single stranger hostile, so a row that registered an arm but never + /// emits its twin (the vacuous-guard failure) fails HERE rather than in a + /// runtime sweep nobody re-runs. Invokes `probes_for` directly — the same + /// generator the real sweep drives — so the two cannot drift. + #[test] + fn multi_principal_rows_emit_one_twin_per_arm() { + use crate::support::probe::{probes_for, tests_support::fx, Expect, Signer}; + + let fixture = fx(); + let mut checked = 0usize; + for r in deny_bearing_routes() { + if r.gate != GateClass::MultiPrincipalGate { + continue; + } + checked += 1; + let ps = probes_for(r, &fixture); + + let hostile = ps + .iter() + .filter(|p| p.signer == Signer::Stranger && matches!(p.expect, Expect::Deny(403))) + .count(); + assert_eq!( + hostile, 1, + "multi-principal row {} {} must drive exactly one stranger-403 hostile, drove {hostile}", + r.method, r.path + ); + + let twins = ps + .iter() + .filter(|p| matches!(p.expect, Expect::Not403)) + .count(); + assert_eq!( + twins, + r.principals.len(), + "multi-principal row {} {} declares {} arms but emits {twins} Not403 twins", + r.method, + r.path, + r.principals.len(), + ); + + // Each declared arm maps to a distinct signer, and every twin's signer + // is one of the declared arms (no phantom / wrong-arm twin). + for arm in r.principals { + let want = crate::support::probe::signer_for_principal(*arm); + let present = ps + .iter() + .any(|p| p.signer == want && matches!(p.expect, Expect::Not403)); + assert!( + present, + "multi-principal row {} {} declares arm {:?} but emits no Not403 twin for it", + r.method, r.path, arm + ); + } + + // The twins' signers must be DISTINCT and number the declared arms. The + // per-arm any() check above is satisfied by a single twin when two arms + // map to the same Signer (a signer_for_principal collision), so without + // this a wrong mapping would pass with one arm silently untested. + let mut twin_signers: Vec = ps + .iter() + .filter(|p| matches!(p.expect, Expect::Not403)) + .map(|p| p.signer) + .collect(); + twin_signers.sort_by_key(|s| format!("{s:?}")); + twin_signers.dedup(); + assert_eq!( + twin_signers.len(), + r.principals.len(), + "multi-principal row {} {} emits twins with {} DISTINCT signers but declares {} arms \ + (Principal->Signer collision?)", + r.method, + r.path, + twin_signers.len(), + r.principals.len(), + ); + } + assert!( + checked >= 8, + "expected at least the eight known multi-principal rows (close_pr, \ + close_issue, dispute/submit/approve/cancel bounty, complete_task, \ + fail_task), checked {checked}" + ); + } + + /// STRUCTURAL enforcement for `SignerSelfGate` rows (#195, F3): each caller-self + /// field-binding row's generator output must be exactly one Stranger/Deny(403) + /// hostile plus one Owner/Not403 control, and both bodies must bind the self + /// field to the owner DID (so the hostile is a real signer-vs-body mismatch and + /// the control a real match). Mirrors `multi_principal_rows_emit_one_twin_per_arm` + /// so a SignerSelfGate row that stopped emitting its twin (the vacuous-guard + /// failure) fails HERE, not only in a DB sweep nobody re-runs. Invokes + /// `probes_for` directly, the same generator the real sweep drives. + #[test] + fn signer_self_rows_emit_one_hostile_and_one_control() { + use crate::support::probe::{probes_for, tests_support::fx, Expect, Signer}; + + let fixture = fx(); + let mut checked = 0usize; + for r in deny_bearing_routes() { + if r.gate != GateClass::SignerSelfGate { + continue; + } + checked += 1; + let ps = probes_for(r, &fixture); + assert_eq!( + ps.len(), + 2, + "signer-self row {} {} must emit exactly two probes, emitted {}", + r.method, + r.path, + ps.len() + ); + + let hostile = ps + .iter() + .filter(|p| p.signer == Signer::Stranger && matches!(p.expect, Expect::Deny(403))) + .count(); + assert_eq!( + hostile, 1, + "signer-self row {} {} must drive exactly one stranger-403 hostile, drove {hostile}", + r.method, r.path + ); + + let control = ps + .iter() + .filter(|p| p.signer == Signer::Owner && matches!(p.expect, Expect::Not403)) + .count(); + assert_eq!( + control, 1, + "signer-self row {} {} must drive exactly one owner Not403 control, drove {control}", + r.method, r.path + ); + + // The bound self field (delegator_did / assignee_did / did) must resolve + // to the owner DID in BOTH bodies: the hostile is a stranger signing a + // body that names the owner (mismatch), the control an owner signing the + // same (match). A row whose body did not run through `fill()` would carry + // a literal `{owner}` and fail here. + for p in &ps { + let body = String::from_utf8(p.body.clone()).expect("probe body is UTF-8 JSON"); + assert!( + body.contains(&fixture.owner_did), + "signer-self row {} {} body must bind the self field to the owner DID, got {body}", + r.method, + r.path + ); + } + } + assert!( + checked >= 3, + "expected at least the three SignerSelfGate rows (create_task, claim_task, \ + register), checked {checked}" + ); + } + + /// STRUCTURAL enforcement for `OwnerGate` rows: each row's generator must + /// emit exactly one stranger-403 hostile and one owner Not403 control. + #[test] + fn owner_gate_rows_emit_one_hostile_and_one_control() { + use crate::support::probe::{probes_for, tests_support::fx, Expect, Signer}; + + let fixture = fx(); + let mut checked = 0usize; + for r in deny_bearing_routes() { + if r.gate != GateClass::OwnerGate { + continue; + } + checked += 1; + let ps = probes_for(r, &fixture); + assert_eq!( + ps.len(), + 2, + "owner-gate row {} {} must emit exactly two probes, emitted {}", + r.method, + r.path, + ps.len() + ); + let hostile = ps + .iter() + .filter(|p| p.signer == Signer::Stranger && matches!(p.expect, Expect::Deny(403))) + .count(); + assert_eq!( + hostile, 1, + "owner-gate row {} {} must drive exactly one stranger-403 hostile, drove {hostile}", + r.method, r.path + ); + let control = ps + .iter() + .filter(|p| p.signer == Signer::Owner && matches!(p.expect, Expect::Not403)) + .count(); + assert_eq!( + control, 1, + "owner-gate row {} {} must drive exactly one owner Not403 control, drove {control}", + r.method, r.path + ); + } + assert!( + checked >= 6, + "expected at least six OwnerGate rows, checked {checked}" + ); + } + + /// STRUCTURAL enforcement for `ReadGate` rows: anon hostile, signed-non-reader + /// hostile, and one reachability twin per row. + #[test] + fn read_gate_rows_emit_hostiles_and_reach_twin() { + use crate::support::probe::{probes_for, tests_support::fx, Expect, Signer}; + + let fixture = fx(); + let mut checked = 0usize; + for r in deny_bearing_routes() { + if r.gate != GateClass::ReadGate { + continue; + } + checked += 1; + let ps = probes_for(r, &fixture); + assert_eq!( + ps.len(), + 3, + "read-gate row {} {} must emit exactly three probes, emitted {}", + r.method, + r.path, + ps.len() + ); + let anon = ps + .iter() + .filter(|p| p.signer == Signer::Anon && matches!(p.expect, Expect::Deny(_))) + .count(); + assert_eq!( + anon, 1, + "read-gate row {} {} must drive exactly one anon hostile, drove {anon}", + r.method, r.path + ); + let stranger = ps + .iter() + .filter(|p| p.signer == Signer::Stranger && matches!(p.expect, Expect::Deny(404))) + .count(); + assert_eq!( + stranger, 1, + "read-gate row {} {} must drive exactly one signed-non-reader 404 hostile, drove {stranger}", + r.method, r.path + ); + let twin = ps + .iter() + .filter(|p| matches!(p.expect, Expect::Ok2xx)) + .count(); + assert_eq!( + twin, 1, + "read-gate row {} {} must drive exactly one Ok2xx reach twin, drove {twin}", + r.method, r.path + ); + } + assert!( + checked >= 29, + "expected at least twenty-nine ReadGate rows, checked {checked}" + ); + } +} diff --git a/crates/gitlawb-node/tests/support/signing.rs b/crates/gitlawb-node/tests/support/signing.rs new file mode 100644 index 000000000..3e0d543cb --- /dev/null +++ b/crates/gitlawb-node/tests/support/signing.rs @@ -0,0 +1,32 @@ +//! Real RFC-9421 request signing for the deny harness (U2). Wraps +//! `gitlawb_core::http_sig::sign_request` (the same entry point the +//! git-remote-gitlawb helper uses in production) so a test can present a valid +//! signature for any identity, reaching the authenticated-but-wrong-owner path +//! that an injected-DID shortcut cannot. + +use gitlawb_core::http_sig::sign_request; +use gitlawb_core::identity::Keypair; + +/// Build a reqwest request carrying a valid RFC-9421 signature for `keypair`. +/// +/// `path` is both appended to `base_url` to form the request URL and signed as +/// the `@path` component, so the value the server reconstructs and the value +/// that was signed are guaranteed identical. `body` is signed via its +/// content-digest and sent as the request body. +pub fn signed_request( + client: &reqwest::Client, + method: reqwest::Method, + base_url: &str, + path: &str, + body: Vec, + keypair: &Keypair, +) -> reqwest::RequestBuilder { + let sig = sign_request(keypair, method.as_str(), path, &body); + let url = format!("{base_url}{path}"); + client + .request(method, url) + .header("content-digest", sig.content_digest) + .header("signature-input", sig.signature_input) + .header("signature", sig.signature) + .body(body) +}