From c570c172746367824eec1d2c9993f3fb195fe003 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sat, 19 Sep 2026 15:04:36 +0800 Subject: [PATCH 01/23] perf(http): use pooled Hyper async hosts --- Cargo.lock | 72 +- Cargo.toml | 14 +- docs/host-sdk-descriptors.md | 42 +- docs/http-client.md | 41 +- pd-host-function/src/lib.rs | 29 + src/builtins/runtime/http/mod.rs | 164 +- src/builtins/runtime/http/policy.rs | 54 + src/builtins/runtime/http/request.rs | 1773 +++--------------- src/builtins/runtime/http/sse.rs | 1370 +++----------- src/builtins/runtime/mod.rs | 4 +- src/vm/async_host/mod.rs | 45 +- src/vm/host.rs | 145 +- src/vm/tests.rs | 14 + tests/http_async_arch_tests.rs | 85 + tests/standard_host_descriptor_arch_tests.rs | 18 +- tests/vm/http_host_tests.rs | 137 +- tests/vm/http_sse_tests.rs | 5 + tests/vm/io_http_coexistence_tests.rs | 33 +- 18 files changed, 1168 insertions(+), 2877 deletions(-) create mode 100644 tests/http_async_arch_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 944ffff6..21d8bf4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -381,17 +381,6 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" -[[package]] -name = "futures-macro" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.4", -] - [[package]] name = "futures-task" version = "0.3.34" @@ -405,10 +394,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", - "futures-macro", "futures-task", "pin-project-lite", - "slab", ] [[package]] @@ -547,6 +534,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + [[package]] name = "hyper-util" version = "0.1.20" @@ -554,11 +557,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ "bytes", + "futures-channel", + "futures-util", "http", "http-body", "hyper", + "libc", "pin-project-lite", + "socket2", "tokio", + "tower-service", + "tracing", ] [[package]] @@ -839,9 +848,9 @@ dependencies = [ "cranelift-module", "cranelift-native", "futures-channel", - "futures-util", "http-body-util", "hyper", + "hyper-rustls", "hyper-util", "libc", "paste", @@ -851,16 +860,14 @@ dependencies = [ "regex", "rt-format", "rusqlite", - "rustls", "rustyline", "self_cell", "serde", "serde_json", "syn 2.0.117", "tokio", - "tokio-rustls", + "tower-service", "url", - "webpki-roots", "windows-sys 0.59.0", ] @@ -1186,12 +1193,6 @@ dependencies = [ "libc", ] -[[package]] -name = "slab" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" - [[package]] name = "smallvec" version = "1.15.1" @@ -1360,6 +1361,31 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + [[package]] name = "try-lock" version = "0.2.5" diff --git a/Cargo.toml b/Cargo.toml index 25741efb..0b3cb604 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,14 +31,12 @@ runtime = [] async = ["runtime", "dep:tokio"] http-client = [ "async", - "dep:futures-util", "dep:http-body-util", "dep:hyper", + "dep:hyper-rustls", "dep:hyper-util", - "dep:rustls", - "dep:tokio-rustls", + "dep:tower-service", "dep:url", - "dep:webpki-roots", ] sqlite = ["runtime", "dep:rusqlite"] edge-abi = [ @@ -103,13 +101,11 @@ rustyline = { version = "14", optional = true } [target.'cfg(not(target_family = "wasm"))'.dependencies] http-body-util = { version = "0.1", optional = true } hyper = { version = "1", default-features = false, features = ["client", "http1"], optional = true } -hyper-util = { version = "0.1", default-features = false, features = ["tokio"], optional = true } -rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"], optional = true } -tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"], optional = true } +hyper-rustls = { version = "0.27.7", default-features = false, features = ["http1", "ring", "tls12", "webpki-roots"], optional = true } +hyper-util = { version = "0.1", default-features = false, features = ["client-legacy", "http1", "tokio"], optional = true } +tower-service = { version = "0.3", optional = true } url = { version = "2", optional = true } -futures-util = { version = "0.3", optional = true } tokio = { version = "1", features = ["rt-multi-thread", "net", "time", "sync", "fs", "io-util", "process", "macros"], optional = true } -webpki-roots = { version = "1", optional = true } [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.59", features = ["Win32_System_Diagnostics_Debug", "Win32_System_Memory", "Win32_System_ProcessStatus", "Win32_System_Threading"] } diff --git a/docs/host-sdk-descriptors.md b/docs/host-sdk-descriptors.md index d1aaea3e..e4282725 100644 --- a/docs/host-sdk-descriptors.md +++ b/docs/host-sdk-descriptors.md @@ -181,25 +181,30 @@ fn io_open_contract() -> vm::HostFunctionSchema { } ``` -The raw-handle functions of a module that schedules its own operation driver -declare the contract alone. `runtime_owned_pending` is a separate opt-in for the -adapter class whose pending operation is owned by the generic VM -operation/stream registries instead of a registered driver — the standard -example is the HTTP request and the SQLite statement family: +Modules that can await library futures directly should declare ordinary async +host functions. The macro captures any `#[pd_host_context]` value before the VM +borrow ends, submits the future through the embedding's async bridge, and maps +the resolved value through the declared contract: ```rust -/// Streams one HTTP request, resolved by the runtime registries. -#[pd_host_function( - name = "http::client::request", - contract = http_request_contract, - runtime_owned_pending -)] -pub(super) fn builtin_http_client_request( - vm: &mut Vm, +/// Performs one bounded request through a shared library client. +#[pd_host_function(name = "http::client::request", contract = http_request_contract)] +pub(super) async fn builtin_http_client_request( + #[pd_host_context] context: HttpRequestContext, request: VmMapHandle, -) -> VmResult> { /* ... */ } +) -> VmResult { + context.request(request).await +} ``` +The context and parameter types of an async host function must own every value +that crosses the suspension boundary. Owned callable schemas, including bare +function types such as `VmCallable Action>`, are accepted. When an +async opening phase must hand control to generic VM continuation machinery, +return `HostFutureOutput` and use `HostFutureOutput::continue_with`; value +mapping preserves that continuation. The HTTP SSE builtin uses this only to +transfer an opened Hyper response into the generic callable-stream driver. + What the contract does and does not change: - The contract **replaces only the guest schema**. The adapter, binding class, and @@ -215,10 +220,11 @@ What the contract does and does not change: source. Deriving the contract's key from that declaration (as above) keeps the two from drifting. -`runtime_owned_pending` requires a stack-shaped signature and is only valid -alongside a declared contract. `io::*` (and `sqlite::open`) do **not** use it: -they schedule a concrete operation driver in the execution scope and declare -only `contract`. +`runtime_owned_pending` remains available for stack-shaped synchronous adapters +whose pending operation is already owned by a generic VM operation registry. +It requires a declared contract. Prefer an ordinary async declaration whenever +the implementation can await the library future directly; do not wrap such a +function in a domain-specific operation, resource, runtime, or thread. ## 4. Installing a module diff --git a/docs/http-client.md b/docs/http-client.md index 4ff33f4b..16da6a47 100644 --- a/docs/http-client.md +++ b/docs/http-client.md @@ -32,12 +32,16 @@ On a supported native target, enabling `http-client` preserves the public API: capability policy; - `HttpExtension` and `HttpHostExt` install the native HTTP host integration; - `register_http_builtin_module` and `http_host_catalog` expose the native - resource schema and callable metadata; + callable metadata; - `http::client::request` returns a bounded `HttpResponse`; and - `http::client::sse` drives a bounded SSE stream through a script callback. -The HTTP and SSE behavior, resource lifecycle, cancellation, and native async -bridge contracts are unchanged by the wasm boundary. See +Both builtins are ordinary `#[pd_host_function] async fn` declarations. Their +macro-generated wrappers own async-host submission; the HTTP module does not +publish transient request, response, or stream resources. + +The HTTP and SSE behavior, cancellation, and native async bridge contracts are +uniform across supported native targets. See [`callable-runtime.md`](callable-runtime.md) for the general callable and host-runtime contract. @@ -252,7 +256,7 @@ The network future never owns or re-enters the VM. Callback error, protocol comp | `max_stream_duration` | 5 min | Host maximum total duration for SSE calls | | `stream_idle_timeout` | 30 s | Wait-for-network-data bound | -The shared in-flight connection default is 64. Zero values for streaming byte limits or any timeout are invalid configuration; buffered `max_request_body_bytes` and `max_response_body_bytes` may be zero to prohibit request or response payload bytes. `HttpConfig::default()` allows `https`. Embeddings should set explicit host and port allowlists and add `http` only when cleartext transport is required. Buffered HTTP and SSE accept only `http`/`https`. +The shared in-flight HTTP call default is 64. Zero values for streaming byte limits or any timeout are invalid configuration; buffered `max_request_body_bytes` and `max_response_body_bytes` may be zero to prohibit request or response payload bytes. `HttpConfig::default()` allows `https`. Embeddings should set explicit host and port allowlists and add `http` only when cleartext transport is required. Buffered HTTP and SSE accept only `http`/`https`. ## Destination policy and protocol transports @@ -268,18 +272,37 @@ Every protocol uses the same admission, address-pinning, and security policy: - ambient proxy settings are ignored. There is no implicit cookie jar, authentication source, or global proxy state. The policy snapshot taken at call admission applies for the complete operation. - -Buffered HTTP and SSE use direct Hyper HTTP/1 over Tokio/Rustls connections and perform no independent DNS lookup outside the shared admission and pinning path. +Every request and redirect target receives an admission-time DNS/private-address +check. Hyper's connector repeats the address check on the DNS results used when +it opens a new connection, retaining the original hostname for HTTP authority +and TLS SNI. + +Buffered HTTP and SSE share one cloneable Hyper client stored in per-VM HTTP +module state. Hyper owns HTTP/1 transport setup, connection pooling, idle +connection lifecycle, and pooled-connection retry behavior. VM reset/reuse +retains this library client and its pool; HTTP configuration replacement builds +a new client for the new policy snapshot. The host does not maintain a custom +pool, sender cache, connection worker, private Tokio runtime, or reconnect state +machine. ## Deliberately absent APIs and semantics RustScript core provides no script-visible HTTP request ID, response/stream handle, `next`, `next_event`, or `cancel` callable. Streams cannot detach from their caller. There is no multiplexing, background reader, automatic reconnect, provider/model interpretation, agent loop, or platform retry policy. Applications implement provider-specific JSON, `[DONE]`, tool-call deltas, retry rules, and reconnect decisions in RSS or downstream hosts. -## Cancellation migration +## Async ownership and cancellation -PR #13 introduced HTTP-private pending-operation and abort-handle maps, one abort pair per request, HTTP owner routes, request-local runtimes, and HTTP-synthesized cancellation errors. The callable streaming contract supersedes those mechanisms. Buffered requests and SSE submit ordinary futures through the embedding-owned async bridge; HTTP has no private pending map, abort map, operation-ID namespace, token owner route, or cancellation state machine. +Buffered requests and SSE opening run as macro-owned async host futures. HTTP has +no private pending map, abort map, operation-ID namespace, token owner route, or +cancellation state machine. After an SSE response opens, only the generic +callable-stream driver retains the Hyper response body, parser, deadlines, +callback continuation, and in-flight permit needed for callback re-entry and +backpressure. -The generic `src/builtins/runtime/cancellation.rs` remains for non-HTTP runtime callers. HTTP does not depend on `CancellationToken`, `CancellationReason`, `OperationOwner::Http`, or owner-wide cancellation routing. Embedding-owned retirement of a pending future remains VM lifecycle control and rejects late completion; dropping an `Invocation` also retires active producer/callback waits and returns the VM and connection permit for reuse. This lifecycle cleanup is not an HTTP API-level cancellation facility. +The generic `src/builtins/runtime/cancellation.rs` remains for non-HTTP runtime +callers. Embedding-owned retirement drops a pending HTTP future and rejects late +completion. Dropping an `Invocation` also retires active producer/callback waits +and returns the VM and connection permit for reuse. This lifecycle cleanup is +not an HTTP API-level cancellation facility. ## Target and backend notes diff --git a/pd-host-function/src/lib.rs b/pd-host-function/src/lib.rs index ef26c054..f5836e29 100644 --- a/pd-host-function/src/lib.rs +++ b/pd-host-function/src/lib.rs @@ -224,6 +224,16 @@ fn is_async_owned_type(ty: &Type) -> bool { Type::Paren(paren) => is_async_owned_type(&paren.elem), Type::Reference(_) | Type::Slice(_) => false, Type::Tuple(tuple) => tuple.elems.iter().all(is_async_owned_type), + Type::BareFn(function) => { + function + .inputs + .iter() + .all(|input| is_async_owned_type(&input.ty)) + && match &function.output { + ReturnType::Default => true, + ReturnType::Type(_, output) => is_async_owned_type(output), + } + } Type::Path(path) => { let Some(segment) = path.path.segments.last() else { return false; @@ -1715,6 +1725,25 @@ mod tests { ); } + #[test] + fn async_callable_wrapper_accepts_owned_bare_function_schema() { + let attr: Punctuated = parse_quote!(name = "test::async_stream"); + let item: ItemFn = parse_quote! { + /// Streams through an owned callback asynchronously. + async fn async_stream( + callback: VmCallable VmMap>, + ) -> VmResult> { + todo!() + } + }; + + let expanded = expand_pd_host_function(attr, item) + .expect("an owned callable wrapper may cross the async boundary") + .to_string(); + assert!(expanded.contains("VmCallable < fn (VmMap) -> VmMap >")); + assert!(expanded.contains("submit_host_future")); + } + #[test] fn callable_wrapper_preserves_parameter_and_result_schema() { let ty: Type = parse_quote!(VmCallable VmMap>); diff --git a/src/builtins/runtime/http/mod.rs b/src/builtins/runtime/http/mod.rs index 38e4cd58..67b5b292 100644 --- a/src/builtins/runtime/http/mod.rs +++ b/src/builtins/runtime/http/mod.rs @@ -4,13 +4,12 @@ use std::time::{Duration, Instant}; use pd_host_function::pd_host_function; use super::typed::{VmMap, VmMapHandle}; -use super::{borrow_arg, take_arg}; -use crate::HostCallResult; +use super::{CallOutcome, CaptureAsyncHostContext, borrow_arg, return_one}; use crate::host_api::{ HostApiCatalog, HostFunctionSchema, HostParamPassing, HostParamSchema, HostStructField, HostStructSchema, HostTypeSchema, }; -use crate::vm::{HostFunctionRegistry, Vm, VmError, VmResult}; +use crate::vm::{HostFunctionRegistry, HostFutureOutput, Value, Vm, VmError, VmResult}; mod config; pub(super) mod policy; @@ -19,20 +18,6 @@ pub(super) mod sse; pub use config::HttpConfig; use policy::{ConnectionAdmission, ConnectionPermit}; -pub use request::{HttpRequestResource, HttpResponseResource}; - -impl crate::host_extension::HostResourceType for HttpRequestResource { - const KEY: &'static str = "http.request"; - const DESCRIPTION: &'static str = - "An in-flight HTTP request under the configured network policy"; -} - -impl crate::host_extension::HostResourceType for HttpResponseResource { - const KEY: &'static str = "http.response"; - const DESCRIPTION: &'static str = "An open HTTP response body stream"; -} - -pub(crate) use sse::SseStreamResource; const DEFAULT_MAX_HTTP_IN_FLIGHT: usize = 64; @@ -47,13 +32,16 @@ const DEFAULT_MAX_HTTP_IN_FLIGHT: usize = 64; struct HttpHostState { config: Option, admission: ConnectionAdmission, + client: request::HttpClient, } impl Default for HttpHostState { fn default() -> Self { + let config = HttpConfig::default(); Self { config: None, admission: ConnectionAdmission::new(DEFAULT_MAX_HTTP_IN_FLIGHT), + client: request::build_client(&config), } } } @@ -82,9 +70,11 @@ impl HttpHostExt for Vm { .module_state::() .map(|state| state.admission.clone()) .unwrap_or_else(|| ConnectionAdmission::new(DEFAULT_MAX_HTTP_IN_FLIGHT)); + let client = request::build_client(&config); ctx.set_module_state(HttpHostState { config: Some(config), admission, + client, }); Ok(()) } @@ -123,21 +113,21 @@ impl HttpHostExt for Vm { } } -/// Captured HTTP configuration plus a connection permit, used to open a -/// request/stream without re-entering the VM. +/// Captured HTTP configuration, shared Hyper client, and one in-flight permit. pub(super) struct HttpRequestContext { - pub(super) config: HttpConfig, + config: HttpConfig, + client: request::HttpClient, permit: ConnectionPermit, + prepared_request: Option<(request::HttpRequest, Instant)>, } impl HttpRequestContext { - /// Captures the persistent HTTP policy plus a shared in-flight permit for - /// one connection-oriented adapter. + /// Captures persistent HTTP state without leaving a VM borrow in the + /// macro-owned future. /// - /// The deadline is validated *before* the permit is acquired, preserving - /// the historical ordering guarantee (a script timeout that cannot form a - /// deadline is rejected even when the in-flight capacity is exhausted). - fn capture( + /// Deadline validation precedes admission so an unrepresentable script + /// timeout remains the first reported error even when capacity is full. + pub(super) fn capture_for( vm: &mut Vm, script_timeout: Option, protocol: &str, @@ -163,13 +153,33 @@ impl HttpRequestContext { VmError::HostError("HTTP max_stream_duration cannot form a deadline".to_string()) })?; let permit = state.admission.acquire()?; - Ok((Self { config, permit }, deadline)) + Ok(( + Self { + config, + client: state.client.clone(), + permit, + prepared_request: None, + }, + deadline, + )) } +} - /// Consumes the captured permit, transferring it to the caller (e.g. the - /// SSE driver that releases it when the stream finishes). - fn into_permit(self) -> ConnectionPermit { - self.permit +impl CaptureAsyncHostContext for HttpRequestContext { + fn capture(vm: &mut Vm) -> VmResult { + Self::capture_for(vm, None, "HTTP").map(|(context, _)| context) + } + + fn capture_with_args(vm: &mut Vm, args: &[Value]) -> VmResult { + let mut context = Self::capture(vm)?; + let request = match args.first() { + Some(Value::Map(request)) => request, + Some(_) => return Err(VmError::TypeMismatch("http request map")), + None => return Err(VmError::StackUnderflow), + }; + context.prepared_request = + Some(request::prepare_buffered_request(&context.config, request)?); + Ok(context) } } @@ -180,16 +190,7 @@ impl HttpRequestContext { /// registered by [`HttpExtension`] byte-for-byte. pub fn http_host_catalog() -> Arc { Arc::clone(HTTP_HOST_CATALOG.get_or_init(|| { - super::host_modules::module_catalog( - "http", - HTTP_CATALOG_FUNCTIONS, - &[ - http_request_resource, - http_response_resource, - sse_stream_resource, - ], - HTTP_NAMED_STRUCTS, - ) + super::host_modules::module_catalog("http", HTTP_CATALOG_FUNCTIONS, &[], HTTP_NAMED_STRUCTS) })) } @@ -197,9 +198,8 @@ static HTTP_HOST_CATALOG: OnceLock> = OnceLock::new(); /// Guest contract for `http::client::request`. /// -/// The runtime drives a pending operation under the configured network policy; -/// the guest contract is the typed `http.request` resource it opens and the -/// typed `HttpResponse` it resolves to. +/// The request resolves directly to a typed `HttpResponse`; transport and pool +/// state stay hidden in the host implementation. fn http_request_contract() -> HostFunctionSchema { HostFunctionSchema::with_return( "http::client::request", @@ -260,15 +260,7 @@ const HTTP_CATALOG_FUNCTIONS: &[fn() -> crate::host_extension::HostFunctionDescr ]; fn http_catalog_module() -> crate::host_extension::HostModuleDescriptor { - super::host_modules::catalog_module( - "http", - HTTP_CATALOG_FUNCTIONS, - &[ - http_request_resource, - http_response_resource, - sse_stream_resource, - ], - ) + super::host_modules::catalog_module("http", HTTP_CATALOG_FUNCTIONS, &[]) } /// The standard `http` host module. @@ -407,19 +399,6 @@ pub(super) fn sse_summary_struct(response_header: &HostStructSchema) -> HostStru ) } -/// The canonical declarations for the HTTP resource types. -pub(super) fn http_request_resource() -> crate::host_extension::HostResourceTypeMeta { - crate::host_extension::HostResourceTypeMeta::of::() -} - -pub(super) fn http_response_resource() -> crate::host_extension::HostResourceTypeMeta { - crate::host_extension::HostResourceTypeMeta::of::() -} - -pub(super) fn sse_stream_resource() -> crate::host_extension::HostResourceTypeMeta { - crate::host_extension::HostResourceTypeMeta::of::() -} - /// Registers every HTTP host function into `registry` using the exact /// catalog schema path and the authoritative [`standard_host_catalog`] /// snapshot. @@ -474,24 +453,23 @@ impl crate::vm::HostExtension for HttpExtension { } /// Starts an HTTP request under the VM's configured network policy. -/// -/// The request is a named `HttpRequest` record with `method`, `url`, optional -/// `headers` as an array of typed `HttpRequestHeader` wrappers, and optional -/// `body` as a typed `HttpRequestBody` wrapper. `HttpRequestBody` discriminates -/// between `{ kind: "text", text: string }` and `{ kind: "bytes", bytes: bytes }`; -/// the unused payload field is null. The response is a named `HttpResponse` -/// record with `status`, typed `HttpResponseHeader` entries in `headers`, raw -/// response `body` bytes, and the final validated `url`. -#[pd_host_function( - name = "http::client::request", - contract = http_request_contract, - runtime_owned_pending -)] -pub(super) fn builtin_http_client_request( - vm: &mut Vm, +#[pd_host_function(name = "http::client::request", contract = http_request_contract)] +pub(super) async fn builtin_http_client_request( + #[pd_host_context] context: HttpRequestContext, request: VmMapHandle, -) -> VmResult> { - request::perform_buffered_request(vm, request) +) -> VmResult { + let HttpRequestContext { + config, + client, + permit, + prepared_request, + } = context; + let _permit = permit; + let _ = request; + let (request, deadline) = prepared_request.ok_or_else(|| { + VmError::HostError("HTTP request capture did not prepare the request".to_string()) + })?; + request::perform_buffered_request(&client, &config, &request, deadline).await } #[cfg(test)] @@ -521,7 +499,7 @@ mod tests { vm.configure_http(HttpConfig::default()) .expect("default config should be valid"); - let error = super::HttpRequestContext::capture(&mut vm, Some(Duration::MAX), "SSE") + let error = super::HttpRequestContext::capture_for(&mut vm, Some(Duration::MAX), "SSE") .err() .expect("an unrepresentable script timeout should be rejected"); assert!(error.to_string().contains("timeout_ms"), "{error}"); @@ -558,8 +536,8 @@ mod tests { assert!(validate_url(&config, SchemeFamily::Http, &default_port).is_err()); } - #[test] - fn pinned_resolution_preserves_the_original_host_and_validated_address() { + #[tokio::test(flavor = "current_thread")] + async fn pinned_resolution_preserves_the_original_host_and_validated_address() { let config = HttpConfig { allowed_schemes: vec!["http".to_string()], allowed_hosts: vec!["127.0.0.1".to_string()], @@ -568,17 +546,9 @@ mod tests { ..HttpConfig::default() }; let url = "http://127.0.0.1:8080/".parse().expect("valid pinned URL"); - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("runtime should build"); - - let target = runtime - .block_on(super::policy::resolve_url( - &config, - SchemeFamily::Http, - &url, - )) + + let target = super::policy::resolve_url(&config, SchemeFamily::Http, &url) + .await .expect("target should resolve under policy"); assert_eq!(target.host, "127.0.0.1"); diff --git a/src/builtins/runtime/http/policy.rs b/src/builtins/runtime/http/policy.rs index f94a8434..ab2e2897 100644 --- a/src/builtins/runtime/http/policy.rs +++ b/src/builtins/runtime/http/policy.rs @@ -1,8 +1,14 @@ +use std::future::Future; use std::net::{IpAddr, SocketAddr}; +use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::task::{Context, Poll}; use std::time::Instant; +use hyper_util::client::legacy::connect::dns::Name; +use tower_service::Service; + use super::config::HttpConfig; use crate::vm::{VmError, VmResult}; @@ -86,6 +92,54 @@ impl Drop for ConnectionPermit { } } +/// DNS resolver used by Hyper's connector. +/// +/// The request path validates scheme, host, port, and every current DNS answer +/// before dispatch. Hyper invokes this resolver whenever its pool needs a new +/// connection, so the address used by the actual socket is checked again and +/// cannot bypass the private-address policy through DNS rebinding. +#[derive(Clone, Debug)] +pub(super) struct PolicyResolver { + allow_private_ips: bool, +} + +impl PolicyResolver { + pub(super) fn new(config: &HttpConfig) -> Self { + Self { + allow_private_ips: config.allow_private_ips, + } + } +} + +impl Service for PolicyResolver { + type Response = std::vec::IntoIter; + type Error = std::io::Error; + type Future = + Pin> + Send + 'static>>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, name: Name) -> Self::Future { + let host = name.as_str().to_string(); + let allow_private_ips = self.allow_private_ips; + Box::pin(async move { + let addresses = tokio::net::lookup_host((host.as_str(), 0)) + .await? + .collect::>(); + let config = HttpConfig { + allow_private_ips, + ..HttpConfig::default() + }; + validate_resolved_addresses(&config, &addresses).map_err(|error| { + std::io::Error::new(std::io::ErrorKind::PermissionDenied, error.to_string()) + })?; + Ok(addresses.into_iter()) + }) + } +} + pub(super) fn validate_url_policy( config: &HttpConfig, family: SchemeFamily, diff --git a/src/builtins/runtime/http/request.rs b/src/builtins/runtime/http/request.rs index 883d43c0..640af9cf 100644 --- a/src/builtins/runtime/http/request.rs +++ b/src/builtins/runtime/http/request.rs @@ -1,325 +1,107 @@ +use std::error::Error; +use std::future::Future; use std::pin::Pin; use std::sync::Arc; -use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering}; use std::task::{Context, Poll}; use std::time::Instant; -use futures_util::task::AtomicWaker; -use http_body_util::BodyExt; -use hyper::body::Body as _; -use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; -use tokio::sync::Notify; +use http_body_util::Full; +use hyper::body::{Body as _, Bytes, Frame, Incoming}; +use hyper_rustls::{HttpsConnector, HttpsConnectorBuilder}; +use hyper_util::client::legacy::Client; +use hyper_util::client::legacy::connect::HttpConnector; +use hyper_util::rt::TokioExecutor; +use tower_service::Service; -use super::HttpRequestContext; use super::config::HttpConfig; -use super::policy::{ConnectionPermit, SchemeFamily, request_deadline, resolve_url, with_deadline}; -use crate::HostCallResult; +use super::policy::{PolicyResolver, SchemeFamily, phase_deadline, resolve_url, with_deadline}; use crate::builtins::runtime::typed::{VmMap, VmMapHandle}; -use crate::host_api::ResourceTypeKey; -use crate::vm::operation::{ - HostOperation, OperationCancelReason, OperationError, OperationErrorCode, OperationId, - OperationOutcome, OperationResult, OperationSpec, -}; -use crate::vm::resource::{ - CloseProgress, HostResource, ResourceCloseReason, ResourceError, ResourceErrorCode, - ResourceResult, -}; -use crate::vm::{CallReturn, Value, Vm, VmError, VmResult}; - -#[derive(Clone, Default)] -pub(super) struct ResponseReadObserver { - inner: Arc, -} - -#[derive(Default)] -struct ResponseReadMetrics { - phase: AtomicU8, - transport_waker: AtomicWaker, - remaining_body_bytes: AtomicUsize, -} - -impl ResponseReadObserver { - fn mark_final_head(&self) { - self.inner.phase.store(1, Ordering::Release); - } - - pub(super) fn admit_body(&self, limit: usize) { - self.inner - .remaining_body_bytes - .store(limit, Ordering::Release); - self.inner.phase.store(2, Ordering::Release); - self.inner.transport_waker.wake(); - } - - fn body_is_admitted(&self) -> bool { - self.inner.phase.load(Ordering::Acquire) == 2 - } - - fn register_transport_waker(&self, waker: &std::task::Waker) { - self.inner.transport_waker.register(waker); - } - - fn transport_read_limit(&self) -> usize { - if !self.body_is_admitted() { - 1 - } else { - self.inner - .remaining_body_bytes - .load(Ordering::Acquire) - .saturating_add(1) - } - } - - fn body_remaining(&self) -> usize { - self.inner.remaining_body_bytes.load(Ordering::Acquire) - } +use crate::vm::{Value, VmError, VmResult}; - pub(super) fn observe_application_chunk(&self, bytes: usize) { - let _ = self.inner.remaining_body_bytes.fetch_update( - Ordering::AcqRel, - Ordering::Acquire, - |remaining| Some(remaining.saturating_sub(bytes)), - ); - } -} - -// Rustls accepts a 16 KiB TLS fragment plus at most 2 KiB of protocol -// expansion and the five-byte record header. Bounding the adapter below TLS -// makes raw socket reads explicit. Rustls may retain one such record after the -// final HTTP head; ReadCapIo still exposes only remaining application bytes -// plus one overflow sentinel to Hyper. -const TLS_MAX_WIRE_READ: usize = 16_384 + 2_048 + 5; const HTTP_MAX_HEAD_BYTES: usize = 64 * 1024; +const HTTP_MAX_HEADERS: usize = 100; -struct RawReadCapIo { - inner: T, -} - -impl RawReadCapIo { - fn new(inner: T) -> Self { - Self { inner } - } -} +type BoxConnectError = Box; -impl AsyncRead for RawReadCapIo { - fn poll_read( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - let this = self.get_mut(); - let before = buf.filled().len(); - let mut bounded = buf.take(TLS_MAX_WIRE_READ); - match Pin::new(&mut this.inner).poll_read(cx, &mut bounded) { - Poll::Ready(Ok(())) => { - let read = bounded.filled().len(); - let initialized = bounded.initialized().len(); - unsafe { - buf.assume_init(initialized); - buf.set_filled(before + read); - } - Poll::Ready(Ok(())) - } - other => other, - } - } +#[derive(Clone)] +pub(super) struct ConnectTimeout { + inner: C, + timeout: std::time::Duration, } -impl AsyncWrite for RawReadCapIo { - fn poll_write( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - Pin::new(&mut self.get_mut().inner).poll_write(cx, buf) - } - - fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.get_mut().inner).poll_flush(cx) - } - - fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.get_mut().inner).poll_shutdown(cx) - } +impl Service for ConnectTimeout +where + C: Service + Send, + C::Future: Send + 'static, + C::Response: Send + 'static, + C::Error: Into, +{ + type Response = C::Response; + type Error = BoxConnectError; + type Future = Pin> + Send>>; - fn is_write_vectored(&self) -> bool { - self.inner.is_write_vectored() + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx).map_err(Into::into) } - fn poll_write_vectored( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - bufs: &[std::io::IoSlice<'_>], - ) -> Poll> { - Pin::new(&mut self.get_mut().inner).poll_write_vectored(cx, bufs) + fn call(&mut self, destination: hyper::Uri) -> Self::Future { + let future = self.inner.call(destination); + let timeout = self.timeout; + Box::pin(async move { + tokio::time::timeout(timeout, future) + .await + .map_err(|_| { + BoxConnectError::from(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "HTTP connect phase deadline exceeded", + )) + })? + .map_err(Into::into) + }) } } -struct ReadCapIo { - inner: T, - observer: ResponseReadObserver, - header_suffix: [u8; 4], - header_bytes: usize, - head_total_bytes: usize, - header_complete: bool, - post_body_bytes: usize, - status_prefix: [u8; 12], - status_prefix_len: usize, -} - -impl ReadCapIo { - fn new(inner: T, observer: ResponseReadObserver) -> Self { - Self { - inner, - observer, - header_suffix: [0; 4], - header_bytes: 0, - head_total_bytes: 0, - header_complete: false, - post_body_bytes: 0, - status_prefix: [0; 12], - status_prefix_len: 0, - } - } - - fn observe_head_byte(&mut self, byte: u8) -> std::io::Result<()> { - if self.header_complete { - return Ok(()); - } - if self.status_prefix_len < self.status_prefix.len() { - self.status_prefix[self.status_prefix_len] = byte; - self.status_prefix_len += 1; - } - self.header_suffix.rotate_left(1); - self.header_suffix[3] = byte; - self.head_total_bytes = self.head_total_bytes.checked_add(1).ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::InvalidData, - "HTTP response head exceeds limit", - ) - })?; - self.header_bytes = self.header_bytes.checked_add(1).ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::InvalidData, - "HTTP response head exceeds limit", - ) - })?; - if self.head_total_bytes > HTTP_MAX_HEAD_BYTES || self.header_bytes > HTTP_MAX_HEAD_BYTES { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "HTTP response head exceeds limit", - )); - } - if self.header_bytes < 4 || self.header_suffix != *b"\r\n\r\n" { - return Ok(()); - } +type HttpsPolicyConnector = ConnectTimeout>>; - let status = std::str::from_utf8(&self.status_prefix[9..12]) - .ok() - .and_then(|digits| digits.parse::().ok()); - if matches!(status, Some(100..=199)) && status != Some(101) { - self.header_suffix = [0; 4]; - self.header_bytes = 0; - self.status_prefix = [0; 12]; - self.status_prefix_len = 0; - } else { - self.header_complete = true; - self.observer.mark_final_head(); - } - Ok(()) - } -} +/// Cloneable Hyper client retained in per-VM HTTP module state. +pub(super) type HttpClient = Client>; -impl AsyncRead for ReadCapIo { - fn poll_read( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - let this = self.get_mut(); - if this.header_complete && !this.observer.body_is_admitted() { - this.observer.register_transport_waker(cx.waker()); - if !this.observer.body_is_admitted() { - return Poll::Pending; - } - } - let before = buf.filled().len(); - let post_body_phase = this.header_complete - && this.observer.body_is_admitted() - && this.observer.body_remaining() == 0; - let read_limit = if post_body_phase { - let remaining = HTTP_MAX_HEAD_BYTES.saturating_sub(this.post_body_bytes); - if remaining == 0 { - return Poll::Ready(Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "HTTP response trailers exceed limit", - ))); - } - remaining.min(1) - } else { - this.observer.transport_read_limit() - }; - let mut bounded = buf.take(read_limit); - match Pin::new(&mut this.inner).poll_read(cx, &mut bounded) { - Poll::Ready(Ok(())) => { - let read = bounded.filled().len(); - let initialized = bounded.initialized().len(); - if post_body_phase { - this.post_body_bytes = - this.post_body_bytes.checked_add(read).ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::InvalidData, - "HTTP response trailers exceed limit", - ) - })?; - } - for byte in &bounded.filled()[..read] { - if let Err(error) = this.observe_head_byte(*byte) { - return Poll::Ready(Err(error)); - } - } - unsafe { - buf.assume_init(initialized); - buf.set_filled(before + read); - } - Poll::Ready(Ok(())) - } - other => other, - } - } +pub(super) fn build_client(config: &HttpConfig) -> HttpClient { + let mut http = HttpConnector::new_with_resolver(PolicyResolver::new(config)); + http.enforce_http(false); + http.set_connect_timeout(Some(config.connect_timeout)); + http.set_nodelay(true); + let connector = HttpsConnectorBuilder::new() + .with_webpki_roots() + .https_or_http() + .enable_http1() + .wrap_connector(http); + let connector = ConnectTimeout { + inner: connector, + timeout: config.connect_timeout, + }; + let mut builder = Client::builder(TokioExecutor::new()); + builder + .http1_max_buf_size(HTTP_MAX_HEAD_BYTES) + .http1_max_headers(HTTP_MAX_HEADERS); + builder.build(connector) } -impl AsyncWrite for ReadCapIo { - fn poll_write( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - Pin::new(&mut self.get_mut().inner).poll_write(cx, buf) - } - - fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.get_mut().inner).poll_flush(cx) - } - - fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.get_mut().inner).poll_shutdown(cx) - } - - fn is_write_vectored(&self) -> bool { - self.inner.is_write_vectored() - } - - fn poll_write_vectored( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - bufs: &[std::io::IoSlice<'_>], - ) -> Poll> { - Pin::new(&mut self.get_mut().inner).poll_write_vectored(cx, bufs) - } +async fn validate_target_until( + config: &HttpConfig, + url: &url::Url, + deadline: Instant, +) -> VmResult<()> { + let resolved = with_deadline(deadline, resolve_url(config, SchemeFamily::Http, url)).await?; + debug_assert_eq!(resolved.host, url.host_str().unwrap_or_default()); + debug_assert_eq!( + resolved.address.port(), + url.port_or_known_default().unwrap_or(0) + ); + Ok(()) } -#[derive(Clone)] +#[derive(Clone, Debug)] pub(super) struct HttpRequest { pub(super) method: hyper::Method, pub(super) url: url::Url, @@ -573,633 +355,35 @@ fn map_string(map: &VmMap, key: &str) -> VmResult { } } -// --------------------------------------------------------------------------- -// Shared state for the buffered HTTP request lifecycle -// --------------------------------------------------------------------------- - -/// Shared state that coordinates the buffered HTTP request worker thread, -/// the operation poller, and the resource close lifecycle. -#[repr(u8)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum WorkerLifecycle { - NotStarted = 0, - Running = 1, - Finished = 2, -} - -struct BufferedRequestShared { - /// Notified on cancel/close so the worker can break out of a blocking - /// network read. Race-free: if notify_one() arrives before the worker - /// starts waiting, the next notified() completes immediately. - cancel: Notify, - /// One-shot result from the worker thread. - result: std::sync::Mutex>>, - /// Set by the worker after publishing `result`. - done: std::sync::atomic::AtomicBool, - /// Set by the spawned closure after the worker entry returns. - thread_finished: std::sync::atomic::AtomicBool, - /// Explicit worker lifecycle. `NotStarted` is also the only state from - /// which workerless rollback may publish a terminal result. - worker_lifecycle: std::sync::atomic::AtomicU8, - /// Waker registered by a pending operation poll. `register` is followed by - /// a result recheck by the operation driver. - waker: AtomicWaker, - /// The worker thread handle, taken during close to join. - join_handle: std::sync::Mutex>>, - /// Waker registered by the close poll when the worker is still running. - close_waker: AtomicWaker, - /// Waker registered by the operation registry while waiting for worker quiescence. - quiescence_waker: AtomicWaker, - /// The connection permit, held until the shared state is dropped (after - /// the worker exits and the resource is closed). - _permit: ConnectionPermit, - /// Set after a rollback has retired both the operation and resource. This - /// makes repeated rollback calls no-ops without touching stale handles. - rollback_finished: std::sync::atomic::AtomicBool, -} - -impl BufferedRequestShared { - fn mark_worker_running(&self) { - let _ = self.worker_lifecycle.compare_exchange( - WorkerLifecycle::NotStarted as u8, - WorkerLifecycle::Running as u8, - Ordering::AcqRel, - Ordering::Acquire, - ); - } - - fn mark_worker_finished(&self) { - self.thread_finished.store(true, Ordering::Release); - self.worker_lifecycle - .store(WorkerLifecycle::Finished as u8, Ordering::Release); - self.close_waker.wake(); - self.quiescence_waker.wake(); - } - - /// Publishes a terminal rollback result for a resource that never had a - /// worker. The compare-exchange prevents this path from claiming a worker - /// which successfully started between admission and rollback. - fn terminalize_workerless(&self, result: VmResult) -> bool { - if self - .worker_lifecycle - .compare_exchange( - WorkerLifecycle::NotStarted as u8, - WorkerLifecycle::Finished as u8, - Ordering::AcqRel, - Ordering::Acquire, - ) - .is_err() - { - return false; - } - self.request_stop(); - self.publish(result); - self.thread_finished.store(true, Ordering::Release); - self.close_waker.wake(); - self.quiescence_waker.wake(); - true - } - - fn request_stop(&self) { - self.cancel.notify_one(); - self.waker.wake(); - self.close_waker.wake(); - self.quiescence_waker.wake(); - } - - fn publish(&self, result: VmResult) { - *self - .result - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(result); - // The result mutex write happens-before this release publication. - self.done.store(true, Ordering::Release); - self.waker.wake(); - self.close_waker.wake(); - self.quiescence_waker.wake(); - } - - fn has_result(&self) -> bool { - self.result - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .is_some() - } - - fn try_join_finished(&self) -> ResourceResult { - if self.worker_lifecycle.load(Ordering::Acquire) != WorkerLifecycle::Finished as u8 - || !self.done.load(Ordering::Acquire) - || !self.thread_finished.load(Ordering::Acquire) - { - return Ok(false); - } - let handle = { - let mut guard = self - .join_handle - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - if guard.as_ref().is_some_and(|handle| !handle.is_finished()) { - return Ok(false); - } - guard.take() - }; - let Some(handle) = handle else { - return Ok(true); - }; - handle.join().map(|_| true).map_err(|panic| { - ResourceError::new( - ResourceErrorCode::ResourceCleanupFailed, - "http::request::resource", - worker_panic_message(&panic), - ) - }) - } - - fn is_quiescent(&self) -> bool { - self.worker_lifecycle.load(Ordering::Acquire) == WorkerLifecycle::Finished as u8 - && self.done.load(Ordering::Acquire) - && self.thread_finished.load(Ordering::Acquire) - && self - .join_handle - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .is_none() - } -} - -fn worker_panic_message(panic: &Box) -> String { - if let Some(message) = panic.downcast_ref::<&str>() { - (*message).to_string() - } else if let Some(message) = panic.downcast_ref::() { - message.clone() - } else { - "HTTP request worker thread panicked".to_string() - } -} - -#[cfg(test)] -static FAIL_NEXT_WORKER_SPAWN: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); - -#[cfg(test)] -static REJECT_NEXT_OPERATION_ADMISSION: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); - -#[allow(clippy::result_large_err)] -fn start_operation( - vm: &mut Vm, - operation: T, -) -> crate::vm::host_context::HostContextResult { - #[cfg(test)] - if REJECT_NEXT_OPERATION_ADMISSION.swap(false, Ordering::AcqRel) { - return Err(crate::vm::host_context::HostContextError::new( - "http::operation", - "injected operation admission rejection", - )); - } - vm.host_context() - .start_operation(OperationSpec::new(operation)) -} - -fn spawn_worker(name: &str, function: F) -> std::io::Result> -where - F: FnOnce() + Send + 'static, -{ - #[cfg(test)] - if FAIL_NEXT_WORKER_SPAWN.swap(false, Ordering::AcqRel) { - return Err(std::io::Error::other("injected HTTP worker spawn failure")); - } - std::thread::Builder::new() - .name(name.to_string()) - .spawn(function) -} - -// --------------------------------------------------------------------------- -// Generic scoped host resources and operations -// --------------------------------------------------------------------------- - -/// An HTTP request being processed under the configured network policy. -/// -/// The request resource is registered in the execution scope and associated -/// with the buffered HTTP operation. Its close is the terminal teardown; -/// the scope lifecycle closes the resource (and cancels the operation) on -/// reset/shutdown, ensuring the worker thread is retired. -pub struct HttpRequestResource { - shared: Option>, -} - -impl HttpRequestResource { - fn new(shared: Arc) -> Self { - Self { - shared: Some(shared), - } - } -} - -impl HostResource for HttpRequestResource { - fn resource_type_key() -> Option { - ResourceTypeKey::new("http.request").ok() - } - - fn begin_close(&mut self, reason: ResourceCloseReason) -> ResourceResult { - let _ = reason; - let Some(shared) = self.shared.as_ref() else { - return Ok(CloseProgress::Ready); - }; - // Notify the worker to stop promptly, even if it is blocked on a - // network read. The operation's cancel also does this, but the - // resource close is the authoritative teardown path. - shared.request_stop(); - match shared.try_join_finished()? { - true => Ok(CloseProgress::Ready), - false => Ok(CloseProgress::Pending), - } - } - - fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll> { - let Some(shared) = self.shared.as_ref() else { - return Poll::Ready(Ok(())); - }; - match shared.try_join_finished() { - Ok(true) => Poll::Ready(Ok(())), - Ok(false) => { - shared.close_waker.register(cx.waker()); - match shared.try_join_finished() { - Ok(true) => Poll::Ready(Ok(())), - Ok(false) => Poll::Pending, - Err(error) => Poll::Ready(Err(error)), - } - } - Err(error) => Poll::Ready(Err(error)), - } - } -} - -/// The open HTTP response body stream, used as the parent resource for SSE -/// reader children. -/// -/// Closing it aborts the response stream (the child is closed first by the -/// generic child-first scope shutdown). The SSE reader is registered as a -/// child of this resource so the close order is deterministic: SSE reader -/// first, then the response stream parent. -pub struct HttpResponseResource; - -impl HostResource for HttpResponseResource { - fn resource_type_key() -> Option { - ResourceTypeKey::new("http.response").ok() - } - - fn begin_close(&mut self, reason: ResourceCloseReason) -> ResourceResult { - let _ = reason; - Ok(CloseProgress::Ready) - } -} - -/// Driver for the *buffered* HTTP request operation: runs the request on a -/// worker thread and publishes the response map into a shared cell. -pub(super) struct HttpRequestOperation { - shared: Arc, -} - -impl HttpRequestOperation { - fn new(shared: Arc) -> Self { - Self { shared } - } -} - -impl HostOperation for HttpRequestOperation { - fn poll(&mut self, cx: &mut Context<'_>) -> Poll> { - let result = self - .shared - .result - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - match result.as_ref() { - Some(Ok(_)) => Poll::Ready(Ok(())), - Some(Err(error)) => Poll::Ready(Err(OperationError::new( - OperationErrorCode::OperationDriverFailed, - "http::client::request", - error.to_string(), - ))), - None => { - drop(result); - self.shared.waker.register(cx.waker()); - let ready = self.shared.has_result(); - if ready { self.poll(cx) } else { Poll::Pending } - } - } - } - - fn is_quiescent(&self) -> bool { - self.shared.is_quiescent() - } - - fn register_quiescence_waker(&mut self, cx: &Context<'_>) { - self.shared.quiescence_waker.register(cx.waker()); - } - - fn poll_quiescent(&mut self, cx: &mut Context<'_>) -> Poll<()> { - if self.shared.is_quiescent() { - return Poll::Ready(()); - } - self.shared.quiescence_waker.register(cx.waker()); - if self.shared.is_quiescent() { - Poll::Ready(()) - } else { - let _ = self.shared.try_join_finished(); - if self.shared.is_quiescent() { - Poll::Ready(()) - } else { - Poll::Pending - } - } - } - - fn cancel(&mut self, reason: OperationCancelReason) -> OperationResult<()> { - let _ = reason; - self.shared.request_stop(); - Ok(()) - } - - fn cancel_and_wait(&mut self, reason: OperationCancelReason) -> OperationResult<()> { - self.cancel(reason)?; - if !self.shared.is_quiescent() { - return Err(OperationError::new( - OperationErrorCode::OperationDriverFailed, - "http::client::request", - "HTTP request worker cancellation is still pending", - )); - } - Ok(()) - } -} - -impl BufferedRequestShared { - /// Cancellation path used when admission has not yet installed an - /// operation id. It synchronously joins the worker so rollback can close - /// and reclaim the resource before returning the primary admission error. - fn cancel_and_join(&self) -> VmResult<()> { - self.request_stop(); - if !self.is_quiescent() { - return Err(VmError::HostError( - "HTTP request worker cancellation is still pending".to_string(), - )); - } - Ok(()) - } -} - -pub(super) fn host_boundary_error(error: crate::vm::HostContextError) -> VmError { - VmError::HostError(error.to_string()) -} - -fn close_buffered_request_resource( - vm: &mut Vm, - handle: crate::vm::resource::ResourceHandle, -) -> VmResult<()> { - match vm - .host_context() - .close_resource::(handle, ResourceCloseReason::Requested) - .map_err(host_boundary_error)? - { - CloseProgress::Ready => Ok(()), - CloseProgress::Pending => Err(VmError::HostError( - "HTTP request resource close remained pending after worker quiescence".to_string(), - )), - } -} - -fn preserve_cleanup_context(primary: VmError, cleanup: Vec) -> VmError { - if cleanup.is_empty() { - return primary; - } - let mut message = primary.to_string(); - for error in cleanup { - use std::fmt::Write as _; - let _ = write!(message, "; cleanup failed: {error}"); - } - VmError::HostError(message) -} - -fn rollback_buffered_request( - vm: &mut Vm, - resource_handle: crate::vm::resource::ResourceHandle, - shared: &Arc, - op_id: Option, - primary: VmError, -) -> VmError { - if shared.rollback_finished.load(Ordering::Acquire) { - return primary; - } - let mut cleanup = Vec::new(); - let _ = shared.terminalize_workerless(Err(VmError::HostError( - "HTTP request worker was not started".to_string(), - ))); - if let Some(op_id) = op_id { - vm.discard_scoped_operation_completion(op_id); - if let Err(error) = vm - .host_context() - .abort_operation(op_id, OperationCancelReason::Requested) - .map(|_| ()) - .map_err(host_boundary_error) - { - cleanup.push(error); - } - } - if let Err(error) = shared.cancel_and_join() { - cleanup.push(error); - } - if let Err(error) = close_buffered_request_resource(vm, resource_handle) { - cleanup.push(error); - } - if cleanup.is_empty() { - shared.rollback_finished.store(true, Ordering::Release); - } - preserve_cleanup_context(primary, cleanup) -} - -// --------------------------------------------------------------------------- -// Buffered request -// --------------------------------------------------------------------------- - -/// Performs one buffered HTTP request as a generic execution-scope operation. -pub(super) fn perform_buffered_request( - vm: &mut Vm, - request: VmMapHandle, -) -> VmResult> { - let (context, _) = HttpRequestContext::capture(vm, None, "HTTP")?; - let config = context.config.clone(); - let permit = context.into_permit(); - let request = parse_request(&request, &config)?; - let deadline = request_deadline(config.request_timeout)?; - - // Shared state that coordinates the worker thread, operation poll, and - // resource close lifecycle. The permit is held here until the shared - // state is dropped (after the worker exits and the resource is closed). - let shared = Arc::new(BufferedRequestShared { - cancel: Notify::new(), - result: std::sync::Mutex::new(None), - done: std::sync::atomic::AtomicBool::new(false), - thread_finished: std::sync::atomic::AtomicBool::new(false), - worker_lifecycle: std::sync::atomic::AtomicU8::new(WorkerLifecycle::NotStarted as u8), - waker: AtomicWaker::new(), - join_handle: std::sync::Mutex::new(None), - close_waker: AtomicWaker::new(), - quiescence_waker: AtomicWaker::new(), - _permit: permit, - rollback_finished: std::sync::atomic::AtomicBool::new(false), - }); - - // Register an HTTP request resource in the scope and associate the - // operation with it. The scope lifecycle closes the resource (and - // cancels the operation) on reset/shutdown. - let request_resource = HttpRequestResource::new(Arc::clone(&shared)); - let resource_token = vm - .host_context() - .push_resource(request_resource) - .map_err(host_boundary_error)?; - let resource_handle = resource_token.handle(); - - // Admit the operation before spawning the worker. Every later handoff - // step can therefore use the operation id for deterministic rollback; a - // failed spawn never leaves a workerless resource/operation pair behind. - let op = HttpRequestOperation::new(Arc::clone(&shared)); - let op_id = match start_operation(vm, op) { - Ok(op_id) => op_id, - Err(error) => { - return Err(rollback_buffered_request( - vm, - resource_handle, - &shared, - None, - host_boundary_error(error), - )); - } - }; - - let pending_result = Arc::clone(&shared); - if let Err(error) = vm.register_scoped_operation_completion(op_id, move |_vm, outcome| { - let result = match outcome { - OperationOutcome::Completed => pending_result - .result - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .take() - .unwrap_or_else(|| { - Err(VmError::HostError( - "HTTP request produced no result".to_string(), - )) - }), - // Cancellation is an internal teardown path. Resource cleanup is - // still performed below, while callers that explicitly poll a - // cancelled operation receive no guest value. - OperationOutcome::Cancelled(_) => Ok(CallReturn::none()), - OperationOutcome::Failed(error) => Err(VmError::HostError(error.to_string())), - }; - let cleanup = close_buffered_request_resource(_vm, resource_handle); - match (result, cleanup) { - (Ok(values), Ok(())) => Ok(values), - (Err(primary), Ok(())) => Err(primary), - (Ok(_), Err(cleanup)) => Err(cleanup), - (Err(primary), Err(cleanup)) => Err(preserve_cleanup_context(primary, vec![cleanup])), - } - }) { - return Err(rollback_buffered_request( - vm, - resource_handle, - &shared, - Some(op_id), - error, - )); - } - - // Run the request on a worker thread; the operation driver polls the - // shared completion cell. The worker uses tokio::select! to respond - // promptly to cancellation even while blocked on network I/O. - let worker_config = config.clone(); - let worker_request = request.clone(); - let join_handle = match spawn_worker("rustscript-http-request", { - let worker_shared = Arc::clone(&shared); - move || { - let worker_state = Arc::clone(&worker_shared); - worker_state.mark_worker_running(); - let value = match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - match runtime_block_on(async { - tokio::select! { - biased; - _ = worker_shared.cancel.notified() => { - Err(VmError::HostError("HTTP request cancelled".to_string())) - } - result = with_deadline( - deadline, - execute_request_until( - &worker_config, - &worker_request, - ResponseReadObserver::default(), - deadline, - None, - ), - ) => { - result.map(|map| CallReturn::one(Value::Map(Arc::new(map)))) - } - } - }) { - Ok(value) => value, - Err(error) => Err(error), - } - })) { - Ok(value) => value, - Err(panic) => Err(VmError::HostError(format!( - "HTTP request worker panicked: {}", - worker_panic_message(&panic) - ))), - }; - worker_shared.publish(value); - worker_state.mark_worker_finished(); - worker_state.waker.wake(); - } - }) { - Ok(join_handle) => join_handle, - Err(error) => { - return Err(rollback_buffered_request( - vm, - resource_handle, - &shared, - Some(op_id), - VmError::HostError(format!("failed to start HTTP worker: {error}")), - )); - } - }; - - shared.mark_worker_running(); - - // Store the join handle so the resource can join it during close. - *shared - .join_handle - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(join_handle); - - let raw = op_id.raw(); - Ok(HostCallResult::Pending(raw)) +/// Parses and validates a buffered request before async-host submission. +pub(super) fn prepare_buffered_request( + config: &HttpConfig, + request: &VmMapHandle, +) -> VmResult<(HttpRequest, Instant)> { + let request = parse_request(request, config)?; + let deadline = super::policy::request_deadline(config.request_timeout)?; + Ok((request, deadline)) } -/// Builds a current-thread tokio runtime to run the blocking HTTP transport. -fn runtime_block_on(future: F) -> VmResult { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| { - VmError::HostError(format!("HTTP worker runtime build failed: {error}")) - })?; - Ok(runtime.block_on(future)) +/// Executes one bounded request with the shared Hyper client. +pub(super) async fn perform_buffered_request( + client: &HttpClient, + config: &HttpConfig, + request: &HttpRequest, + deadline: Instant, +) -> VmResult { + with_deadline( + deadline, + execute_request_until(client, config, request, deadline), + ) + .await } async fn execute_request_until( + client: &HttpClient, config: &HttpConfig, request: &HttpRequest, - observer: ResponseReadObserver, request_deadline: Instant, - tls_config: Option>, ) -> VmResult { let mut method = request.method.clone(); let mut url = request.url.clone(); @@ -1207,32 +391,12 @@ async fn execute_request_until( let mut headers = request.headers.clone(); for redirect_index in 0..=config.max_redirects { - let connect_deadline = request_deadline.min( - Instant::now() - .checked_add(config.connect_timeout) - .ok_or_else(|| { - VmError::HostError("HTTP connect_timeout cannot form a deadline".to_string()) - })?, - ); - let resolved = with_deadline( - connect_deadline, - resolve_url(config, SchemeFamily::Http, &url), - ) - .await?; - let mut response = send_request( - &method, - &url, - &resolved, - &headers, - body.as_deref(), - ConnectionStage { - observer: observer.clone(), - deadline: connect_deadline, - response_deadline: None, - tls_config: tls_config.clone(), - }, - ) - .await?; + let connect_deadline = phase_deadline(request_deadline, config.connect_timeout); + // Validate every current DNS answer before dispatch. The connector's + // policy resolver repeats the private-address check for the address + // used by any newly opened pooled connection. + validate_target_until(config, &url, connect_deadline).await?; + let mut response = send_request(client, &method, &url, &headers, body.as_deref()).await?; validate_response_framing(response.response())?; if follows_location(response.response().status()) { if redirect_index == config.max_redirects { @@ -1273,7 +437,6 @@ async fn execute_request_until( if !has_body { return Ok(response_map(status, response_headers, Vec::new(), &url)); } - observer.admit_body(config.max_response_body_bytes); let mut bytes = Vec::with_capacity( response .response() @@ -1288,7 +451,6 @@ async fn execute_request_until( let Ok(chunk) = frame.into_data() else { continue; }; - observer.observe_application_chunk(chunk.len()); if bytes.len().saturating_add(chunk.len()) > config.max_response_body_bytes { return Err(response_body_limit_error()); } @@ -1302,67 +464,120 @@ async fn execute_request_until( )) } -type BoxConnection = - Pin> + Send + 'static>>; - +/// Response body owned by Hyper's client and pooled connection lifecycle. pub(super) struct OwnedResponse { - connection: Option, - response: hyper::Response, + response: hyper::Response, } impl OwnedResponse { - pub(super) fn response(&self) -> &hyper::Response { + pub(super) fn response(&self) -> &hyper::Response { &self.response } - pub(super) async fn next_frame( + pub(super) fn poll_next_frame( &mut self, - ) -> VmResult>> { - enum Progress { - Frame(Option, hyper::Error>>), - Connection(Result<(), hyper::Error>), + cx: &mut Context<'_>, + ) -> Poll>>> { + match Pin::new(self.response.body_mut()).poll_frame(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(Some(Ok(frame))) => { + if let Err(error) = validate_response_frame(&frame) { + return Poll::Ready(Err(error)); + } + Poll::Ready(Ok(Some(frame))) + } + Poll::Ready(Some(Err(error))) => Poll::Ready(Err(VmError::HostError(format!( + "HTTP response read failed: {error}" + )))), + Poll::Ready(None) => Poll::Ready(Ok(None)), } + } - loop { - let Some(connection) = self.connection.as_mut() else { - let frame = - self.response - .body_mut() - .frame() - .await - .transpose() - .map_err(|error| { - VmError::HostError(format!("HTTP response read failed: {error}")) - })?; - if let Some(frame) = &frame { - validate_response_frame(frame)?; - } - return Ok(frame); - }; - let progress = tokio::select! { - biased; - frame = self.response.body_mut().frame() => Progress::Frame(frame), - result = connection.as_mut() => Progress::Connection(result), - }; - match progress { - Progress::Frame(frame) => { - let frame = frame.transpose().map_err(|error| { - VmError::HostError(format!("HTTP response read failed: {error}")) - })?; - if let Some(frame) = &frame { - validate_response_frame(frame)?; - } - return Ok(frame); - } - Progress::Connection(Ok(())) => self.connection = None, - Progress::Connection(Err(error)) => { - return Err(VmError::HostError(format!( - "HTTP connection failed: {error}" - ))); - } + pub(super) async fn next_frame(&mut self) -> VmResult>> { + std::future::poll_fn(|cx| self.poll_next_frame(cx)).await + } +} + +pub(super) async fn open_stream_response( + client: &HttpClient, + config: &HttpConfig, + request: &HttpRequest, + opening_deadline: Instant, +) -> VmResult<(OwnedResponse, url::Url)> { + let mut method = request.method.clone(); + let mut url = request.url.clone(); + let mut body = request.body.clone(); + let mut headers = request.headers.clone(); + for redirect_index in 0..=config.max_redirects { + let connect_deadline = phase_deadline(opening_deadline, config.connect_timeout); + validate_target_until(config, &url, connect_deadline).await?; + let response = send_request(client, &method, &url, &headers, body.as_deref()).await?; + validate_response_framing(response.response())?; + if follows_location(response.response().status()) { + if redirect_index == config.max_redirects { + return Err(VmError::HostError( + "HTTP redirect limit exceeded".to_string(), + )); } + let location = response + .response() + .headers() + .get(hyper::header::LOCATION) + .ok_or_else(|| VmError::HostError("HTTP redirect has no location".to_string()))? + .to_str() + .map_err(|_| VmError::HostError("HTTP redirect location is invalid".to_string()))? + .to_string(); + let next_url = url + .join(&location) + .map_err(|error| VmError::HostError(format!("invalid HTTP redirect: {error}")))?; + super::policy::validate_url_policy(config, SchemeFamily::Http, &next_url)?; + prepare_redirect( + &url, + &next_url, + response.response().status(), + &mut method, + &mut body, + &mut headers, + ); + url = next_url; + continue; } + return Ok((response, url)); + } + Err(VmError::HostError( + "HTTP redirect processing failed".to_string(), + )) +} + +async fn send_request( + client: &HttpClient, + method: &hyper::Method, + url: &url::Url, + headers: &[(hyper::header::HeaderName, hyper::header::HeaderValue)], + body: Option<&[u8]>, +) -> VmResult { + let uri = url + .as_str() + .parse::() + .map_err(|error| VmError::HostError(format!("HTTP request setup failed: {error}")))?; + let mut builder = hyper::Request::builder().method(method.clone()).uri(uri); + for (name, value) in headers { + builder = builder.header(name, value); } + let request = builder + .body(Full::new(Bytes::copy_from_slice(body.unwrap_or_default()))) + .map_err(|error| VmError::HostError(format!("HTTP request setup failed: {error}")))?; + let response = client.request(request).await.map_err(|error| { + let mut message = format!("HTTP request failed: {error}"); + let mut source = error.source(); + while let Some(error) = source { + message.push_str(": "); + message.push_str(&error.to_string()); + source = error.source(); + } + VmError::HostError(message) + })?; + Ok(OwnedResponse { response }) } fn response_has_body(method: &hyper::Method, status: hyper::StatusCode) -> bool { @@ -1463,78 +678,6 @@ pub(super) fn response_header_entries(headers: &hyper::HeaderMap) -> Vec .collect() } -pub(super) async fn open_stream_response( - config: &HttpConfig, - request: &HttpRequest, - observer: ResponseReadObserver, - opening_deadline: Instant, - opening_response_deadline: Instant, -) -> VmResult<(OwnedResponse, url::Url)> { - let mut method = request.method.clone(); - let mut url = request.url.clone(); - let mut body = request.body.clone(); - let mut headers = request.headers.clone(); - for redirect_index in 0..=config.max_redirects { - // Each hop may use the connect phase limit, but never beyond the one - // opening deadline supplied by the SSE lifecycle. - let connect_deadline = - super::policy::phase_deadline(opening_deadline, config.connect_timeout); - let resolved = with_deadline( - connect_deadline, - resolve_url(config, SchemeFamily::Http, &url), - ) - .await?; - let response = send_request( - &method, - &url, - &resolved, - &headers, - body.as_deref(), - ConnectionStage { - observer: observer.clone(), - deadline: connect_deadline, - response_deadline: Some(opening_response_deadline), - tls_config: None, - }, - ) - .await?; - validate_response_framing(response.response())?; - if follows_location(response.response().status()) { - if redirect_index == config.max_redirects { - return Err(VmError::HostError( - "HTTP redirect limit exceeded".to_string(), - )); - } - let location = response - .response() - .headers() - .get(hyper::header::LOCATION) - .ok_or_else(|| VmError::HostError("HTTP redirect has no location".to_string()))? - .to_str() - .map_err(|_| VmError::HostError("HTTP redirect location is invalid".to_string()))? - .to_string(); - let next_url = url - .join(&location) - .map_err(|error| VmError::HostError(format!("invalid HTTP redirect: {error}")))?; - super::policy::validate_url_policy(config, SchemeFamily::Http, &next_url)?; - prepare_redirect( - &url, - &next_url, - response.response().status(), - &mut method, - &mut body, - &mut headers, - ); - url = next_url; - continue; - } - return Ok((response, url)); - } - Err(VmError::HostError( - "HTTP redirect processing failed".to_string(), - )) -} - fn response_map( status: hyper::StatusCode, headers: Vec, @@ -1552,7 +695,41 @@ fn response_map( ]) } +fn validate_response_head_size(response: &hyper::Response) -> VmResult<()> { + let version = match response.version() { + hyper::Version::HTTP_10 => "HTTP/1.0", + _ => "HTTP/1.1", + }; + let mut bytes = version + .len() + .checked_add(1 + 3 + 2) + .and_then(|bytes| { + response + .status() + .canonical_reason() + .map_or(Some(bytes), |reason| bytes.checked_add(1 + reason.len())) + }) + .ok_or_else(|| VmError::HostError("HTTP response head exceeds limit".to_string()))?; + for (name, value) in response.headers() { + bytes = bytes + .checked_add(name.as_str().len()) + .and_then(|bytes| bytes.checked_add(value.len())) + .and_then(|bytes| bytes.checked_add(4)) + .ok_or_else(|| VmError::HostError("HTTP response head exceeds limit".to_string()))?; + } + bytes = bytes + .checked_add(2) + .ok_or_else(|| VmError::HostError("HTTP response head exceeds limit".to_string()))?; + if bytes > HTTP_MAX_HEAD_BYTES { + return Err(VmError::HostError( + "HTTP response head exceeds limit".to_string(), + )); + } + Ok(()) +} + fn validate_response_framing(response: &hyper::Response) -> VmResult<()> { + validate_response_head_size(response)?; let headers = response.headers(); let content_lengths: Vec<_> = headers .get_all(hyper::header::CONTENT_LENGTH) @@ -1653,229 +830,16 @@ fn reject_declared_oversize( Ok(()) } -struct ConnectionStage { - observer: ResponseReadObserver, - deadline: Instant, - /// Bounds the response-header wait after the request is written. Streaming - /// adapters pass one absolute opening deadline; buffered requests leave - /// this `None` because their outer request deadline covers the whole call. - response_deadline: Option, - tls_config: Option>, -} - -async fn send_request( - method: &hyper::Method, - url: &url::Url, - resolved: &super::policy::ResolvedTarget, - headers: &[(hyper::header::HeaderName, hyper::header::HeaderValue)], - body: Option<&[u8]>, - stage: ConnectionStage, -) -> VmResult { - let ConnectionStage { - observer, - deadline: connect_deadline, - response_deadline, - tls_config, - } = stage; - let stream = with_deadline(connect_deadline, async { - tokio::net::TcpStream::connect(resolved.address) - .await - .map_err(|error| VmError::HostError(format!("HTTP request failed: {error}"))) - }) - .await?; - let peer = stream - .peer_addr() - .map_err(|error| VmError::HostError(format!("HTTP request failed: {error}")))?; - if peer != resolved.address { - return Err(VmError::HostError( - "HTTP connected peer does not match the validated address".to_string(), - )); - } - stream - .set_nodelay(true) - .map_err(|error| VmError::HostError(format!("HTTP request failed: {error}")))?; - - let raw = RawReadCapIo::new(stream); - if url.scheme() == "https" { - let mut tls_config = tls_config.map_or_else( - || { - let mut roots = rustls::RootCertStore::empty(); - roots.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); - rustls::ClientConfig::builder() - .with_root_certificates(roots) - .with_no_client_auth() - }, - Arc::unwrap_or_clone, - ); - tls_config.alpn_protocols = vec![b"http/1.1".to_vec()]; - let server_name = rustls::pki_types::ServerName::try_from(resolved.host.clone()) - .map_err(|_| VmError::HostError("HTTP TLS server name is invalid".to_string()))?; - let stream = with_deadline(connect_deadline, async { - tokio_rustls::TlsConnector::from(Arc::new(tls_config)) - .connect(server_name, raw) - .await - .map_err(|error| VmError::HostError(format!("HTTP request failed: {error}"))) - }) - .await?; - send_over_io( - method, - url, - headers, - body, - ReadCapIo::new(stream, observer), - response_deadline, - ) - .await - } else { - send_over_io( - method, - url, - headers, - body, - ReadCapIo::new(raw, observer), - response_deadline, - ) - .await - } -} - -async fn send_over_io( - method: &hyper::Method, - url: &url::Url, - headers: &[(hyper::header::HeaderName, hyper::header::HeaderValue)], - body: Option<&[u8]>, - io: ReadCapIo, - response_deadline: Option, -) -> VmResult -where - T: AsyncRead + AsyncWrite + Unpin + Send + 'static, -{ - let mut connection_builder = hyper::client::conn::http1::Builder::new(); - connection_builder - .read_buf_exact_size(Some(8 * 1024)) - .max_buf_size(HTTP_MAX_HEAD_BYTES * 2) - .max_headers(100); - let (mut sender, connection) = connection_builder - .handshake(hyper_util::rt::TokioIo::new(io)) - .await - .map_err(|error| VmError::HostError(format!("HTTP request failed: {error}")))?; - - let path_and_query = match url.query() { - Some(query) => format!("{}?{query}", url.path()), - None => url.path().to_string(), - }; - let mut builder = hyper::Request::builder() - .method(method.clone()) - .uri(path_and_query) - .header( - hyper::header::HOST, - &url[url::Position::BeforeHost..url::Position::AfterPort], - ); - for (name, value) in headers { - builder = builder.header(name, value); - } - let request_body = http_body_util::Full::new(hyper::body::Bytes::copy_from_slice( - body.unwrap_or_default(), - )); - let request = builder - .body(request_body) - .map_err(|error| VmError::HostError(format!("HTTP request setup failed: {error}")))?; - let mut connection: BoxConnection = Box::pin(connection); - // The response wait (including the request write) is bounded by the - // absolute opening deadline when one is supplied. That deadline was - // captured before stream admission and is never recreated per redirect; - // buffered requests use their outer request deadline instead. - let send_response = async { - let response = sender.send_request(request); - tokio::pin!(response); - let (response, connection) = { - tokio::select! { - biased; - response = &mut response => ( - response.map_err(|error| { - VmError::HostError(format!("HTTP request failed: {error}")) - })?, - Some(connection), - ), - connection_result = connection.as_mut() => { - let response_result = response.await; - let response = match (connection_result, response_result) { - (_, Ok(response)) => response, - (Ok(()), Err(error)) => { - return Err(VmError::HostError(format!( - "HTTP request failed: {error}" - ))); - } - (Err(connection_error), Err(request_error)) => { - return Err(VmError::HostError(format!( - "HTTP connection failed before the response: {connection_error}; request failed: {request_error}" - ))); - } - }; - (response, None) - } - } - }; - Ok::<_, VmError>((response, connection)) - }; - let (response, connection) = match response_deadline { - Some(deadline) => with_deadline(deadline, send_response).await?, - None => send_response.await?, - }; - Ok(OwnedResponse { - connection, - response, - }) -} - #[cfg(test)] mod tests { use std::sync::Arc; - use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use super::{ - BufferedRequestShared, FAIL_NEXT_WORKER_SPAWN, HTTP_MAX_HEAD_BYTES, HttpRequestOperation, - HttpRequestResource, REJECT_NEXT_OPERATION_ADMISSION, ReadCapIo, RequestHeaderBudget, - ResponseReadObserver, WorkerLifecycle, parse_request, parse_request_body, - rollback_buffered_request, spawn_worker, start_operation, validate_response_trailers, + RequestHeaderBudget, parse_request, parse_request_body, validate_response_trailers, }; use crate::builtins::runtime::typed::VmMap; use crate::vm::{Value, VmError}; - fn empty_vm() -> crate::vm::Vm { - crate::vm::Vm::new(crate::vm::Program::new( - Vec::new(), - vec![crate::vm::OpCode::Ret as u8], - )) - } - - fn buffered_shared() -> std::sync::Arc { - let permit = crate::builtins::runtime::http::policy::ConnectionAdmission::new(1) - .acquire() - .expect("test permit"); - std::sync::Arc::new(BufferedRequestShared { - cancel: tokio::sync::Notify::new(), - result: std::sync::Mutex::new(None), - done: AtomicBool::new(false), - thread_finished: AtomicBool::new(false), - worker_lifecycle: AtomicU8::new(WorkerLifecycle::NotStarted as u8), - waker: futures_util::task::AtomicWaker::new(), - join_handle: std::sync::Mutex::new(None), - close_waker: futures_util::task::AtomicWaker::new(), - quiescence_waker: futures_util::task::AtomicWaker::new(), - _permit: permit, - rollback_finished: AtomicBool::new(false), - }) - } - - fn request_with_body(body: Value) -> VmMap { - let mut request = VmMap::new(); - request.insert(Value::string("method"), Value::string("POST")); - request.insert(Value::string("url"), Value::string("http://example.test/")); - request.insert(Value::string("body"), body); - request - } - fn value_map(entries: impl IntoIterator) -> Value { Value::Map(Arc::new(VmMap::from_entries( entries @@ -1885,176 +849,55 @@ mod tests { ))) } - fn request_with_headers(headers: Vec<(&str, &str)>) -> VmMap { - let mut request = VmMap::new(); - request.insert(Value::string("method"), Value::string("GET")); - request.insert(Value::string("url"), Value::string("http://example.test/")); - request.insert( - Value::string("headers"), - Value::Array(Arc::new( - headers - .into_iter() - .map(|(name, value)| { - Value::Map(std::sync::Arc::new(VmMap::from_entries(vec![ - (Value::string("name"), Value::string(name)), - (Value::string("value"), Value::string(value)), - ]))) - }) - .collect(), - )), - ); - request + fn request_with_body(body: Value) -> VmMap { + VmMap::from_entries(vec![ + (Value::string("method"), Value::string("POST")), + (Value::string("url"), Value::string("http://example.test/")), + (Value::string("body"), body), + ]) } #[test] fn request_body_payload_checks_limit_before_copying() { - let exact_config = crate::builtins::runtime::http::HttpConfig { - max_request_body_bytes: 7, + let config = crate::builtins::runtime::http::HttpConfig { + max_request_body_bytes: 6, ..Default::default() }; - let text = value_map([ + let body = value_map([ ("kind", Value::string("text")), ("text", Value::string("payload")), ]); - assert_eq!( - parse_request_body(Some(&text), &exact_config) - .expect("exact text limit should be accepted"), - Some(b"payload".to_vec()) - ); - - let zero_config = crate::builtins::runtime::http::HttpConfig { - max_request_body_bytes: 0, - ..Default::default() - }; - let empty_bytes = value_map([ - ("kind", Value::string("bytes")), - ("bytes", Value::bytes(Vec::new())), - ]); - assert_eq!( - parse_request_body(Some(&empty_bytes), &zero_config) - .expect("zero-length bytes should fit zero limit"), - Some(Vec::new()) + let error = parse_request_body(Some(&body), &config).unwrap_err(); + assert!( + matches!(error, VmError::HostError(message) if message == "HTTP request body exceeds limit") ); - - let limited_config = crate::builtins::runtime::http::HttpConfig { - max_request_body_bytes: 6, - ..Default::default() - }; - let text_error = parse_request_body(Some(&text), &limited_config) - .expect_err("one byte over the text limit must be rejected before copying"); - assert!(matches!( - text_error, - VmError::HostError(message) if message == "HTTP request body exceeds limit" - )); - - let bytes = value_map([ - ("kind", Value::string("bytes")), - ("bytes", Value::bytes(b"payload".to_vec())), - ]); - let bytes_error = parse_request_body(Some(&bytes), &limited_config) - .expect_err("one byte over the bytes limit must be rejected before copying"); - assert!(matches!( - bytes_error, - VmError::HostError(message) if message == "HTTP request body exceeds limit" - )); } #[test] fn request_body_discriminator_rejects_invalid_variants() { let config = crate::builtins::runtime::http::HttpConfig::default(); - let cases = [ - ( - value_map([ - ("kind", Value::string("json")), - ("text", Value::string("payload")), - ]), - "HTTP request body kind must", - ), - ( - value_map([("kind", Value::string("text"))]), - "HTTP request body text payload", - ), - ( - value_map([("kind", Value::string("bytes"))]), - "HTTP request body bytes payload", - ), - ( - value_map([ - ("kind", Value::string("text")), - ("text", Value::string("payload")), - ("bytes", Value::bytes(b"raw".to_vec())), - ]), - "text variant cannot contain bytes", - ), - ( - value_map([ - ("kind", Value::string("bytes")), - ("text", Value::string("payload")), - ("bytes", Value::bytes(b"raw".to_vec())), - ]), - "bytes variant cannot contain text", - ), - ( - value_map([ - ("kind", Value::string("text")), - ("text", Value::string("payload")), - ("extra", Value::Null), - ]), - "contains unknown field 'extra'", - ), - ]; - for (body, expected) in cases { - let error = match parse_request(&request_with_body(body), &config) { - Ok(_) => panic!("invalid request body variant was accepted"), - Err(error) => error, - }; - assert!( - error.to_string().contains(expected), - "expected {expected:?}, got {error}" - ); - } + let body = value_map([ + ("kind", Value::string("text")), + ("text", Value::string("payload")), + ("bytes", Value::bytes(b"raw".to_vec())), + ]); + let error = parse_request(&request_with_body(body), &config).unwrap_err(); + assert!( + error + .to_string() + .contains("text variant cannot contain bytes") + ); } #[test] fn request_header_budget_counts_wire_overhead_at_exact_boundary() { let mut budget = RequestHeaderBudget::new(1, 8); - budget - .admit(b"x", b"y") - .expect("one header line is four bytes"); - budget.finish().expect("the final CRLF is two bytes"); + budget.admit(b"x", b"y").unwrap(); + budget.finish().unwrap(); assert_eq!(budget.count(), 1); assert_eq!(budget.bytes(), 8); } - #[test] - fn request_header_budget_rejects_over_limit_before_header_conversion() { - let config = crate::builtins::runtime::http::HttpConfig { - max_request_header_count: 1, - max_request_header_bytes: 8, - ..Default::default() - }; - let error = match parse_request(&request_with_headers(vec![("x", "yy")]), &config) { - Ok(_) => panic!("header line plus terminator exceeds eight bytes"), - Err(error) => error, - }; - assert!(matches!(error, VmError::HostError(message) if message.contains("header bytes"))); - } - - #[test] - fn request_header_budget_rejects_many_tiny_headers_by_count_and_bytes() { - let mut count_limited = RequestHeaderBudget::new(2, 1024); - count_limited.admit(b"a", b"b").unwrap(); - count_limited.admit(b"c", b"d").unwrap(); - let error = count_limited.admit(b"e", b"f").unwrap_err(); - assert!(error.to_string().contains("header count")); - - let mut bytes_limited = RequestHeaderBudget::new(16, 13); - bytes_limited.admit(b"a", b"b").unwrap(); - bytes_limited.admit(b"c", b"d").unwrap(); - let error = bytes_limited.finish().unwrap_err(); - assert!(error.to_string().contains("header bytes")); - } - #[test] fn response_trailer_budget_rejects_aggregate_without_per_field_overflow() { let mut headers = hyper::HeaderMap::new(); @@ -2070,88 +913,4 @@ mod tests { let error = validate_response_trailers(&headers).unwrap_err(); assert!(error.to_string().contains("trailers")); } - - #[test] - fn response_head_budget_accepts_exact_limit_and_rejects_one_byte_over() { - fn head_with_size(size: usize) -> Vec { - let prefix = b"HTTP/1.1 204 No Content\r\nX-Pad: "; - let suffix = b"\r\n\r\n"; - let value_len = size - prefix.len() - suffix.len(); - let mut head = Vec::with_capacity(size); - head.extend_from_slice(prefix); - head.extend(std::iter::repeat_n(b'a', value_len)); - head.extend_from_slice(suffix); - assert_eq!(head.len(), size); - head - } - - let exact = head_with_size(HTTP_MAX_HEAD_BYTES); - let mut exact_io = ReadCapIo::new(tokio::io::empty(), ResponseReadObserver::default()); - for byte in exact { - exact_io - .observe_head_byte(byte) - .expect("exact response head should be admitted"); - } - assert!(exact_io.header_complete); - - let over = head_with_size(HTTP_MAX_HEAD_BYTES + 1); - let mut over_io = ReadCapIo::new(tokio::io::empty(), ResponseReadObserver::default()); - let error = over - .into_iter() - .try_for_each(|byte| over_io.observe_head_byte(byte)) - .expect_err("one byte over the response-head limit must fail"); - assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); - } - - #[test] - fn admission_rollback_reclaims_workerless_request_resource() { - let mut vm = empty_vm(); - let shared = buffered_shared(); - let token = vm - .execution_scope() - .push_resource(HttpRequestResource::new(std::sync::Arc::clone(&shared))) - .expect("request resource"); - let primary = VmError::HostError("operation admission rejected".to_string()); - REJECT_NEXT_OPERATION_ADMISSION.store(true, Ordering::Release); - let admission = start_operation(&mut vm, HttpRequestOperation::new(Arc::clone(&shared))); - assert!(admission.is_err()); - - let error = rollback_buffered_request(&mut vm, token.handle(), &shared, None, primary); - - assert!(error.to_string().contains("operation admission rejected")); - assert_eq!(vm.execution_scope().resources().len(), 0); - assert_eq!( - shared.worker_lifecycle.load(Ordering::Acquire), - WorkerLifecycle::Finished as u8 - ); - assert!(shared.done.load(Ordering::Acquire)); - assert!(shared.thread_finished.load(Ordering::Acquire)); - - let repeated = rollback_buffered_request( - &mut vm, - token.handle(), - &shared, - None, - VmError::HostError("repeated rollback".to_string()), - ); - assert!(repeated.to_string().contains("repeated rollback")); - } - - #[test] - fn worker_spawn_abstraction_can_inject_a_builder_failure() { - FAIL_NEXT_WORKER_SPAWN.store(true, Ordering::Release); - let result = spawn_worker("injected-http-worker", || {}); - let error = match result { - Ok(handle) => { - handle.join().expect("unexpected worker"); - panic!("spawn should have been rejected") - } - Err(error) => error, - }; - assert!( - error - .to_string() - .contains("injected HTTP worker spawn failure") - ); - } } diff --git a/src/builtins/runtime/http/sse.rs b/src/builtins/runtime/http/sse.rs index 9283aba0..8f904ce6 100644 --- a/src/builtins/runtime/http/sse.rs +++ b/src/builtins/runtime/http/sse.rs @@ -1,51 +1,23 @@ +use std::collections::VecDeque; use std::future::Future; +use std::pin::Pin; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::task::{Context, Poll}; use std::time::{Duration, Instant}; -use futures_util::task::AtomicWaker; use pd_host_function::pd_host_function; -use tokio::sync::{Notify, mpsc}; use super::request::{ - HttpRequest, OwnedResponse, ResponseReadObserver, open_stream_response, parse_request, - response_header_entries, validate_request_header_budget, + HttpRequest, OwnedResponse, open_stream_response, parse_request, response_header_entries, + validate_request_header_budget, }; -use super::{HttpRequestContext, policy}; -use crate::builtins::runtime::HostCallResult; +use super::{CaptureAsyncHostContext, HostFutureOutput, HttpRequestContext, policy}; use crate::builtins::runtime::typed::{VmCallable, VmMap, VmMapHandle}; -use crate::host_api::ResourceTypeKey; -use crate::vm::async_host::{ - HostStreamAction, HostStreamDriver, HostStreamPoll, HostStreamTermination, -}; -use crate::vm::operation::{ - HostOperation, OperationCancelReason, OperationError, OperationErrorCode, OperationId, - OperationResult, OperationSpec, -}; -use crate::vm::resource::{ - CloseProgress, HostResource, ResourceCloseReason, ResourceError, ResourceErrorCode, - ResourceHandle, ResourceResult, -}; -use crate::vm::{ - CallOutcome, Value, Vm, VmError, VmResult, - execution_scope::{ExecutionScope, ExecutionScopeError}, -}; - -/// Maximum number of SSE items buffered between the worker and the stream -/// driver before publishing applies backpressure. A small bounded queue -/// preserves ordering without letting the worker run arbitrarily far ahead of -/// the per-item callback, and without unbounded memory growth on a slow or -/// stalled callback. The worker blocks on an under-capacity send, which keeps -/// it in sync with the driver and prevents both event loss and runaway queue -/// growth. -const SSE_CHANNEL_CAPACITY: usize = 1; +use crate::vm::async_host::{HostStreamAction, HostStreamDriver, HostStreamPoll}; +use crate::vm::operation::OperationCancelReason; +use crate::vm::{CallOutcome, Value, Vm, VmError, VmResult}; -#[cfg(test)] -const _: () = assert!(SSE_CHANNEL_CAPACITY == 1); - -/// The error surfaced when the absolute stream deadline (the minimum of the -/// host maximum stream duration and the script `timeout_ms`) is exceeded. +/// The error surfaced when the absolute stream deadline is exceeded. const SSE_TOTAL_DEADLINE_ERROR: &str = "SSE total deadline exceeded"; #[derive(Debug, PartialEq, Eq)] @@ -369,334 +341,90 @@ fn parse_stream_timeout(request: &VmMap) -> VmResult> { } } -/// Shared SSE stream state owned by the child [`SseStreamResource`]. -/// -/// The child resource is registered under the opened response stream -/// resource, so the generic child-first scope shutdown closes the SSE reader -/// before its underlying response stream. The stop flag is set by the child's -/// [`HostResource::begin_close`] and by the SSE poll operation's cancel; the -/// worker observes it between items and stops promptly. -#[repr(u8)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum SseWorkerLifecycle { - NotStarted = 0, - Running = 1, - Finished = 2, -} - -pub(super) struct SseShared { - /// Set on close/cancel; the worker stops polling the network. - pub(super) stopping: AtomicBool, - /// Notified on close/cancel so the worker can break out of a - /// blocking network read. Race-free: if notify_one() arrives before - /// the worker starts waiting, the next notified() completes immediately. - pub(super) cancel: Notify, - /// The first cancellation reason is retained for the producer and cleanup - /// diagnostics. Later cancellation requests cannot overwrite it. - pub(super) cancellation_reason: std::sync::Mutex>, - /// One acknowledgement is issued by the VM after each callback. The - /// acknowledgement arrives. - pub(super) item_ack: Notify, - /// Waker registered by a pending stream poll. Channel readiness is handled - /// by `Receiver::poll_recv`; this waker covers stop and terminal state. - pub(super) waker: AtomicWaker, - /// Bounded FIFO of published items awaiting the stream driver. - /// The worker `send`s with backpressure; the driver `try_recv`s. - /// This preserves item ordering and never drops events, unlike a - /// single-slot overwrite slot. - pub(super) items: mpsc::Sender, - /// Set when the worker thread has finished running. - pub(super) done: AtomicBool, - /// Set by the spawned closure after the worker entry has returned. - pub(super) thread_finished: AtomicBool, - /// Explicit worker lifecycle. Workerless rollback may transition only from - /// `NotStarted` to `Finished`. - pub(super) worker_lifecycle: std::sync::atomic::AtomicU8, - /// The final result from the worker thread (Ok or error). - pub(super) result: std::sync::Mutex>>, - /// The worker thread handle, taken during close to join. - pub(super) join_handle: std::sync::Mutex>>, - /// Waker registered by the close poll when the worker is still running. - pub(super) close_waker: AtomicWaker, - /// Waker registered by the scoped operation while the producer is still - /// running. - pub(super) quiescence_waker: AtomicWaker, - /// The permit is owned by shared stream state, so dropping the driver - /// cannot release admission while the worker or transport is alive. - pub(super) _permit: super::ConnectionPermit, - /// Set after rollback has retired both the operation and resource. - pub(super) rollback_finished: AtomicBool, -} - -impl SseShared { - fn mark_worker_running(&self) { - let _ = self.worker_lifecycle.compare_exchange( - SseWorkerLifecycle::NotStarted as u8, - SseWorkerLifecycle::Running as u8, - Ordering::AcqRel, - Ordering::Acquire, - ); - } - - fn mark_worker_finished(&self) { - self.thread_finished.store(true, Ordering::Release); - self.worker_lifecycle - .store(SseWorkerLifecycle::Finished as u8, Ordering::Release); - self.close_waker.wake(); - self.quiescence_waker.wake(); - } - - fn terminalize_workerless(&self, reason: OperationCancelReason, result: VmResult<()>) -> bool { - if self - .worker_lifecycle - .compare_exchange( - SseWorkerLifecycle::NotStarted as u8, - SseWorkerLifecycle::Finished as u8, - Ordering::AcqRel, - Ordering::Acquire, - ) - .is_err() - { - return false; - } - self.request_stop(reason); - self.publish(result); - self.thread_finished.store(true, Ordering::Release); - self.close_waker.wake(); - self.quiescence_waker.wake(); - true - } - - fn request_stop(&self, reason: OperationCancelReason) { - self.stopping.store(true, Ordering::Release); - let mut cancellation = self - .cancellation_reason - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - if cancellation.is_none() { - *cancellation = Some(reason); - } - self.cancel.notify_one(); - self.cancel.notify_waiters(); - self.waker.wake(); - self.close_waker.wake(); - self.quiescence_waker.wake(); - } - - fn publish(&self, result: VmResult<()>) { - *self - .result - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(result); - // The result mutex write happens-before this release publication. - self.done.store(true, Ordering::Release); - self.waker.wake(); - self.close_waker.wake(); - self.quiescence_waker.wake(); - } - - fn take_result(&self) -> Option> { - self.result - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .take() - } - - fn is_quiescent(&self) -> bool { - self.worker_lifecycle.load(Ordering::Acquire) == SseWorkerLifecycle::Finished as u8 - && self.done.load(Ordering::Acquire) - && self.thread_finished.load(Ordering::Acquire) - && self - .join_handle - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .is_none() - } - - fn join_worker(&self) -> Result<(), String> { - if self.worker_lifecycle.load(Ordering::Acquire) != SseWorkerLifecycle::Finished as u8 - || !self.done.load(Ordering::Acquire) - || !self.thread_finished.load(Ordering::Acquire) - { - return Err("SSE worker is still running".to_string()); - } - if self - .join_handle - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .as_ref() - .is_some_and(|handle| !handle.is_finished()) - { - return Err("SSE worker thread has not exited".to_string()); - } - let handle = self - .join_handle - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .take(); - let Some(handle) = handle else { - return Ok(()); - }; - handle.join().map_err(|panic| { - let message = worker_panic_message(&panic); - match self.cancellation_reason() { - Some(reason) => format!("{message} (cancellation reason: {reason})"), - None => message, - } - }) - } - - fn try_join_finished(&self) -> ResourceResult { - if self.worker_lifecycle.load(Ordering::Acquire) != SseWorkerLifecycle::Finished as u8 - || !self.done.load(Ordering::Acquire) - || !self.thread_finished.load(Ordering::Acquire) - { - return Ok(false); - } - let handle = { - let mut guard = self - .join_handle - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - if guard.as_ref().is_some_and(|handle| !handle.is_finished()) { - return Ok(false); - } - guard.take() - }; - let Some(handle) = handle else { - return Ok(true); - }; - handle - .join() - .map(|_| true) - .map_err(|panic| resource_cleanup_error(&worker_panic_message(&panic))) +fn prepare_sse_request(request: &VmMap, config: &super::HttpConfig) -> VmResult { + let mut request = parse_request(request, config)?; + if request.method != hyper::Method::GET && request.method != hyper::Method::POST { + return Err(VmError::HostError( + "SSE requests require GET or POST".to_string(), + )); } - - fn cancellation_reason(&self) -> Option { - self.cancellation_reason - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .as_ref() - .copied() + if !request + .headers + .iter() + .any(|(name, _)| name == hyper::header::ACCEPT) + { + request.headers.push(( + hyper::header::ACCEPT, + hyper::header::HeaderValue::from_static("text/event-stream"), + )); } + validate_request_header_budget(&request.headers, config)?; + Ok(request) } -fn worker_panic_message(panic: &Box) -> String { - if let Some(message) = panic.downcast_ref::<&str>() { - (*message).to_string() - } else if let Some(message) = panic.downcast_ref::() { - message.clone() - } else { - "SSE worker thread panicked".to_string() - } +/// Per-call state captured before the macro submits the async SSE future. +pub(super) struct SseRequestContext { + http: HttpRequestContext, + deadline: Instant, + request: HttpRequest, } -#[cfg(test)] -static FAIL_NEXT_WORKER_SPAWN: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); - -fn spawn_worker(name: &str, function: F) -> std::io::Result> -where - F: FnOnce() + Send + 'static, -{ - #[cfg(test)] - if FAIL_NEXT_WORKER_SPAWN.swap(false, Ordering::AcqRel) { - return Err(std::io::Error::other("injected SSE worker spawn failure")); +impl CaptureAsyncHostContext for SseRequestContext { + fn capture(_vm: &mut Vm) -> VmResult { + Err(VmError::HostError( + "SSE context requires call arguments".to_string(), + )) } - std::thread::Builder::new() - .name(name.to_string()) - .spawn(function) -} - -#[cfg(test)] -static REJECT_NEXT_OPERATION_ADMISSION: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); -#[allow(clippy::result_large_err)] -fn start_operation( - vm: &mut Vm, - operation: T, -) -> crate::vm::host_context::HostContextResult { - #[cfg(test)] - if REJECT_NEXT_OPERATION_ADMISSION.swap(false, Ordering::AcqRel) { - return Err(crate::vm::host_context::HostContextError::new( - "http::operation", - "injected operation admission rejection", - )); + fn capture_with_args(vm: &mut Vm, args: &[Value]) -> VmResult { + let request = match args.first() { + Some(Value::Map(request)) => request, + Some(_) => return Err(VmError::TypeMismatch("SSE request")), + None => return Err(VmError::HostError("missing SSE request".to_string())), + }; + let callback = args + .get(1) + .ok_or_else(|| VmError::HostError("missing SSE callback".to_string()))?; + vm.validate_sse_callback_value(callback)?; + let script_timeout = parse_stream_timeout(request)?; + let (http, deadline) = HttpRequestContext::capture_for(vm, script_timeout, "SSE")?; + let request = prepare_sse_request(request, &http.config)?; + Ok(Self { + http, + deadline, + request, + }) } - vm.host_context() - .start_operation(OperationSpec::new(operation)) } -fn resource_cleanup_error(message: &str) -> ResourceError { - ResourceError::new( - ResourceErrorCode::ResourceCleanupFailed, - "http::sse::resource", - message, - ) -} - -/// Runs the whole SSE lifecycle on a worker thread: open the response stream -/// (following redirects), validate it, read body frames, parse events and -/// publish each item into the shared completion channel. The guest callback -/// is invoked by the VM between items via the pending-result adapter. -struct SseWorker { - config: super::HttpConfig, - request: HttpRequest, - /// The one absolute stream deadline captured at stream admission. It is - /// passed unchanged through opening, redirects, body reads, and delivery. +/// Generic callable-stream continuation that owns only the Hyper response, +/// parser, callback-facing queue, deadlines, and the in-flight permit. +struct SseStreamDriver { + response: OwnedResponse, + parser: SseParser, + pending: VecDeque, + status: u16, + headers: Arc>, + url: String, + items: usize, + bytes_received: usize, deadline: Instant, - shared: Arc, - items: Arc, - bytes_received: Arc, - status: std::sync::Mutex>, - headers: std::sync::Mutex>>>, - url: std::sync::Mutex>, + total_sleep: Pin>, + idle_timeout: Duration, + idle_sleep: Pin>, + body_started: bool, + eof: bool, + _permit: super::policy::ConnectionPermit, } -impl SseWorker { - fn run(self: Arc) { - // The permit is held by shared stream state until cleanup completes. - let result = - match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.run_inner())) { - Ok(result) => result, - Err(panic) => Err(VmError::HostError(format!( - "SSE worker panicked: {}", - worker_panic_message(&panic) - ))), - }; - self.shared.publish(result); - } - - fn run_inner(&self) -> VmResult<()> { - // The entire SSE network lifecycle (open the response stream, then - // read every body frame) MUST run inside a single Tokio runtime. The - // owned response ties the hyper connection future and body receiver to - // one I/O driver; recreating a fresh current-thread runtime per frame - // moves a live socket across reactors and corrupts the body framing, - // surfacing hyper errors like "error reading a body from connection". - runtime_block_on(self.stream_lifecycle())? - } - - async fn stream_lifecycle(self: &SseWorker) -> VmResult<()> { - let mut parser = SseParser::new( - self.config.max_sse_line_bytes, - self.config.max_stream_item_bytes, - self.config.max_stream_total_bytes, - ); - let observer = ResponseReadObserver::default(); - - // The absolute deadline was captured before admission and is shared by - // every opening hop, body read, and callback publication. - let deadline = self.deadline; - - // Opening response headers must arrive before the earlier of the - // captured total deadline and this opening phase's idle boundary. - let opening_idle_deadline = - super::policy::phase_deadline(deadline, self.config.stream_idle_timeout); - let (mut response, url) = self - .open_response(observer.clone(), opening_idle_deadline) - .await?; +impl SseStreamDriver { + fn new( + response: OwnedResponse, + url: url::Url, + config: &super::HttpConfig, + deadline: Instant, + permit: super::policy::ConnectionPermit, + ) -> VmResult { let status = response.response().status(); if !status.is_success() { return Err(VmError::HostError(format!( @@ -704,7 +432,7 @@ impl SseWorker { status.as_u16() ))); } - let content_type = response + response .response() .headers() .get(hyper::header::CONTENT_TYPE) @@ -717,226 +445,38 @@ impl SseWorker { "SSE response Content-Type must be text/event-stream".to_string(), ) })?; - let _ = content_type; let headers = Arc::new(response_header_entries(response.response().headers())); - *self - .status - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(status.as_u16()); - *self - .headers - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::clone(&headers)); - *self - .url - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(url.to_string()); - observer.admit_body(self.config.max_stream_total_bytes); - self.publish( - sse_open_event(status.as_u16(), headers, url.as_str()), + let mut pending = VecDeque::new(); + pending.push_back(sse_open_event( + status.as_u16(), + Arc::clone(&headers), + url.as_str(), + )); + let total_at = tokio::time::Instant::from_std(deadline); + Ok(Self { + response, + parser: SseParser::new( + config.max_sse_line_bytes, + config.max_stream_item_bytes, + config.max_stream_total_bytes, + ), + pending, + status: status.as_u16(), + headers, + url: url.to_string(), + items: 0, + bytes_received: 0, deadline, - ) - .await?; - - // Body phase: every delivered frame resets the idle deadline, while - // the absolute total deadline is computed once and never reset by - // progress. - let mut idle_deadline = - super::policy::phase_deadline(deadline, self.config.stream_idle_timeout); - loop { - if self.shared.stopping.load(Ordering::SeqCst) { - return Err(VmError::HostError("SSE stream closed".to_string())); - } - let frame = self - .next_frame(&mut response, idle_deadline, deadline) - .await?; - let Some(frame) = frame else { - break; - }; - let Ok(data) = frame.into_data() else { - continue; - }; - // Any delivered body bytes count as progress: reset the idle - // deadline, but never touch the absolute total deadline. - idle_deadline = - super::policy::phase_deadline(deadline, self.config.stream_idle_timeout); - parser.admit_chunk(data.len())?; - observer.observe_application_chunk(data.len()); - self.bytes_received.fetch_add(data.len(), Ordering::SeqCst); - let mut offset = 0; - while offset < data.len() { - let (consumed, event) = parser.push_until_event(&data[offset..])?; - offset += consumed; - if let Some(event) = event { - self.items.fetch_add(1, Ordering::SeqCst); - self.publish(sse_data_event(event), deadline).await?; - } - } - } - parser.finish()?; - self.publish(sse_end_event(), deadline).await - } - - /// Opens the response stream with one absolute deadline shared by DNS, - /// connection setup, TLS, request/response headers, and every redirect. - /// The outer select also applies the opening idle phase limit; whichever - /// boundary is earlier wins. - /// - /// Cancellation is selected alongside the opening deadlines. Dropping the - /// whole opening future also drops every DNS/connect/TLS/header future and - /// every redirect hop owned by `open_stream_response`. - async fn open_response( - &self, - observer: ResponseReadObserver, - opening_idle_deadline: Instant, - ) -> VmResult<(OwnedResponse, url::Url)> { - if self.shared.stopping.load(Ordering::Acquire) { - return Err(VmError::HostError("SSE stream cancelled".to_string())); - } - tokio::select! { - biased; - _ = self.shared.cancel.notified() => { - Err(VmError::HostError("SSE stream cancelled".to_string())) - } - _ = tokio::time::sleep_until(tokio::time::Instant::from_std(opening_idle_deadline)) => { - if self.deadline <= opening_idle_deadline { - Err(VmError::HostError(SSE_TOTAL_DEADLINE_ERROR.to_string())) - } else { - Err(VmError::HostError( - "SSE stream idle timeout while opening response".to_string(), - )) - } - } - opened = open_stream_response( - &self.config, - &self.request, - observer, - self.deadline, - opening_idle_deadline, - ) => { - opened.map_err(|error| { - if error.to_string().contains("HTTP request deadline exceeded") { - let now = Instant::now(); - if now >= self.deadline { - VmError::HostError(SSE_TOTAL_DEADLINE_ERROR.to_string()) - } else if now >= opening_idle_deadline { - VmError::HostError( - "SSE stream idle timeout while opening response".to_string(), - ) - } else { - error - } - } else { - error - } - }) - } - } - } - - /// Reads one body frame bounded by cancel, the absolute total deadline and - /// the current idle deadline. Simultaneous boundary expiry is resolved - /// deterministically in favour of the total deadline. - async fn next_frame( - &self, - response: &mut OwnedResponse, - idle_deadline: Instant, - deadline: Instant, - ) -> VmResult>> { - if self.shared.stopping.load(Ordering::Acquire) { - return Err(VmError::HostError("SSE stream cancelled".to_string())); - } - let boundary = deadline.min(idle_deadline); - tokio::select! { - biased; - _ = self.shared.cancel.notified() => { - Err(VmError::HostError("SSE stream cancelled".to_string())) - } - _ = tokio::time::sleep_until(tokio::time::Instant::from_std(boundary)) => { - if deadline <= idle_deadline { - Err(VmError::HostError(SSE_TOTAL_DEADLINE_ERROR.to_string())) - } else { - Err(VmError::HostError("SSE stream idle timeout".to_string())) - } - } - frame = response.next_frame() => frame, - } + total_sleep: Box::pin(tokio::time::sleep_until(total_at)), + idle_timeout: config.stream_idle_timeout, + // Opening callback time is excluded from the first body idle window. + idle_sleep: Box::pin(tokio::time::sleep_until(total_at)), + body_started: false, + eof: false, + _permit: permit, + }) } - /// Publishes one item into the bounded FIFO with backpressure. The send is - /// bounded by cancel and the absolute total deadline, so a stalled - /// callback or full queue cannot extend the stream past its deadline. - /// Wakes the stream driver's waker so the VM re-polls and drains the item. - async fn publish(&self, item: Value, deadline: Instant) -> VmResult<()> { - if self.shared.stopping.load(Ordering::Acquire) { - return Err(VmError::HostError("SSE stream cancelled".to_string())); - } - let sender = &self.shared.items; - tokio::select! { - biased; - _ = self.shared.cancel.notified() => { - Err(VmError::HostError("SSE stream cancelled".to_string())) - } - _ = tokio::time::sleep_until(tokio::time::Instant::from_std(deadline)) => { - Err(VmError::HostError(SSE_TOTAL_DEADLINE_ERROR.to_string())) - } - sent = sender.send(item) => { - sent.map_err(|_| VmError::HostError("SSE stream closed".to_string()))?; - self.shared.waker.wake(); - if self.shared.stopping.load(Ordering::Acquire) { - return Err(VmError::HostError("SSE stream cancelled".to_string())); - } - tokio::select! { - biased; - _ = self.shared.cancel.notified() => { - Err(VmError::HostError("SSE stream cancelled".to_string())) - } - _ = tokio::time::sleep_until(tokio::time::Instant::from_std(deadline)) => { - Err(VmError::HostError(SSE_TOTAL_DEADLINE_ERROR.to_string())) - } - _ = self.shared.item_ack.notified() => Ok(()), - } - } - } - } -} - -fn runtime_block_on(future: F) -> VmResult { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| VmError::HostError(format!("SSE worker runtime build failed: {error}")))?; - Ok(runtime.block_on(future)) -} - -/// Stream driver for the SSE stream: the VM's async host polls this driver -/// through [`submit_callable_stream`] for each item, then invokes the script -/// callback and calls [`apply_action`](Self::apply_action) with the result. -struct SseStreamDriver { - shared: Arc, - /// Bounded FIFO receiver for items published by the worker. - receiver: mpsc::Receiver, - status: u16, - headers: Arc>, - url: String, - items: usize, - bytes_received: Arc, - /// The absolute total deadline; the driver enforces it in - /// [`apply_action`](Self::apply_action) so a slow callback cannot extend - /// the stream past its deadline. - deadline: Instant, - scope_operation: crate::vm::operation::OperationId, - resource: ResourceHandle, - termination: Option, -} - -struct SseTerminationState { - operation_done: bool, - resource_done: bool, - first_error: Option, -} - -impl SseStreamDriver { fn summary(&self, outcome: &str) -> Value { map_value(vec![ ("outcome", Value::string(outcome)), @@ -944,177 +484,97 @@ impl SseStreamDriver { ("headers", Value::Array(Arc::clone(&self.headers))), ("url", Value::string(&self.url)), ("items", Value::Int(self.items as i64)), - ( - "bytes_received", - Value::Int(self.bytes_received.load(Ordering::Acquire) as i64), - ), + ("bytes_received", Value::Int(self.bytes_received as i64)), ("bytes_sent", Value::Int(0)), ]) } -} -impl HostStreamDriver for SseStreamDriver { - fn acknowledge_item(&mut self) { - self.shared.item_ack.notify_one(); + fn reset_idle_deadline(&mut self) { + let idle_at = policy::phase_deadline(self.deadline, self.idle_timeout); + self.idle_sleep + .as_mut() + .reset(tokio::time::Instant::from_std(idle_at)); } - fn terminate( - &mut self, - scope: &mut ExecutionScope, - termination: HostStreamTermination, - ) -> VmResult<()> { - self.begin_termination(scope, termination)?; - let waker = std::task::Waker::noop(); - let mut cx = Context::from_waker(waker); - match self.poll_termination(scope, termination, &mut cx) { - Poll::Ready(result) => result, - Poll::Pending => Err(VmError::HostError( - "SSE stream termination is still pending".to_string(), - )), - } + fn pop_item(&mut self) -> Option { + self.pending.pop_front().map(|item| { + self.items = self.items.saturating_add(1); + HostStreamPoll::Item(item) + }) } - fn begin_termination( - &mut self, - scope: &mut ExecutionScope, - termination: HostStreamTermination, - ) -> VmResult<()> { - if self.termination.is_some() { - return Ok(()); - } - if let HostStreamTermination::Cancelled(reason) = termination { - self.shared.request_stop(reason); + fn queue_data(&mut self, data: &[u8]) -> VmResult<()> { + self.parser.admit_chunk(data.len())?; + self.bytes_received = self.bytes_received.saturating_add(data.len()); + let mut offset = 0; + while offset < data.len() { + let (consumed, event) = self.parser.push_until_event(&data[offset..])?; + offset += consumed; + if let Some(event) = event { + self.pending.push_back(sse_data_event(event)); + } } - match termination { - HostStreamTermination::Completed => scope - .complete_operation(self.scope_operation) - .map_err(VmError::ExecutionScope)?, - HostStreamTermination::Cancelled(reason) => scope - .cancel_operation(self.scope_operation, reason) - .map_err(VmError::ExecutionScope)?, - }; - let resource_reason = sse_resource_close_reason(termination); - let resource_done = match scope - .close_resource::(self.resource, resource_reason) - .map_err(VmError::ExecutionScope)? - { - CloseProgress::Ready => true, - CloseProgress::Pending => false, - }; - self.termination = Some(SseTerminationState { - operation_done: false, - resource_done, - first_error: None, - }); Ok(()) } +} + +impl HostStreamDriver for SseStreamDriver { + fn acknowledge_item(&mut self) { + if !self.body_started { + self.body_started = true; + self.reset_idle_deadline(); + } + } - fn poll_termination( - &mut self, - scope: &mut ExecutionScope, - _termination: HostStreamTermination, - cx: &mut Context<'_>, - ) -> Poll> { - let Some(state) = self.termination.as_mut() else { + fn poll_next(&mut self, cx: &mut Context<'_>) -> Poll> { + if self.total_sleep.as_mut().poll(cx).is_ready() { return Poll::Ready(Err(VmError::HostError( - "SSE stream termination was not started".to_string(), + SSE_TOTAL_DEADLINE_ERROR.to_string(), ))); - }; - if !state.operation_done { - match scope.poll_operation_quiescence(self.scope_operation, cx) { - Poll::Pending => {} - Poll::Ready(Ok(_)) => state.operation_done = true, - Poll::Ready(Err(error)) => { - state.operation_done = true; - if state.first_error.is_none() { - state.first_error = Some(VmError::ExecutionScope(error)); - } - } - } } - if !state.resource_done { - match scope.poll_resource_close::(self.resource, cx) { - Poll::Pending => {} - Poll::Ready(Ok(())) => state.resource_done = true, - Poll::Ready(Err(ExecutionScopeError::Resource(error))) - if error.code() == ResourceErrorCode::ResourceAlreadyClosed => - { - state.resource_done = true; - } - Poll::Ready(Err(error)) => { - state.resource_done = true; - if state.first_error.is_none() { - state.first_error = Some(VmError::ExecutionScope(error)); - } - } - } + if let Some(item) = self.pop_item() { + return Poll::Ready(Ok(item)); } - if state.operation_done && state.resource_done { - let state = self.termination.take().expect("termination state exists"); - match state.first_error { - Some(error) => Poll::Ready(Err(error)), - None => Poll::Ready(Ok(())), - } - } else { - Poll::Pending + if self.eof { + return Poll::Ready(Ok(HostStreamPoll::Complete(self.summary("eof")))); + } + if self.body_started && self.idle_sleep.as_mut().poll(cx).is_ready() { + return Poll::Ready(Err(VmError::HostError( + "SSE stream idle timeout".to_string(), + ))); } - } - fn poll_next(&mut self, cx: &mut Context<'_>) -> Poll> { - match self.receiver.poll_recv(cx) { - Poll::Ready(Some(item)) => { - // Track items and capture metadata from the open item. - if let Value::Map(ref map) = item { - match map.get(&Value::string("kind")) { - Some(Value::String(kind)) if kind.as_str() == "open" => { - if let Some(Value::Int(status)) = map.get(&Value::string("status")) { - self.status = *status as u16; - } - if let Some(Value::Array(headers)) = map.get(&Value::string("headers")) - { - self.headers = Arc::clone(headers); - } - if let Some(Value::String(url)) = map.get(&Value::string("url")) { - self.url = url.as_ref().clone(); - } - } - _ => {} - } - } - self.items = self.items.saturating_add(1); - Poll::Ready(Ok(HostStreamPoll::Item(item))) - } - Poll::Ready(None) => self.poll_terminal("eof"), - Poll::Pending => { - if self.shared.done.load(Ordering::Acquire) { - return self.poll_terminal(if self.shared.stopping.load(Ordering::Acquire) { - "stopped" - } else { - "eof" - }); + match self.response.poll_next_frame(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(Err(error)) => Poll::Ready(Err(error)), + Poll::Ready(Ok(Some(frame))) => { + let Ok(data) = frame.into_data() else { + cx.waker().wake_by_ref(); + return Poll::Pending; + }; + self.reset_idle_deadline(); + if let Err(error) = self.queue_data(&data) { + return Poll::Ready(Err(error)); } - self.shared.waker.register(cx.waker()); - if self.shared.done.load(Ordering::Acquire) { - self.poll_terminal(if self.shared.stopping.load(Ordering::Acquire) { - "stopped" - } else { - "eof" - }) + if let Some(item) = self.pop_item() { + Poll::Ready(Ok(item)) } else { - // A stop is terminal only after the worker publishes its - // result. The stop notification wakes this poll through - // the atomic waker while the worker is still unwinding. + cx.waker().wake_by_ref(); Poll::Pending } } + Poll::Ready(Ok(None)) => { + if let Err(error) = self.parser.finish() { + return Poll::Ready(Err(error)); + } + self.eof = true; + self.pending.push_back(sse_end_event()); + Poll::Ready(Ok(self.pop_item().expect("end item was queued"))) + } } } fn apply_action(&mut self, action: Value) -> VmResult { - // The absolute total deadline is enforced here too: a slow callback - // (e.g. one awaiting a host future) must not extend the stream past - // its deadline. Once the deadline has passed, every callback action - // fails deterministically. if Instant::now() >= self.deadline { return Err(VmError::HostError(SSE_TOTAL_DEADLINE_ERROR.to_string())); } @@ -1141,366 +601,78 @@ impl HostStreamDriver for SseStreamDriver { } } -impl SseStreamDriver { - fn poll_terminal(&mut self, outcome: &str) -> Poll> { - match self.shared.take_result() { - Some(Ok(())) => Poll::Ready(Ok(HostStreamPoll::Complete(self.summary(outcome)))), - Some(Err(error)) => Poll::Ready(Err(error)), - None => Poll::Ready(Err(VmError::HostError( - "SSE worker completed without a terminal result".to_string(), - ))), - } - } -} - -fn sse_resource_close_reason(termination: HostStreamTermination) -> ResourceCloseReason { - match termination { - HostStreamTermination::Completed => ResourceCloseReason::ResourceClosed, - HostStreamTermination::Cancelled(reason) => match reason { - OperationCancelReason::Requested => ResourceCloseReason::Requested, - OperationCancelReason::Deadline => ResourceCloseReason::Deadline, - OperationCancelReason::VmReset => ResourceCloseReason::VmReset, - OperationCancelReason::Parent => ResourceCloseReason::Parent, - OperationCancelReason::ResourceClosed => ResourceCloseReason::ResourceClosed, - OperationCancelReason::VmDrop => ResourceCloseReason::VmDrop, - }, - } -} - -fn close_sse_resource(vm: &mut Vm, resource: ResourceHandle) -> VmResult<()> { - let progress = vm - .host_context() - .close_resource::(resource, ResourceCloseReason::ResourceClosed) - .map_err(|error| VmError::HostError(format!("failed to close SSE resource: {error}")))?; - match progress { - CloseProgress::Ready => Ok(()), - CloseProgress::Pending => Err(VmError::HostError( - "SSE resource close remained pending after producer retirement".to_string(), - )), - } -} - -fn rollback_sse_admission( - vm: &mut Vm, - shared: &Arc, - resource: ResourceHandle, - operation: Option, - primary: VmError, -) -> VmError { - if shared.rollback_finished.load(Ordering::Acquire) { - return primary; - } - let mut cleanup_errors = Vec::new(); - let _ = shared.terminalize_workerless( - OperationCancelReason::Requested, - Err(VmError::HostError("SSE worker was not started".to_string())), - ); - if let Some(operation) = operation - && let Err(error) = vm - .host_context() - .abort_operation(operation, OperationCancelReason::Requested) - { - cleanup_errors.push(VmError::HostError(format!( - "failed to abort SSE operation: {error}" - ))); - } - if let Err(error) = shared.join_worker() { - cleanup_errors.push(VmError::HostError(format!( - "failed to join SSE worker: {error}" - ))); - } - if let Err(error) = close_sse_resource(vm, resource) { - cleanup_errors.push(error); - } - if cleanup_errors.is_empty() { - shared.rollback_finished.store(true, Ordering::Release); - } - cleanup_errors - .into_iter() - .fold(primary, |primary, cleanup| { - crate::vm::async_host::preserve_stream_cleanup(primary, Err(cleanup)) - }) -} - -/// The SSE stream reader registered as a child resource in the execution -/// scope. Closing it via the scope lifecycle sets `stopping` on the shared -/// state, which the worker observes between items and stops promptly. -pub(crate) struct SseStreamResource { - shared: Arc, -} - -impl crate::host_extension::HostResourceType for SseStreamResource { - const KEY: &'static str = "http.sse"; - const DESCRIPTION: &'static str = - "An incremental SSE stream reader over an open response body stream"; -} - -impl HostResource for SseStreamResource { - fn resource_type_key() -> Option { - ResourceTypeKey::new("http.sse").ok() - } - - fn begin_close(&mut self, reason: ResourceCloseReason) -> ResourceResult { - self.shared.request_stop(match reason { - ResourceCloseReason::Requested => OperationCancelReason::Requested, - ResourceCloseReason::Deadline => OperationCancelReason::Deadline, - ResourceCloseReason::VmReset => OperationCancelReason::VmReset, - ResourceCloseReason::Parent => OperationCancelReason::Parent, - ResourceCloseReason::ResourceClosed => OperationCancelReason::ResourceClosed, - ResourceCloseReason::VmDrop => OperationCancelReason::VmDrop, - }); - match self.shared.try_join_finished()? { - true => Ok(CloseProgress::Ready), - false => Ok(CloseProgress::Pending), - } - } - - fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll> { - match self.shared.try_join_finished() { - Ok(true) => Poll::Ready(Ok(())), - Ok(false) => { - self.shared.close_waker.register(cx.waker()); - match self.shared.try_join_finished() { - Ok(true) => Poll::Ready(Ok(())), - Ok(false) => Poll::Pending, - Err(error) => Poll::Ready(Err(error)), +async fn open_sse_response( + context: &SseRequestContext, + request: &HttpRequest, +) -> VmResult<(OwnedResponse, url::Url)> { + let opening_idle_deadline = + policy::phase_deadline(context.deadline, context.http.config.stream_idle_timeout); + tokio::select! { + biased; + _ = tokio::time::sleep_until(tokio::time::Instant::from_std(opening_idle_deadline)) => { + if context.deadline <= opening_idle_deadline { + Err(VmError::HostError(SSE_TOTAL_DEADLINE_ERROR.to_string())) + } else { + Err(VmError::HostError( + "SSE stream idle timeout while opening response".to_string(), + )) + } + } + opened = open_stream_response( + &context.http.client, + &context.http.config, + request, + context.deadline, + ) => opened.map_err(|error| { + if error.to_string().contains("HTTP request deadline exceeded") { + let now = Instant::now(); + if now >= context.deadline { + VmError::HostError(SSE_TOTAL_DEADLINE_ERROR.to_string()) + } else if now >= opening_idle_deadline { + VmError::HostError( + "SSE stream idle timeout while opening response".to_string(), + ) + } else { + error } - } - Err(error) => Poll::Ready(Err(error)), - } - } -} - -/// Scope operation that tracks the pending SSE network poll. Cancel sets -/// `stopping` on the shared state so the worker stops promptly. The actual -/// item delivery is driven by the `SseStreamDriver` through the callable -/// stream path; this operation exists only for scope lifecycle management. -pub(super) struct SseScopeOperation { - shared: Arc, -} - -impl HostOperation for SseScopeOperation { - fn poll(&mut self, _cx: &mut Context<'_>) -> Poll> { - if self.shared.done.load(Ordering::SeqCst) { - Poll::Ready(Ok(())) - } else { - Poll::Pending - } - } - - fn cancel(&mut self, reason: OperationCancelReason) -> OperationResult<()> { - self.shared.request_stop(reason); - Ok(()) - } - - fn cancel_and_wait(&mut self, reason: OperationCancelReason) -> OperationResult<()> { - self.shared.request_stop(reason); - if !self.shared.is_quiescent() { - return Err(OperationError::new( - OperationErrorCode::OperationDriverFailed, - "http::sse", - "SSE worker cancellation is still pending", - )); - } - self.shared.join_worker().map_err(|message| { - OperationError::new( - OperationErrorCode::OperationDriverFailed, - "http::sse", - message, - ) - }) - } - - fn is_quiescent(&self) -> bool { - self.shared.is_quiescent() - } - - fn register_quiescence_waker(&mut self, cx: &Context<'_>) { - self.shared.quiescence_waker.register(cx.waker()); - } - - fn poll_quiescent(&mut self, cx: &mut Context<'_>) -> Poll<()> { - if self.shared.is_quiescent() { - return Poll::Ready(()); - } - self.shared.waker.register(cx.waker()); - self.shared.close_waker.register(cx.waker()); - self.shared.quiescence_waker.register(cx.waker()); - if self.shared.is_quiescent() { - Poll::Ready(()) - } else { - let _ = self.shared.try_join_finished(); - if self.shared.is_quiescent() { - Poll::Ready(()) } else { - Poll::Pending + error } - } + }) } } -/// Streams one bounded SSE item into one script callback at a time. -#[pd_host_function(name = "http::client::sse", contract = super::http_sse_contract, runtime_owned_pending)] -pub(super) fn builtin_http_client_sse( - vm: &mut Vm, +/// Opens an SSE response with the shared client, then transfers the response +/// body into the VM's generic callable-stream continuation. +#[pd_host_function(name = "http::client::sse", contract = super::http_sse_contract)] +pub(super) async fn builtin_http_client_sse( + #[pd_host_context] context: SseRequestContext, request: VmMapHandle, on_event: VmCallable VmMap>, -) -> VmResult> { +) -> VmResult> { let callback = on_event.into_value(); - vm.validate_sse_callback_value(&callback)?; - let script_timeout = parse_stream_timeout(&request)?; - let (context, deadline) = HttpRequestContext::capture(vm, script_timeout, "SSE")?; - let mut request = parse_request(&request, &context.config)?; - policy::validate_url_policy(&context.config, policy::SchemeFamily::Http, &request.url)?; - if request.method != hyper::Method::GET && request.method != hyper::Method::POST { - return Err(VmError::HostError( - "SSE requests require GET or POST".to_string(), - )); - } - if !request - .headers - .iter() - .any(|(name, _)| name == hyper::header::ACCEPT) - { - request.headers.push(( - hyper::header::ACCEPT, - hyper::header::HeaderValue::from_static("text/event-stream"), - )); - } - validate_request_header_budget(&request.headers, &context.config)?; - - let config = context.config.clone(); - let permit = context.into_permit(); - let (items, receiver) = mpsc::channel(SSE_CHANNEL_CAPACITY); - let shared = Arc::new(SseShared { - stopping: AtomicBool::new(false), - cancel: Notify::new(), - cancellation_reason: std::sync::Mutex::new(None), - item_ack: Notify::new(), - waker: AtomicWaker::new(), - items, - done: AtomicBool::new(false), - thread_finished: AtomicBool::new(false), - worker_lifecycle: std::sync::atomic::AtomicU8::new(SseWorkerLifecycle::NotStarted as u8), - result: std::sync::Mutex::new(None), - join_handle: std::sync::Mutex::new(None), - close_waker: AtomicWaker::new(), - quiescence_waker: AtomicWaker::new(), - _permit: permit, - rollback_finished: AtomicBool::new(false), - }); - - // The SSE stream itself is a typed scope resource. The underlying response - // is owned by the stream worker and is closed after producer quiescence. - let sse_token = vm - .host_context() - .push_resource(SseStreamResource { - shared: Arc::clone(&shared), - }) - .map_err(|error| { - VmError::HostError(format!("failed to push SSE child resource: {error}")) - })?; - let resource = sse_token.handle(); - let op = SseScopeOperation { - shared: Arc::clone(&shared), - }; - let scope_operation = match start_operation(vm, op) { - Ok(operation) => operation, - Err(error) => { - return Err(rollback_sse_admission( - vm, - &shared, - resource, - None, - VmError::HostError(format!("failed to start SSE operation: {error}")), - )); - } - }; - - let worker = Arc::new(SseWorker { - config: config.clone(), - request, + let _ = request; + let (response, url) = open_sse_response(&context, &context.request).await?; + let SseRequestContext { + http, deadline, - shared: Arc::clone(&shared), - items: Arc::new(AtomicUsize::new(0)), - bytes_received: Arc::new(AtomicUsize::new(0)), - status: std::sync::Mutex::new(None), - headers: std::sync::Mutex::new(None), - url: std::sync::Mutex::new(None), - }); - let bytes_received = worker.bytes_received.clone(); - - let join_handle = match spawn_worker("rustscript-sse-worker", { - let worker_shared = Arc::clone(&shared); - move || { - worker_shared.mark_worker_running(); - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - worker.run(); - })); - if let Err(panic) = result { - worker_shared.publish(Err(VmError::HostError(format!( - "SSE worker panicked: {}", - worker_panic_message(&panic) - )))); - } - worker_shared.mark_worker_finished(); - worker_shared.waker.wake(); - } - }) { - Ok(handle) => handle, - Err(error) => { - return Err(rollback_sse_admission( - vm, - &shared, - resource, - Some(scope_operation), - VmError::HostError(format!("failed to start SSE worker: {error}")), - )); + request: _, + } = context; + let driver = SseStreamDriver::new(response, url, &http.config, deadline, http.permit)?; + Ok(HostFutureOutput::continue_with(move |vm| { + match vm.submit_callable_stream(callback, driver) { + Ok(CallOutcome::Pending(op_id)) => Ok(CallOutcome::Pending(op_id)), + Ok(_) => Err(VmError::InvalidFrameState( + "callable stream admission returned a non-pending outcome", + )), + Err(rejection) => Err(vm.rollback_rejected_callable_stream(rejection)), } - }; - shared.mark_worker_running(); - *shared - .join_handle - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(join_handle); - - let driver = SseStreamDriver { - shared: Arc::clone(&shared), - receiver, - status: 0, - headers: Arc::new(Vec::new()), - url: String::new(), - items: 0, - bytes_received, - deadline, - scope_operation, - resource, - termination: None, - }; - - match vm.submit_callable_stream(callback, driver) { - Ok(CallOutcome::Pending(op_id)) => Ok(HostCallResult::Pending(op_id)), - Ok(_) => Err(rollback_sse_admission( - vm, - &shared, - resource, - Some(scope_operation), - VmError::InvalidFrameState("callable stream admission returned a non-pending outcome"), - )), - Err(rejection) => Err(vm.rollback_rejected_callable_stream(rejection)), - } + })) } #[cfg(test)] mod tests { - use std::sync::atomic::{AtomicBool, Ordering}; - - use super::{ - FAIL_NEXT_WORKER_SPAWN, REJECT_NEXT_OPERATION_ADMISSION, SseEvent, SseParser, - SseScopeOperation, SseShared, SseStreamResource, SseWorkerLifecycle, - rollback_sse_admission, spawn_worker, start_operation, - }; - use crate::vm::VmError; + use super::{SseEvent, SseParser}; fn event(data: &str, event: Option<&str>, id: Option<&str>, retry_ms: Option) -> SseEvent { SseEvent { @@ -1725,88 +897,4 @@ mod tests { ] ); } - - #[test] - fn admission_rollback_reclaims_workerless_sse_resource() { - let mut vm = crate::vm::Vm::new(crate::vm::Program::new( - Vec::new(), - vec![crate::vm::OpCode::Ret as u8], - )); - let permit = crate::builtins::runtime::http::policy::ConnectionAdmission::new(1) - .acquire() - .expect("test permit"); - let (items, _receiver) = tokio::sync::mpsc::channel(1); - let shared = std::sync::Arc::new(SseShared { - stopping: AtomicBool::new(false), - cancel: tokio::sync::Notify::new(), - cancellation_reason: std::sync::Mutex::new(None), - item_ack: tokio::sync::Notify::new(), - waker: futures_util::task::AtomicWaker::new(), - items, - done: AtomicBool::new(false), - thread_finished: AtomicBool::new(false), - worker_lifecycle: std::sync::atomic::AtomicU8::new( - SseWorkerLifecycle::NotStarted as u8, - ), - result: std::sync::Mutex::new(None), - join_handle: std::sync::Mutex::new(None), - close_waker: futures_util::task::AtomicWaker::new(), - quiescence_waker: futures_util::task::AtomicWaker::new(), - _permit: permit, - rollback_finished: AtomicBool::new(false), - }); - let token = vm - .execution_scope() - .push_resource(SseStreamResource { - shared: std::sync::Arc::clone(&shared), - }) - .expect("SSE resource"); - let primary = crate::vm::VmError::HostError("operation admission rejected".to_string()); - REJECT_NEXT_OPERATION_ADMISSION.store(true, Ordering::Release); - let admission = start_operation( - &mut vm, - SseScopeOperation { - shared: std::sync::Arc::clone(&shared), - }, - ); - assert!(admission.is_err()); - - let error = rollback_sse_admission(&mut vm, &shared, token.handle(), None, primary); - - assert!(error.to_string().contains("operation admission rejected")); - assert_eq!(vm.execution_scope().resources().len(), 0); - assert_eq!( - shared.worker_lifecycle.load(Ordering::Acquire), - SseWorkerLifecycle::Finished as u8 - ); - assert!(shared.done.load(Ordering::Acquire)); - assert!(shared.thread_finished.load(Ordering::Acquire)); - - let repeated = rollback_sse_admission( - &mut vm, - &shared, - token.handle(), - None, - VmError::HostError("repeated rollback".to_string()), - ); - assert!(repeated.to_string().contains("repeated rollback")); - } - - #[test] - fn worker_spawn_abstraction_can_inject_a_builder_failure() { - FAIL_NEXT_WORKER_SPAWN.store(true, Ordering::Release); - let result = spawn_worker("injected-sse-worker", || {}); - let error = match result { - Ok(handle) => { - handle.join().expect("unexpected worker"); - panic!("spawn should have been rejected") - } - Err(error) => error, - }; - assert!( - error - .to_string() - .contains("injected SSE worker spawn failure") - ); - } } diff --git a/src/builtins/runtime/mod.rs b/src/builtins/runtime/mod.rs index 2ab70d48..f26d60c5 100644 --- a/src/builtins/runtime/mod.rs +++ b/src/builtins/runtime/mod.rs @@ -5,10 +5,10 @@ use std::sync::{Arc, OnceLock}; use crate::builtins::BuiltinFunction; use crate::host_api::{HostApiCatalog, HostApiFingerprint}; -#[cfg(all(feature = "async", not(target_family = "wasm")))] -use crate::vm::CaptureAsyncHostContext; #[allow(unused_imports)] use crate::vm::{CallOutcome, CallReturn, HostOpId, Value, Vm, VmError, VmResult}; +#[cfg(all(feature = "async", not(target_family = "wasm")))] +use crate::vm::{CaptureAsyncHostContext, HostFutureOutput}; mod aot; mod bytes; diff --git a/src/vm/async_host/mod.rs b/src/vm/async_host/mod.rs index bb7bf8c8..8e90fd00 100644 --- a/src/vm/async_host/mod.rs +++ b/src/vm/async_host/mod.rs @@ -36,14 +36,32 @@ pub(crate) use stream::{ /// future has resolved. pub type HostVmCompletion = Box VmResult + Send + 'static>; +/// A continuation installed after the async phase has resolved. +/// +/// This is the narrow escape hatch for host calls that must transfer from a +/// library future into a generic VM-owned continuation such as a callable +/// stream. The continuation runs once on the VM thread and may return a new +/// pending operation without letting the async future borrow or re-enter the +/// VM while it is being polled. +pub type HostVmContinuation = Box VmResult + Send + 'static>; + +/// The result of resolving a [`HostFutureOutput`] against the VM. +#[derive(Debug)] +pub(crate) enum HostFutureResolution { + Return(T), + Continue(CallOutcome), +} + /// The terminal result of a submitted async host call. /// /// `T` is the value produced without further VM access (`Return`), or the /// value produced by a completion closure that borrows the VM once -/// (`VmCompletion`). +/// (`VmCompletion`). `VmContinuation` transfers the call into a generic +/// VM-owned pending continuation after the library future has completed. pub enum HostFutureOutput { Return(T), VmCompletion(HostVmCompletion), + VmContinuation(HostVmContinuation), } impl HostFutureOutput { @@ -58,8 +76,18 @@ impl HostFutureOutput { Self::VmCompletion(Box::new(completion)) } + /// Wraps a continuation that runs once on the VM thread after the async + /// phase and may transfer the host call to another generic pending driver. + pub fn continue_with( + continuation: impl FnOnce(&mut Vm) -> VmResult + Send + 'static, + ) -> Self { + Self::VmContinuation(Box::new(continuation)) + } + /// Maps the produced value through `map`, deferring the mapping until - /// the completion closure (if any) has run against the VM. + /// the completion closure (if any) has run against the VM. A VM + /// continuation already returns a call-level outcome and passes through + /// unchanged. pub fn map( self, map: impl FnOnce(T) -> U + Send + 'static, @@ -72,17 +100,22 @@ impl HostFutureOutput { Self::VmCompletion(completion) => { HostFutureOutput::VmCompletion(Box::new(move |vm| completion(vm).map(map))) } + Self::VmContinuation(continuation) => HostFutureOutput::VmContinuation(continuation), } } } impl HostFutureOutput { /// Resolves the terminal output against the VM: a `Return` value is - /// returned directly; a `VmCompletion` closure runs with `&mut Vm`. - pub(crate) fn finish(self, vm: &mut Vm) -> VmResult { + /// returned directly, a `VmCompletion` closure runs with `&mut Vm`, and a + /// `VmContinuation` produces the next call-level outcome. + pub(crate) fn finish(self, vm: &mut Vm) -> VmResult { match self { - Self::Return(values) => Ok(values), - Self::VmCompletion(completion) => completion(vm), + Self::Return(values) => Ok(HostFutureResolution::Return(values)), + Self::VmCompletion(completion) => completion(vm).map(HostFutureResolution::Return), + Self::VmContinuation(continuation) => { + continuation(vm).map(HostFutureResolution::Continue) + } } } } diff --git a/src/vm/host.rs b/src/vm/host.rs index 68f0386b..7c83629d 100644 --- a/src/vm/host.rs +++ b/src/vm/host.rs @@ -11,7 +11,9 @@ use crate::vm::resource::error::{ResourceError, ResourceErrorCode}; use crate::vm::resource::handle::ResourceHandle; use crate::vm::resource::table::ResourceTable; -use super::async_host::{HostFuture, HostFutureOutput, preserve_stream_cleanup}; +use super::async_host::{ + HostFuture, HostFutureOutput, HostFutureResolution, preserve_stream_cleanup, +}; use super::capability::CapabilityProfile; use super::*; @@ -3130,8 +3132,8 @@ impl Vm { match poll_result { Poll::Pending => Poll::Pending, Poll::Ready(Ok(output)) => { - let values = match output.finish(self) { - Ok(values) => values, + let resolution = match output.finish(self) { + Ok(resolution) => resolution, Err(err) => { if bridge_owned { let cleanup = self.host.complete_bridge_operation( @@ -3147,36 +3149,117 @@ impl Vm { return Poll::Ready(Err(err)); } }; - if bridge_owned { - let validation = validate_host_call_return( - &values, - waiting.expected_return_type, - waiting.expected_return_schema.as_ref(), - &self.program, - self.host.execution_scope.resources(), - ); - if let Err(error) = validation { - let cleanup = self - .host - .complete_bridge_operation(waiting.op_id, HostAsyncOpTerminal::Failed); - self.instance.waiting_host_op = None; - return Poll::Ready(Err(cleanup.err().unwrap_or(error))); - } - if let Err(error) = self - .host - .complete_bridge_operation(waiting.op_id, HostAsyncOpTerminal::Completed) - { - self.instance.waiting_host_op = None; - return Poll::Ready(Err(error)); + match resolution { + HostFutureResolution::Return(values) => { + if bridge_owned { + let validation = validate_host_call_return( + &values, + waiting.expected_return_type, + waiting.expected_return_schema.as_ref(), + &self.program, + self.host.execution_scope.resources(), + ); + if let Err(error) = validation { + let cleanup = self.host.complete_bridge_operation( + waiting.op_id, + HostAsyncOpTerminal::Failed, + ); + self.instance.waiting_host_op = None; + return Poll::Ready(Err(cleanup.err().unwrap_or(error))); + } + if let Err(error) = self.host.complete_bridge_operation( + waiting.op_id, + HostAsyncOpTerminal::Completed, + ) { + self.instance.waiting_host_op = None; + return Poll::Ready(Err(error)); + } + self.instance.waiting_host_op = None; + values.push_onto_stack(&mut self.instance.stack); + return Poll::Ready(Ok(())); + } + if let Err(error) = self.complete_waiting_host_op(waiting.op_id, values) { + return Poll::Ready(Err(error)); + } + Poll::Ready(Ok(())) } - self.instance.waiting_host_op = None; - values.push_onto_stack(&mut self.instance.stack); - return Poll::Ready(Ok(())); - } - if let Err(error) = self.complete_waiting_host_op(waiting.op_id, values) { - return Poll::Ready(Err(error)); + HostFutureResolution::Continue(outcome) => match outcome { + CallOutcome::Return(values) => { + let validation = validate_host_call_return( + &values, + waiting.expected_return_type, + waiting.expected_return_schema.as_ref(), + &self.program, + self.host.execution_scope.resources(), + ); + let terminal = if validation.is_ok() { + HostAsyncOpTerminal::Completed + } else { + HostAsyncOpTerminal::Failed + }; + if bridge_owned + && let Err(error) = + self.host.complete_bridge_operation(waiting.op_id, terminal) + { + self.instance.waiting_host_op = None; + return Poll::Ready(Err(error)); + } + self.instance.waiting_host_op = None; + if let Err(error) = validation { + return Poll::Ready(Err(error)); + } + values.push_onto_stack(&mut self.instance.stack); + Poll::Ready(Ok(())) + } + CallOutcome::Pending(op_id) => { + if bridge_owned + && let Err(error) = self.host.complete_bridge_operation( + waiting.op_id, + HostAsyncOpTerminal::Completed, + ) + { + self.instance.waiting_host_op = None; + return Poll::Ready(Err(error)); + } + self.instance.waiting_host_op = None; + let source = self.host_call_pending_source(op_id); + if let Err(error) = self.set_waiting_host_op_with_return( + op_id, + source, + waiting.expected_return_type, + waiting.expected_return_schema.as_ref(), + ) { + let cleanup = + if matches!(source, WaitingHostOpSource::CallableStream) { + self.cancel_callable_stream_with_reason( + OperationCancelReason::Requested, + ) + } else { + Ok(()) + }; + return Poll::Ready(Err(preserve_stream_cleanup(error, cleanup))); + } + cx.waker().wake_by_ref(); + Poll::Pending + } + CallOutcome::Halt | CallOutcome::Yield => { + if bridge_owned + && let Err(error) = self.host.complete_bridge_operation( + waiting.op_id, + HostAsyncOpTerminal::Failed, + ) + { + self.instance.waiting_host_op = None; + return Poll::Ready(Err(error)); + } + self.instance.waiting_host_op = None; + Poll::Ready(Err(VmError::HostError( + "async host continuation returned a control-flow outcome" + .to_string(), + ))) + } + }, } - Poll::Ready(Ok(())) } Poll::Ready(Err(err)) => { if matches!(waiting.source, WaitingHostOpSource::HostBridge) { diff --git a/src/vm/tests.rs b/src/vm/tests.rs index 24bd25d9..f6cb010d 100644 --- a/src/vm/tests.rs +++ b/src/vm/tests.rs @@ -416,6 +416,20 @@ fn empty_host_future() -> HostFuture { Box::pin(async { Ok(HostFutureOutput::returning(CallReturn::none())) }) } +#[test] +fn host_future_output_vm_continuation_survives_value_mapping() { + let output = HostFutureOutput::::continue_with(|_vm| Ok(CallOutcome::Pending(41))) + .map(|value| CallReturn::one(Value::Int(value))); + let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8])); + + match output.finish(&mut vm).expect("continuation should run") { + async_host::HostFutureResolution::Continue(CallOutcome::Pending(op_id)) => { + assert_eq!(op_id, 41) + } + _ => panic!("mapped host future output must retain its VM continuation"), + } +} + #[test] fn submitted_bridge_operation_reserves_before_submit_and_rolls_back_on_failure() { let submissions = Arc::new(Mutex::new(Vec::new())); diff --git a/tests/http_async_arch_tests.rs b/tests/http_async_arch_tests.rs new file mode 100644 index 00000000..a313d38a --- /dev/null +++ b/tests/http_async_arch_tests.rs @@ -0,0 +1,85 @@ +#![cfg(all(feature = "http-client", not(target_family = "wasm")))] + +use std::fs; +use std::path::PathBuf; + +fn source(path: &str) -> String { + fs::read_to_string(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(path)) + .unwrap_or_else(|error| panic!("failed to read {path}: {error}")) +} + +#[test] +fn http_hosts_are_macro_owned_async_functions_without_private_drivers() { + let module = source("src/builtins/runtime/http/mod.rs"); + let request = source("src/builtins/runtime/http/request.rs"); + let sse = source("src/builtins/runtime/http/sse.rs"); + let cargo = source("Cargo.toml"); + + assert!( + module.contains("async fn builtin_http_client_request"), + "the buffered host implementation must be an async macro function" + ); + assert!( + sse.contains("async fn builtin_http_client_sse"), + "the SSE host implementation must be an async macro function" + ); + assert!( + request.contains("hyper_util::client::legacy::Client"), + "HTTP transport and pooling must be owned by hyper-util" + ); + assert!( + module.contains("client: request::HttpClient"), + "the cloneable Hyper client must live in per-VM HTTP module state" + ); + assert!( + cargo.contains("\"client-legacy\"") && cargo.contains("\"http1\""), + "the HTTP feature must enable Hyper's maintained pooled client" + ); + + for (path, text) in [ + ("src/builtins/runtime/http/mod.rs", module.as_str()), + ("src/builtins/runtime/http/request.rs", request.as_str()), + ("src/builtins/runtime/http/sse.rs", sse.as_str()), + ] { + for forbidden in [ + "runtime_owned_pending", + "submit_host_future", + "HostAsyncBridge", + "std::thread", + "JoinHandle", + "tokio::runtime", + "runtime_block_on", + "HostOperation", + "HostResource", + "AtomicWaker", + ] { + assert!( + !text.contains(forbidden), + "{path} must not contain HTTP/SSE-owned async plumbing: {forbidden}" + ); + } + } + + for forbidden in [ + "HttpRequestResource", + "HttpResponseResource", + "SseStreamResource", + "SseScopeOperation", + "BufferedRequestShared", + "SseShared", + ] { + assert!( + !module.contains(forbidden) && !request.contains(forbidden) && !sse.contains(forbidden), + "transient HTTP/SSE operation or resource state remains: {forbidden}" + ); + } + + assert!( + sse.contains("impl HostStreamDriver for SseStreamDriver"), + "SSE may retain only the generic callable-stream continuation driver" + ); + assert!( + sse.contains("submit_callable_stream"), + "SSE callback re-entry must use the generic callable-stream continuation" + ); +} diff --git a/tests/standard_host_descriptor_arch_tests.rs b/tests/standard_host_descriptor_arch_tests.rs index 29c8d7d3..979ced13 100644 --- a/tests/standard_host_descriptor_arch_tests.rs +++ b/tests/standard_host_descriptor_arch_tests.rs @@ -25,12 +25,12 @@ const HTTP_SURFACE_ENABLED: bool = cfg!(all(feature = "http-client", not(target_ /// Fingerprint of the published standard host catalog **with** the HTTP /// surface. /// -/// Both fingerprints are exact goldens of the descriptor migration: the whole -/// point of the change is that guest contracts did not move. The standard -/// catalog is the merge of the composed module surfaces, so the golden depends -/// on the composed set — the default build (no `http-client`) composes one -/// module fewer and must reproduce [`STANDARD_CATALOG_FINGERPRINT_NO_HTTP`]. -const STANDARD_CATALOG_FINGERPRINT: &str = "6607e4fcb3187e73"; +/// The standard catalog is the merge of the current composed module surfaces, +/// so the golden depends on the composed set. The HTTP surface intentionally +/// omits transport-only resources; the default build (no `http-client`) +/// composes one module fewer and must reproduce +/// [`STANDARD_CATALOG_FINGERPRINT_NO_HTTP`]. +const STANDARD_CATALOG_FINGERPRINT: &str = "4e3b7572a59b3dae"; /// Fingerprint of the published standard host catalog **without** the HTTP /// surface: the `--workspace` default build and every wasm build. const STANDARD_CATALOG_FINGERPRINT_NO_HTTP: &str = "a6b4b2dcadc5df14"; @@ -39,7 +39,7 @@ const SQLITE_CATALOG_FINGERPRINT: &str = "b6d4c278145edacf"; const JIT_CATALOG_FINGERPRINT: &str = "d0a3efbca2d0923c"; const TIMER_CATALOG_FINGERPRINT: &str = "4af2dfa2aee1f42e"; #[cfg(all(feature = "http-client", not(target_family = "wasm")))] -const HTTP_CATALOG_FINGERPRINT: &str = "18a4033f5857c033"; +const HTTP_CATALOG_FINGERPRINT: &str = "66db92730480d50a"; /// The standard catalog fingerprint this build must reproduce exactly. fn standard_catalog_fingerprint() -> &'static str { @@ -53,8 +53,8 @@ fn standard_catalog_fingerprint() -> &'static str { /// Resource keys the standard catalog always publishes. const STANDARD_RESOURCE_KEYS: &[&str] = &["io.file", "sqlite.connection"]; -/// Resource keys the HTTP module publishes when it is composed. -const HTTP_RESOURCE_KEYS: &[&str] = &["http.request", "http.response", "http.sse"]; +/// HTTP uses macro-owned futures and does not publish transient resources. +const HTTP_RESOURCE_KEYS: &[&str] = &[]; /// Named structs the standard catalog always declares. const STANDARD_NAMED_STRUCTS: &[&str] = &[ diff --git a/tests/vm/http_host_tests.rs b/tests/vm/http_host_tests.rs index 6c204a01..6e3ecca2 100644 --- a/tests/vm/http_host_tests.rs +++ b/tests/vm/http_host_tests.rs @@ -53,6 +53,19 @@ impl HostAsyncBridge for TokioHostDriver { fn cancel_op(&mut self, op_id: HostOpId) { self.submitted.remove(&op_id); } + + fn request_cancel_op( + &mut self, + op_id: HostOpId, + _reason: vm::operation::OperationCancelReason, + ) -> VmResult<()> { + self.submitted.remove(&op_id); + Ok(()) + } + + fn poll_cancel_op(&mut self, _op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } } fn install_host_driver(vm: &mut Vm) { @@ -180,6 +193,53 @@ fn spawn_test_server() -> (u16, thread::JoinHandle<()>) { (port, handle) } +fn spawn_keep_alive_server() -> (u16, mpsc::Receiver, thread::JoinHandle<()>) { + let listener = bind_test_listener(); + let port = listener + .local_addr() + .expect("keep-alive listener should have an address") + .port(); + let (sender, receiver) = mpsc::channel(); + let handle = thread::spawn(move || { + let mut accepted = 0; + let mut stream: Option = None; + for _ in 0..2 { + let mut request = Vec::new(); + loop { + if stream.is_none() { + let (next, _) = accept_test_connection(&listener) + .expect("keep-alive request connection should arrive"); + stream = Some(next); + accepted += 1; + } + let socket = stream.as_mut().expect("connection should exist"); + let mut byte = [0_u8; 1]; + match socket.read(&mut byte) { + Ok(0) if request.is_empty() => { + stream = None; + } + Ok(0) => panic!("request ended before its headers"), + Ok(_) => { + request.push(byte[0]); + if request.ends_with(b"\r\n\r\n") { + break; + } + } + Err(error) => panic!("keep-alive request should be readable: {error}"), + } + } + assert!(request.starts_with(b"GET / HTTP/1.1")); + stream + .as_mut() + .expect("connection should exist") + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .expect("keep-alive response should be writable"); + } + sender.send(accepted).expect("connection count receiver"); + }); + (port, receiver, handle) +} + fn spawn_response_server(response: Vec) -> (u16, thread::JoinHandle<()>) { let listener = bind_test_listener(); let port = listener @@ -481,6 +541,36 @@ async fn http_host_executes_a_bounded_request_and_returns_a_response_map() { ); } +#[tokio::test(flavor = "current_thread")] +async fn http_client_pool_reuses_a_connection_across_vm_reset() { + let (port, accepted_connections, server) = spawn_keep_alive_server(); + let mut vm = Vm::new(build_request_program(format!("http://127.0.0.1:{port}/"))); + vm.configure_http(local_http_config(port)) + .expect("HTTP configuration should be valid"); + install_host_driver(&mut vm); + HostFunctionRegistry::new() + .bind_vm_cached(&mut vm) + .expect("default host registry should bind HTTP"); + + drive_vm_to_halt(&mut vm) + .await + .expect("first HTTP request should complete"); + vm.reset_for_reuse() + .expect("an idle VM should reset synchronously"); + drive_vm_to_halt(&mut vm) + .await + .expect("second HTTP request should complete"); + + assert_eq!( + accepted_connections + .recv_timeout(TEST_IO_TIMEOUT) + .expect("server should report connection count"), + 1, + "the per-VM Hyper client must retain and reuse its pooled connection" + ); + server.join().expect("keep-alive server should finish"); +} + #[tokio::test(flavor = "current_thread")] async fn buffered_response_headers_use_canonical_order_duplicates_and_raw_bytes() { let response = run_raw_response( @@ -1040,6 +1130,35 @@ fn cached_plan_refreshes_after_a_sibling_registry_mutation() { .expect("destination should rebuild a plan after sibling mutation"); } +#[tokio::test(flavor = "current_thread")] +async fn tls_handshake_obeys_connect_phase_timeout() { + let listener = bind_test_listener(); + let port = listener.local_addr().unwrap().port(); + let server = thread::spawn(move || { + let (_socket, _) = accept_test_connection(&listener).unwrap(); + thread::sleep(std::time::Duration::from_millis(400)); + }); + let mut vm = Vm::new(build_request_program(format!("https://127.0.0.1:{port}/"))); + let mut config = local_http_config(port); + config.allowed_schemes = vec!["https".to_string()]; + config.connect_timeout = std::time::Duration::from_millis(25); + config.request_timeout = std::time::Duration::from_millis(500); + vm.configure_http(config).unwrap(); + install_host_driver(&mut vm); + HostFunctionRegistry::new().bind_vm_cached(&mut vm).unwrap(); + + let started = Instant::now(); + let error = drive_vm_to_halt(&mut vm) + .await + .expect_err("stalled TLS handshake must time out"); + assert!(started.elapsed() < std::time::Duration::from_millis(250)); + assert!( + error.to_string().contains("connect phase deadline"), + "{error}" + ); + server.join().unwrap(); +} + #[tokio::test(flavor = "current_thread")] async fn max_stream_duration_does_not_shorten_buffered_requests() { let listener = bind_test_listener(); @@ -1293,6 +1412,18 @@ fn spawn_pending_then_response_server() -> (u16, mpsc::Receiver<()>, thread::Joi (port, ready_receiver, handle) } +async fn poll_pending_http_transport(vm: &mut Vm) { + assert!( + tokio::time::timeout( + std::time::Duration::from_millis(100), + vm.await_waiting_host_op(), + ) + .await + .is_err(), + "pending HTTP request unexpectedly completed" + ); +} + async fn reset_and_wait(vm: &mut Vm) -> Result<(), vm::VmError> { vm.reset_for_reuse()?; std::future::poll_fn(|cx| vm.poll_reset_for_reuse(cx)).await @@ -1310,6 +1441,7 @@ async fn reset_retires_buffered_http_future_and_releases_its_permit() { .bind_vm_cached(&mut vm) .expect("default host registry should bind HTTP"); assert!(matches!(vm.run(), Ok(VmStatus::Waiting(_)))); + poll_pending_http_transport(&mut vm).await; ready .recv() .expect("first request should reach the transport"); @@ -1330,8 +1462,8 @@ async fn reset_retires_buffered_http_future_and_releases_its_permit() { server.join().expect("pending server should finish"); } -#[test] -fn shutdown_and_drop_retire_buffered_http_futures() { +#[tokio::test(flavor = "current_thread")] +async fn shutdown_and_drop_retire_buffered_http_futures() { for shutdown in [true, false] { let (port, ready, server) = spawn_pending_server(); let mut vm = Vm::new(build_request_program(format!("http://127.0.0.1:{port}/"))); @@ -1343,6 +1475,7 @@ fn shutdown_and_drop_retire_buffered_http_futures() { .bind_vm_cached(&mut vm) .expect("default host registry should bind HTTP"); assert!(matches!(vm.run(), Ok(VmStatus::Waiting(_)))); + poll_pending_http_transport(&mut vm).await; ready .recv() .expect("request should reach the transport before teardown"); diff --git a/tests/vm/http_sse_tests.rs b/tests/vm/http_sse_tests.rs index 63008a00..e0c94f7e 100644 --- a/tests/vm/http_sse_tests.rs +++ b/tests/vm/http_sse_tests.rs @@ -982,6 +982,11 @@ async fn sse_reset_releases_the_connection_permit_before_reuse() { HostFunctionRegistry::new().bind_vm_cached(&mut vm).unwrap(); assert!(matches!(vm.run().unwrap(), VmStatus::Waiting(_))); + let _ = tokio::time::timeout( + std::time::Duration::from_millis(100), + vm.await_waiting_host_op(), + ) + .await; assert!( requests .recv_timeout(std::time::Duration::from_secs(1)) diff --git a/tests/vm/io_http_coexistence_tests.rs b/tests/vm/io_http_coexistence_tests.rs index 071ca410..b79cd762 100644 --- a/tests/vm/io_http_coexistence_tests.rs +++ b/tests/vm/io_http_coexistence_tests.rs @@ -11,8 +11,8 @@ use std::time::Duration; use vm::{ CallReturn, HostAsyncBridge, HostFunctionRegistry, HostFuture, HostFutureOutput, HostOpId, - HttpConfig, HttpHostExt, IoHostExt, IoPolicy, ResourceTypeKey, Value, Vm, VmError, VmResult, - VmStatus, compile_source, register_http_builtin_module, standard_host_catalog, + HttpConfig, HttpHostExt, IoHostExt, IoPolicy, Value, Vm, VmError, VmResult, VmStatus, + compile_source, register_http_builtin_module, standard_host_catalog, }; #[derive(Default)] @@ -321,28 +321,15 @@ async fn worker_cleanup_reaches_quiescence_after_io_and_http() { } #[test] -fn io_and_http_resource_type_keys_are_disjoint() { +fn http_transport_does_not_publish_guest_resources() { let catalog = standard_host_catalog(); - let io_keys = ["io.file", "io.socket", "io.process", "io.worker", "io.pipe"]; - let http_keys = ["http.request", "http.response", "http.sse"]; - for key in catalog - .resources() - .iter() - .map(|resource| resource.key.as_str()) - { - if io_keys.contains(&key) { - assert!(!http_keys.contains(&key)); - } - if http_keys.contains(&key) { - assert!(!io_keys.contains(&key)); - } - } - for key in io_keys { - let _ = ResourceTypeKey::new(key).expect("IO resource key should be valid"); - } - for key in http_keys { - let _ = ResourceTypeKey::new(key).expect("HTTP resource key should be valid"); - } + assert!( + catalog + .resources() + .iter() + .all(|resource| !resource.key.as_str().starts_with("http.")), + "HTTP transport state must stay internal to macro-owned async calls" + ); } #[tokio::test(flavor = "current_thread")] From 082274208770e58dbdd00b852484602ad62c5cc6 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sat, 19 Sep 2026 16:12:13 +0800 Subject: [PATCH 02/23] fix(http): harden SSE stream framing and ABI --- docs/callable-runtime.md | 8 +- docs/host-sdk-descriptors.md | 6 +- docs/http-client.md | 10 +- pd-vm-nostd/README.md | 2 +- pd-vm-nostd/src/vmbc.rs | 4 +- pd-vm-nostd/tests/call_script_tests.rs | 4 +- pd-vm-nostd/tests/embedded_vmbc.rs | 21 +++- src/builtins/runtime/http/sse.rs | 137 ++++++++++++++++--------- src/bytecode.rs | 7 +- src/host_api.rs | 6 +- src/vm/async_host/stream.rs | 119 +++++++++++++++++++-- src/vm/host.rs | 12 +++ src/vmbc.rs | 6 +- tests/host_descriptor_effect_tests.rs | 14 +-- tests/vm/http_sse_tests.rs | 32 ++++++ tests/wire/wire_tests.rs | 29 ++++-- 16 files changed, 322 insertions(+), 95 deletions(-) diff --git a/docs/callable-runtime.md b/docs/callable-runtime.md index 23e49e12..3e3d2d8a 100644 --- a/docs/callable-runtime.md +++ b/docs/callable-runtime.md @@ -1,6 +1,6 @@ # Script call frames and callable values -RustScript bytecode format version 13 (VMBC v13) carries runtime script call frames, first-class callable values, the static builtin ID catalog, the direct script-call opcode, and an explicit guest named-struct declaration section. Version 11 introduced frames, callable values, and the static catalog; version 12 adds `callscript` for statically resolved named calls; version 13 frames guest struct declarations so a 4-byte zero trailer cannot be mistaken for an empty table. +RustScript bytecode format version 14 (VMBC v14) carries runtime script call frames, first-class callable values, the static builtin ID catalog, the direct script-call opcode, and an explicit guest named-struct declaration section. Version 11 introduced frames, callable values, and the static catalog; version 12 added `callscript` for statically resolved named calls; version 13 framed guest struct declarations so a 4-byte zero trailer could not be mistaken for an empty table. Version 14 marks catalog fingerprint format v3 after transport-only HTTP resource declarations were removed. ## Bytecode contract @@ -18,7 +18,7 @@ The three call opcodes differ in who owns the callee and what the frame must pro - `callvalue` — the callee is a `Value::Callable` owned by the caller operand stack at the call site, and remains the caller's responsibility after the call. This path carries environments, closures, and any callable whose identity or capture state is runtime-valued. - `callscript` — the callee is owned by program callable metadata (the prototype table). The frame contributes only `argc` arguments and no callable value, but unlike `call` the callee is a script function rather than a builtin, so the call enters a new script frame with its own local base. -VMBC v13 is the current format. It decodes the legacy v11 stream without host-schema metadata and the v12 stream without a named-struct section, while v13 carries full host schemas, callable metadata, and an explicit guest named-struct table. Unknown versions and malformed resource schemas are rejected deterministically. PDRC v6 recordings and AOT artifacts (format 8, ABI 8) use their corresponding bumped versions and include callable metadata in cache identity. +VMBC v14 is the current format. It decodes the legacy v11 stream without host-schema metadata and the v12 stream without a named-struct section. VMBC v13 is rejected because its catalog fingerprints use the prior catalog surface; source recompilation is required. VMBC v14 carries full host schemas, callable metadata, an explicit guest named-struct table, and catalog fingerprint format v3 identities. Unknown versions and malformed resource schemas are rejected deterministically. PDRC v6 recordings and AOT artifacts (format 8, ABI 8) embed VMBC and therefore reject a nested v13 program; native cache identity includes bytecode ABI 14. ## Static builtin IDs @@ -27,7 +27,7 @@ Every VM-visible builtin (ordinary, internal, and special-call) has one explicit - **Immutable explicit IDs.** IDs never change once assigned. Adding or reordering catalog entries never renumbers existing entries; new builtins take the next free ID in their documented block (extension `0x0000..=0xFF8F` for future builtins and host imports, special-call `0xFF90..=0xFFA1`, ordinary `0xFFA2..=0xFFFF`). The reserved sentinel gap `0xFF90..=0xFF92` stays unassigned. - **Build-time validation.** The build fails on duplicate IDs, duplicate source names, duplicate Rust variants, out-of-block IDs, class/gate inconsistencies, a discovered runtime callable without an explicit ID, or a catalog entry without a runtime callable. - **Shared std/no-std IDs.** `pd-vm-nostd` dispatches on the same static indices through the checked-in generated mirror `pd-vm-nostd/src/generated_builtin_ids.rs`; the workspace test `static_builtin_ids_are_frozen` fails when the mirror drifts from the catalog. -- **Format breaks are permanent.** The static ID migration bumped VMBC to v11 (and the internal bytecode ABI to 11); the `callscript` opcode break bumped both to v12; the guest named-struct section bumped both to v13. Versions below the current encode format are not rewritten in place: v11/v12 remain readable only in their original framing. +- **Format breaks are permanent.** The static ID migration bumped VMBC to v11 (and the internal bytecode ABI to 11); the `callscript` opcode break bumped both to v12; the guest named-struct section bumped both to v13; catalog fingerprint format v3 bumped both to v14. Versions are never rewritten in place: v11/v12 remain readable only in their original framing, while v13 requires recompilation. ## Runtime model @@ -100,4 +100,4 @@ Whole-program AOT and Trace JIT use the same builtin call path (static catalog I ## Embedded runtime -`pd-vm-nostd` decodes the same VMBC v13 callable metadata and executes callable binding, `callvalue`, `callscript`, recursive frames, captures, and direct host targets using `core` plus `alloc`, dispatching on the identical static builtin IDs via its checked-in generated mirror. +`pd-vm-nostd` decodes the same VMBC v14 callable metadata, rejects v13 catalog artifacts, and executes callable binding, `callvalue`, `callscript`, recursive frames, captures, and direct host targets using `core` plus `alloc`, dispatching on the identical static builtin IDs via its checked-in generated mirror. diff --git a/docs/host-sdk-descriptors.md b/docs/host-sdk-descriptors.md index e4282725..3add8411 100644 --- a/docs/host-sdk-descriptors.md +++ b/docs/host-sdk-descriptors.md @@ -314,9 +314,9 @@ Each module declares two things: standard `#[pd_host_function]` has exactly one descriptor owner, every ownership list is declared by a file that belongs to a module `standard_host_modules()` composes (a module this build's gates turn off is the only exemption, and it is -stated in the guard), no gated module leaks into the derived catalog, and the -published catalog fingerprints are byte-for-byte unchanged for the composed -module set. +stated in the guard), no gated module leaks into the derived catalog, and any +intentional catalog revision updates the fingerprint format plus artifact ABI +with explicit prior-artifact rejection coverage. ## 7. Compatibility window diff --git a/docs/http-client.md b/docs/http-client.md index 16da6a47..68ed314d 100644 --- a/docs/http-client.md +++ b/docs/http-client.md @@ -26,7 +26,8 @@ compiled runtime surface and generated metadata synchronized. ## Native API -On a supported native target, enabling `http-client` preserves the public API: +On a supported native target, enabling `http-client` preserves the source-level +HTTP call signatures and embedding entry points: - `HttpConfig` controls request and stream limits, redirects, timeouts, and capability policy; @@ -40,6 +41,13 @@ Both builtins are ordinary `#[pd_host_function] async fn` declarations. Their macro-generated wrappers own async-host submission; the HTTP module does not publish transient request, response, or stream resources. +Removing those transport-only resource declarations is an intentional catalog +artifact break. Catalog fingerprints use format v3, and bytecode/VMBC use ABI +and wire version 14. VMBC v13 artifacts are rejected and must be recompiled; +the runtime does not recreate unused resource declarations solely to retain an +obsolete digest. Function schemas continue to receive exact validation at bind +time. + The HTTP and SSE behavior, cancellation, and native async bridge contracts are uniform across supported native targets. See [`callable-runtime.md`](callable-runtime.md) for the general callable and diff --git a/pd-vm-nostd/README.md b/pd-vm-nostd/README.md index b9d52466..57d7741c 100644 --- a/pd-vm-nostd/README.md +++ b/pd-vm-nostd/README.md @@ -6,7 +6,7 @@ compiler, parser, CLI, debugger, JIT/AOT backends, filesystem support, and opera ## Runtime surface -- VMBC v13 decoding with environment-free `CallScript` direct script calls alongside dynamic callable calls, plus an explicit guest named-struct declaration section +- VMBC v14 decoding with environment-free `CallScript` direct script calls alongside dynamic callable calls, an explicit guest named-struct declaration section, and deterministic rejection of v13 catalog artifacts that require recompilation - stack, local, and recursive script-frame execution for direct bytecode opcodes - instruction fuel with pause/resume support - synchronous named host bindings and dynamic host dispatch diff --git a/pd-vm-nostd/src/vmbc.rs b/pd-vm-nostd/src/vmbc.rs index 1e825c16..4b31ef53 100644 --- a/pd-vm-nostd/src/vmbc.rs +++ b/pd-vm-nostd/src/vmbc.rs @@ -11,6 +11,7 @@ const MAGIC: [u8; 4] = *b"VMBC"; const VERSION_V11: u16 = 11; const VERSION_V12: u16 = 12; const VERSION_V13: u16 = 13; +const VERSION_V14: u16 = 14; const FLAGS: u16 = 0; const MAX_WIRE_PAYLOAD_BYTES: usize = 64 * 1024 * 1024; const MAX_WIRE_BLOB_BYTES: usize = 16 * 1024 * 1024; @@ -75,7 +76,8 @@ pub fn decode_program(bytes: &[u8]) -> Result { let version = cursor.read_u16()?; let has_host_import_schemas = match version { VERSION_V11 => false, - VERSION_V12 | VERSION_V13 => true, + VERSION_V12 | VERSION_V14 => true, + VERSION_V13 => return Err(WireError::UnsupportedVersion(VERSION_V13)), _ => return Err(WireError::UnsupportedVersion(version)), }; let flags = cursor.read_u16()?; diff --git a/pd-vm-nostd/tests/call_script_tests.rs b/pd-vm-nostd/tests/call_script_tests.rs index 28d7ef35..97aa9c4c 100644 --- a/pd-vm-nostd/tests/call_script_tests.rs +++ b/pd-vm-nostd/tests/call_script_tests.rs @@ -70,8 +70,8 @@ fn call_script_executes_direct_call() { let compiled = compile_source("fn add2(value: int) -> int { value + 2 } add2(40);") .expect("direct call source should compile"); let bytes = encode_program(&compiled.program.with_local_count(compiled.locals)) - .expect("direct call program should encode as VMBC v13"); - let program = decode_program(&bytes).expect("no-std should decode VMBC v13"); + .expect("direct call program should encode as VMBC v14"); + let program = decode_program(&bytes).expect("no-std should decode VMBC v14"); assert!( program.code().windows(2).any(|pair| pair[0] == 0x1A), "compiler output should contain CallScript" diff --git a/pd-vm-nostd/tests/embedded_vmbc.rs b/pd-vm-nostd/tests/embedded_vmbc.rs index ab6c0c7c..b641ebde 100644 --- a/pd-vm-nostd/tests/embedded_vmbc.rs +++ b/pd-vm-nostd/tests/embedded_vmbc.rs @@ -31,9 +31,9 @@ fn encoded_scalar_program() -> Vec { } #[test] -fn embedded_decoder_reads_host_generated_v13() { +fn embedded_decoder_reads_host_generated_v14() { let bytes = encoded_scalar_program(); - let program = decode_program(&bytes).expect("embedded decoder should accept VMBC v13"); + let program = decode_program(&bytes).expect("embedded decoder should accept VMBC v14"); assert_eq!( program.code(), @@ -54,6 +54,17 @@ fn embedded_decoder_reads_host_generated_v13() { assert_eq!(program.imports()[0].arity, 1); } +#[test] +fn embedded_decoder_rejects_v13_catalog_artifacts() { + let mut bytes = encoded_scalar_program(); + bytes[4..6].copy_from_slice(&13u16.to_le_bytes()); + + assert!(matches!( + decode_program(&bytes), + Err(WireError::UnsupportedVersion(13)) + )); +} + #[test] fn embedded_decoder_skips_full_host_schema_metadata() { let resource = ResourceTypeKey::new("embedded.resource").expect("resource key"); @@ -139,7 +150,7 @@ fn embedded_decoder_skips_named_host_schema_from_std_encode() { .with_host_import_schemas(vec![schema]) .expect("schema alignment"); let bytes = encode_program(&program).expect("named host schema should encode"); - assert_eq!(u16::from_le_bytes([bytes[4], bytes[5]]), 13); + assert_eq!(u16::from_le_bytes([bytes[4], bytes[5]]), 14); let decoded = decode_program(&bytes).expect("embedded decoder should skip Named host schemas"); assert_eq!(decoded.imports().len(), 1); assert_eq!(decoded.imports()[0].name, "embedded::named"); @@ -629,7 +640,7 @@ fn embedded_decoder_rejects_oversized_nested_named_host_depth() { } #[test] -fn embedded_decoder_reads_v13_guest_named_struct_payload() { +fn embedded_decoder_reads_v14_guest_named_struct_payload() { let compiled = compile_source( r#" struct Point { x: int, y: int } @@ -639,7 +650,7 @@ fn embedded_decoder_reads_v13_guest_named_struct_payload() { ) .expect("guest Named source should compile"); let bytes = encode_program(&compiled.program).expect("struct-bearing program should encode"); - assert_eq!(u16::from_le_bytes([bytes[4], bytes[5]]), 13); + assert_eq!(u16::from_le_bytes([bytes[4], bytes[5]]), 14); let program = decode_program(&bytes).expect("embedded decoder should skip guest named structs"); assert_eq!(program.code().last().copied(), Some(OpCode::Ret as u8)); } diff --git a/src/builtins/runtime/http/sse.rs b/src/builtins/runtime/http/sse.rs index 8f904ce6..0c758e3a 100644 --- a/src/builtins/runtime/http/sse.rs +++ b/src/builtins/runtime/http/sse.rs @@ -1,10 +1,10 @@ -use std::collections::VecDeque; use std::future::Future; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; use std::time::{Duration, Instant}; +use hyper::body::Bytes; use pd_host_function::pd_host_function; use super::request::{ @@ -128,31 +128,31 @@ impl SseParser { Ok((consumed, None)) } - fn finish(&mut self) -> VmResult> { + fn finish(&mut self) -> VmResult> { if self.finished { - return Ok(Vec::new()); + return Ok(None); } self.finished = true; - let mut events = Vec::new(); + let mut event = None; if !self.prefix.is_empty() { let prefix = std::mem::take(&mut self.prefix); for byte in prefix { - if let Some(event) = self.process_byte(byte)? { - events.push(event); + if let Some(next) = self.process_byte(byte)? { + event = Some(next); } } } if !self.line.is_empty() - && let Some(event) = self.process_line()? + && let Some(next) = self.process_line()? { - events.push(event); + event = Some(next); } // EventSource dispatches only on a blank line. EOF discards a partial // event, including a final unterminated data line. self.data.clear(); self.has_data = false; self.event = None; - Ok(events) + Ok(event) } fn process_byte(&mut self, byte: u8) -> VmResult> { @@ -397,12 +397,35 @@ impl CaptureAsyncHostContext for SseRequestContext { } } +struct RetainedSseFrame { + data: Bytes, + offset: usize, +} + +impl RetainedSseFrame { + fn new(data: Bytes) -> Self { + Self { data, offset: 0 } + } + + fn next_event(&mut self, parser: &mut SseParser) -> VmResult> { + let (consumed, event) = parser.push_until_event(&self.data[self.offset..])?; + self.offset += consumed; + Ok(event) + } + + fn is_consumed(&self) -> bool { + self.offset == self.data.len() + } +} + /// Generic callable-stream continuation that owns only the Hyper response, -/// parser, callback-facing queue, deadlines, and the in-flight permit. +/// parser, one retained body frame, deadlines, and the in-flight permit. struct SseStreamDriver { response: OwnedResponse, parser: SseParser, - pending: VecDeque, + open_item: Option, + retained_frame: Option, + eof_event: Option, status: u16, headers: Arc>, url: String, @@ -414,6 +437,7 @@ struct SseStreamDriver { idle_sleep: Pin>, body_started: bool, eof: bool, + end_emitted: bool, _permit: super::policy::ConnectionPermit, } @@ -446,12 +470,7 @@ impl SseStreamDriver { ) })?; let headers = Arc::new(response_header_entries(response.response().headers())); - let mut pending = VecDeque::new(); - pending.push_back(sse_open_event( - status.as_u16(), - Arc::clone(&headers), - url.as_str(), - )); + let open_item = sse_open_event(status.as_u16(), Arc::clone(&headers), url.as_str()); let total_at = tokio::time::Instant::from_std(deadline); Ok(Self { response, @@ -460,7 +479,9 @@ impl SseStreamDriver { config.max_stream_item_bytes, config.max_stream_total_bytes, ), - pending, + open_item: Some(open_item), + retained_frame: None, + eof_event: None, status: status.as_u16(), headers, url: url.to_string(), @@ -473,6 +494,7 @@ impl SseStreamDriver { idle_sleep: Box::pin(tokio::time::sleep_until(total_at)), body_started: false, eof: false, + end_emitted: false, _permit: permit, }) } @@ -496,25 +518,31 @@ impl SseStreamDriver { .reset(tokio::time::Instant::from_std(idle_at)); } - fn pop_item(&mut self) -> Option { - self.pending.pop_front().map(|item| { - self.items = self.items.saturating_add(1); - HostStreamPoll::Item(item) - }) + fn item(&mut self, item: Value) -> HostStreamPoll { + self.items = self.items.saturating_add(1); + HostStreamPoll::Item(item) } - fn queue_data(&mut self, data: &[u8]) -> VmResult<()> { - self.parser.admit_chunk(data.len())?; - self.bytes_received = self.bytes_received.saturating_add(data.len()); - let mut offset = 0; - while offset < data.len() { - let (consumed, event) = self.parser.push_until_event(&data[offset..])?; - offset += consumed; - if let Some(event) = event { - self.pending.push_back(sse_data_event(event)); - } + fn poll_retained_frame(&mut self) -> VmResult> { + let Some(frame) = self.retained_frame.as_mut() else { + return Ok(None); + }; + let event = frame.next_event(&mut self.parser)?; + if frame.is_consumed() { + self.retained_frame = None; } - Ok(()) + Ok(event.map(|event| self.item(sse_data_event(event)))) + } + + fn poll_eof_item(&mut self) -> Option { + if let Some(event) = self.eof_event.take() { + return Some(self.item(event)); + } + if !self.end_emitted { + self.end_emitted = true; + return Some(self.item(sse_end_event())); + } + None } } @@ -532,10 +560,18 @@ impl HostStreamDriver for SseStreamDriver { SSE_TOTAL_DEADLINE_ERROR.to_string(), ))); } - if let Some(item) = self.pop_item() { - return Poll::Ready(Ok(item)); + if let Some(item) = self.open_item.take() { + return Poll::Ready(Ok(self.item(item))); + } + match self.poll_retained_frame() { + Ok(Some(item)) => return Poll::Ready(Ok(item)), + Err(error) => return Poll::Ready(Err(error)), + Ok(None) => {} } if self.eof { + if let Some(item) = self.poll_eof_item() { + return Poll::Ready(Ok(item)); + } return Poll::Ready(Ok(HostStreamPoll::Complete(self.summary("eof")))); } if self.body_started && self.idle_sleep.as_mut().poll(cx).is_ready() { @@ -553,23 +589,30 @@ impl HostStreamDriver for SseStreamDriver { return Poll::Pending; }; self.reset_idle_deadline(); - if let Err(error) = self.queue_data(&data) { + if let Err(error) = self.parser.admit_chunk(data.len()) { return Poll::Ready(Err(error)); } - if let Some(item) = self.pop_item() { - Poll::Ready(Ok(item)) - } else { - cx.waker().wake_by_ref(); - Poll::Pending + self.bytes_received = self.bytes_received.saturating_add(data.len()); + self.retained_frame = Some(RetainedSseFrame::new(data)); + match self.poll_retained_frame() { + Ok(Some(item)) => Poll::Ready(Ok(item)), + Ok(None) => { + cx.waker().wake_by_ref(); + Poll::Pending + } + Err(error) => Poll::Ready(Err(error)), } } Poll::Ready(Ok(None)) => { - if let Err(error) = self.parser.finish() { - return Poll::Ready(Err(error)); - } + let event = match self.parser.finish() { + Ok(event) => event, + Err(error) => return Poll::Ready(Err(error)), + }; self.eof = true; - self.pending.push_back(sse_end_event()); - Poll::Ready(Ok(self.pop_item().expect("end item was queued"))) + self.eof_event = event.map(sse_data_event); + Poll::Ready(Ok(self + .poll_eof_item() + .expect("EOF always produces a pending event or end item"))) } } } diff --git a/src/bytecode.rs b/src/bytecode.rs index f5d6e179..faa6d9c0 100644 --- a/src/bytecode.rs +++ b/src/bytecode.rs @@ -8,10 +8,11 @@ use crate::host_api::HostImportSchema; /// Bytecode ABI version used for VM-internal cache identity (JIT trace cache, /// program cache keys). The VMBC wire format version lives in `src/vmbc.rs` -/// (`VERSION_V13`); both were bumped together for the static builtin ID break +/// (`VERSION_V14`); both were bumped together for the static builtin ID break /// and again for the direct script-call (`CallScript`) opcode break. Version 13 -/// adds an explicit guest named-struct declaration section. -pub const BYTECODE_ABI_VERSION: u16 = 13; +/// adds an explicit guest named-struct declaration section. Version 14 marks +/// the catalog fingerprint v3 break after transient HTTP resources were removed. +pub const BYTECODE_ABI_VERSION: u16 = 14; pub type SharedString = Arc; pub type SharedBytes = Arc>; diff --git a/src/host_api.rs b/src/host_api.rs index 66e88311..a013a8e8 100644 --- a/src/host_api.rs +++ b/src/host_api.rs @@ -121,7 +121,7 @@ const FINGERPRINT_DOMAIN_MAGIC: &[u8; 8] = b"rss-hapi"; /// The fingerprint wire/format version. Bump whenever the canonical byte /// encoding or semantic interpretation changes so old and new digests are /// never compared across versions. -const FINGERPRINT_FORMAT_VERSION: u8 = 2; +const FINGERPRINT_FORMAT_VERSION: u8 = 3; /// Error returned when a [`ResourceTypeKey`] cannot be constructed. #[derive(Clone, Debug, PartialEq, Eq)] @@ -5101,8 +5101,8 @@ mod tests { } #[test] - fn fingerprint_version_is_two() { - assert_eq!(FINGERPRINT_FORMAT_VERSION, 2); + fn fingerprint_version_is_three() { + assert_eq!(FINGERPRINT_FORMAT_VERSION, 3); } #[test] diff --git a/src/vm/async_host/stream.rs b/src/vm/async_host/stream.rs index 1fc709fb..a5e7b144 100644 --- a/src/vm/async_host/stream.rs +++ b/src/vm/async_host/stream.rs @@ -154,6 +154,8 @@ pub(crate) struct HostStreamContinuation { pub(crate) op_id: HostOpId, pub(crate) callback: Value, pub(crate) item: Option, + pub(crate) expected_return_type: Option, + pub(crate) expected_return_schema: Option, pub(crate) phase: HostStreamPhase, pub(crate) parent_stack_base: usize, pub(crate) parent_frame_count: usize, @@ -232,6 +234,8 @@ impl Vm { op_id, callback, item: None, + expected_return_type: None, + expected_return_schema: None, phase: HostStreamPhase::AwaitItem, parent_stack_base: self.instance.stack.len(), parent_frame_count: self.instance.execution_frames.len(), @@ -524,14 +528,25 @@ impl Vm { if let Some(driver) = self.host.stream_drivers.get_mut(&op_id) { driver.acknowledge_item(); } - if let Some(stream) = self.instance.host_stream.as_mut() { - stream.phase = HostStreamPhase::AwaitItem; - } + let (expected_return_type, expected_return_schema) = self + .instance + .host_stream + .as_mut() + .map(|stream| { + stream.phase = HostStreamPhase::AwaitItem; + ( + stream.expected_return_type, + stream.expected_return_schema.clone(), + ) + }) + .ok_or(VmError::InvalidFrameState( + "missing callable stream continuation", + ))?; self.instance.waiting_host_op = Some(crate::vm::host::WaitingHostOp { op_id, source: crate::vm::host::WaitingHostOpSource::CallableStream, - expected_return_type: None, - expected_return_schema: None, + expected_return_type, + expected_return_schema, }); Ok(VmStatus::Waiting(op_id)) } @@ -563,6 +578,15 @@ impl Vm { "missing callable stream continuation", )); }; + let values = crate::vm::CallReturn::one(summary); + let validation_error = crate::vm::host::validate_host_call_return( + &values, + stream.expected_return_type, + stream.expected_return_schema.as_ref(), + &self.program, + self.host.execution_scope.resources(), + ) + .err(); let cleanup = self .host .begin_stream_termination(stream.op_id, termination) @@ -572,17 +596,21 @@ impl Vm { if let Some(item) = stream.item { self.drop_value_with_contract(item); } + if let Some(error) = validation_error { + self.abort_host_invocation(stream.parent_stack_base, stream.parent_frame_count); + return Err(preserve_stream_cleanup(error, cleanup)); + } if let Err(error) = cleanup { self.abort_host_invocation(stream.parent_stack_base, stream.parent_frame_count); return Err(error); } - self.instance.stack.push(summary); + values.push_onto_stack(&mut self.instance.stack); if self.host.has_pending_stream_terminations() { self.instance.waiting_host_op = Some(crate::vm::host::WaitingHostOp { op_id: stream.op_id, source: crate::vm::host::WaitingHostOpSource::CallableStreamTermination, - expected_return_type: None, - expected_return_schema: None, + expected_return_type: stream.expected_return_type, + expected_return_schema: stream.expected_return_schema, }); Ok(false) } else { @@ -619,3 +647,78 @@ impl Vm { cleanup } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::bytecode::{OpCode, Program, ValueType}; + use crate::vm::host::{WaitingHostOp, WaitingHostOpSource}; + + struct CompleteWith(Value); + + impl HostStreamDriver for CompleteWith { + fn poll_next(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(HostStreamPoll::Complete(self.0.clone()))) + } + + fn apply_action(&mut self, _action: Value) -> VmResult { + unreachable!("summary-only test driver has no callback items") + } + } + + fn vm_waiting_for_int_summary(op_id: HostOpId) -> Vm { + let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8])); + vm.host + .stream_drivers + .insert(op_id, Box::new(CompleteWith(Value::string("malformed")))); + vm.instance.host_stream = Some(HostStreamContinuation { + op_id, + callback: Value::Null, + item: None, + expected_return_type: Some(ValueType::Int), + expected_return_schema: None, + phase: HostStreamPhase::AwaitItem, + parent_stack_base: 0, + parent_frame_count: 0, + parent_ip: 0, + }); + vm.instance.waiting_host_op = Some(WaitingHostOp { + op_id, + source: WaitingHostOpSource::CallableStream, + expected_return_type: Some(ValueType::Int), + expected_return_schema: None, + }); + vm + } + + #[test] + fn eof_summary_is_validated_against_the_original_host_return_type() { + let mut vm = vm_waiting_for_int_summary(41); + let waker = std::task::Waker::noop(); + let mut cx = Context::from_waker(waker); + + let result = vm.poll_waiting_host_op(&mut cx); + let Poll::Ready(result) = result else { + panic!("summary driver should complete immediately"); + }; + let error = result.expect_err("malformed EOF summary must fail return validation"); + + assert!(matches!(error, VmError::TypeMismatch("int"))); + assert!(vm.stack().is_empty()); + } + + #[test] + fn callback_stop_summary_is_validated_against_the_original_host_return_type() { + let mut vm = vm_waiting_for_int_summary(42); + + let error = vm + .finish_callable_stream_with_termination( + Value::string("malformed"), + HostStreamTermination::Cancelled(OperationCancelReason::Requested), + ) + .expect_err("malformed callback-stop summary must fail return validation"); + + assert!(matches!(error, VmError::TypeMismatch("int"))); + assert!(vm.stack().is_empty()); + } +} diff --git a/src/vm/host.rs b/src/vm/host.rs index 7c83629d..c6757542 100644 --- a/src/vm/host.rs +++ b/src/vm/host.rs @@ -4495,6 +4495,18 @@ impl Vm { self.host.track_bridge_host_op(op_id)?; } let expected_return_schema = expected_return_schema.cloned(); + if matches!(source, WaitingHostOpSource::CallableStream) { + let stream = self + .instance + .host_stream + .as_mut() + .filter(|stream| stream.op_id == op_id) + .ok_or(VmError::InvalidFrameState( + "missing callable stream continuation for waiting operation", + ))?; + stream.expected_return_type = expected_return_type; + stream.expected_return_schema = expected_return_schema.clone(); + } self.instance.waiting_host_op = Some(WaitingHostOp { op_id, source, diff --git a/src/vmbc.rs b/src/vmbc.rs index e58c5172..af13240d 100644 --- a/src/vmbc.rs +++ b/src/vmbc.rs @@ -21,6 +21,7 @@ const MAGIC: [u8; 4] = *b"VMBC"; const VERSION_V11: u16 = 11; const VERSION_V12: u16 = 12; const VERSION_V13: u16 = 13; +const VERSION_V14: u16 = 14; const FLAGS: u16 = 0; const MAX_WIRE_PAYLOAD_BYTES: usize = 64 * 1024 * 1024; const MAX_WIRE_BLOB_BYTES: usize = 16 * 1024 * 1024; @@ -306,7 +307,7 @@ fn read_constant(cursor: &mut Cursor<'_>, depth: usize) -> Result Result, WireError> { let mut out = Vec::new(); out.extend_from_slice(&MAGIC); - out.extend_from_slice(&VERSION_V13.to_le_bytes()); + out.extend_from_slice(&VERSION_V14.to_le_bytes()); out.extend_from_slice(&FLAGS.to_le_bytes()); write_u32_count("constants", program.constants.len(), &mut out)?; @@ -362,7 +363,8 @@ pub fn decode_program(bytes: &[u8]) -> Result { let version = cursor.read_u16()?; let has_host_import_schemas = match version { VERSION_V11 => false, - VERSION_V12 | VERSION_V13 => true, + VERSION_V12 | VERSION_V14 => true, + VERSION_V13 => return Err(WireError::UnsupportedVersion(VERSION_V13)), _ => return Err(WireError::UnsupportedVersion(version)), }; diff --git a/tests/host_descriptor_effect_tests.rs b/tests/host_descriptor_effect_tests.rs index 5473a46f..be9f91a3 100644 --- a/tests/host_descriptor_effect_tests.rs +++ b/tests/host_descriptor_effect_tests.rs @@ -112,7 +112,7 @@ fn value_only_catalog_fingerprint_is_stable() { assert_eq!(schemas[0].fingerprint, catalog.fingerprint()); let rebuilt = value_only_catalog(); assert_eq!(catalog.fingerprint(), rebuilt.fingerprint()); - assert_eq!(catalog.fingerprint().to_string(), "b8b323eb678acb33"); + assert_eq!(catalog.fingerprint().to_string(), "900fe90d1c222a2a"); } #[test] @@ -132,7 +132,7 @@ fn named_struct_catalog_preserves_inline_fields() { ); let rebuilt = named_struct_catalog(); assert_eq!(catalog.fingerprint(), rebuilt.fingerprint()); - assert_eq!(catalog.fingerprint().to_string(), "1ae5b55fa213c708"); + assert_eq!(catalog.fingerprint().to_string(), "8e9381404a23e061"); } #[test] @@ -146,7 +146,7 @@ fn borrowed_resource_catalog_uses_borrow_passing() { ); let rebuilt = borrowed_resource_catalog(); assert_eq!(catalog.fingerprint(), rebuilt.fingerprint()); - assert_eq!(catalog.fingerprint().to_string(), "3feba549802b8453"); + assert_eq!(catalog.fingerprint().to_string(), "a9669bd807fe6fee"); } #[test] @@ -156,7 +156,7 @@ fn mutable_resource_catalog_uses_borrow_mut_passing() { assert_eq!(schemas[0].params[0].passing, HostParamPassing::BorrowMut); let rebuilt = mutable_resource_catalog(); assert_eq!(catalog.fingerprint(), rebuilt.fingerprint()); - assert_eq!(catalog.fingerprint().to_string(), "5e72bc2ba4ad83c4"); + assert_eq!(catalog.fingerprint().to_string(), "f7b1760ec6e726e9"); } #[test] @@ -166,7 +166,7 @@ fn owned_resource_catalog_uses_take_owned_passing() { assert_eq!(schemas[0].params[0].passing, HostParamPassing::TakeOwned); let rebuilt = owned_resource_catalog(); assert_eq!(catalog.fingerprint(), rebuilt.fingerprint()); - assert_eq!(catalog.fingerprint().to_string(), "0b4c23d5c9d8dbcd"); + assert_eq!(catalog.fingerprint().to_string(), "d523381d8a16c5a4"); } #[test] @@ -179,7 +179,7 @@ fn resource_return_catalog_declares_resource_schema() { ); let rebuilt = resource_return_catalog(); assert_eq!(catalog.fingerprint(), rebuilt.fingerprint()); - assert_eq!(catalog.fingerprint().to_string(), "b2ff5d0e37985029"); + assert_eq!(catalog.fingerprint().to_string(), "7dbb2227244cabb2"); } #[test] @@ -240,7 +240,7 @@ fn guest_resource_effects_are_tied_to_existing_passing_modes() { ); let catalog = borrowed_resource_catalog(); - assert_eq!(catalog.fingerprint().to_string(), "3feba549802b8453"); + assert_eq!(catalog.fingerprint().to_string(), "a9669bd807fe6fee"); } #[test] diff --git a/tests/vm/http_sse_tests.rs b/tests/vm/http_sse_tests.rs index e0c94f7e..12a18ba3 100644 --- a/tests/vm/http_sse_tests.rs +++ b/tests/vm/http_sse_tests.rs @@ -864,6 +864,38 @@ async fn sse_rejects_disallowed_redirect_targets_before_connecting() { } } +#[tokio::test(flavor = "current_thread")] +async fn sse_many_events_in_one_frame_materializes_one_per_callback_acknowledgement() { + let body = b"data: first\n\ndata: second event is deliberately over the line limit\n\n"; + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: {}\r\n\r\n{}", + body.len(), + String::from_utf8_lossy(body) + ) + .into_bytes(); + let response = Box::leak(response.into_boxed_slice()); + let (port, server) = server(vec![response]); + let source = format!( + r#"use http; + fn stop_after_first_event(item: SseEvent) -> SseCallbackAction {{ + {{action: if item.kind == "event" => {{ "stop" }} else => {{ "continue" }} }} + }} + http::client::sse( + {{"method":"GET","url":"http://127.0.0.1:{port}/events"}}, + stop_after_first_event + );"# + ); + let mut limits = config(port); + limits.max_sse_line_bytes = 16; + + let vm = run_sse_source(&source, limits) + .await + .expect("the unacknowledged second event must not be materialized"); + server.join().unwrap(); + assert_eq!(field(&vm.stack()[0], "outcome"), &Value::string("stopped")); + assert_eq!(field(&vm.stack()[0], "items"), &Value::Int(2)); +} + #[tokio::test(flavor = "current_thread")] async fn sse_stop_retires_without_end_and_returns_stopped_summary() { let (port, server) = server(vec![ diff --git a/tests/wire/wire_tests.rs b/tests/wire/wire_tests.rs index 487a9f39..3e30d2ae 100644 --- a/tests/wire/wire_tests.rs +++ b/tests/wire/wire_tests.rs @@ -58,7 +58,7 @@ fn wire_roundtrip_preserves_constants_and_code() { }); let encoded = encode_program(&program).expect("encode should succeed"); - assert_eq!(u16::from_le_bytes([encoded[4], encoded[5]]), 13); + assert_eq!(u16::from_le_bytes([encoded[4], encoded[5]]), 14); let decoded = decode_program(&encoded).expect("decode should succeed"); assert_eq!(decoded.constants, program.constants); @@ -1066,10 +1066,10 @@ fn validate_rejects_call_script_targeting_host_import_prototype() { } #[test] -fn call_script_wire_version_is_v13_and_v11_accepts_schema_less_program() { +fn call_script_wire_version_is_v14_and_v11_accepts_schema_less_program() { let program = Program::new(vec![], vec![vm::OpCode::Ret as u8]); let encoded = encode_program(&program).expect("encode should succeed"); - assert_eq!(u16::from_le_bytes([encoded[4], encoded[5]]), 13); + assert_eq!(u16::from_le_bytes([encoded[4], encoded[5]]), 14); let mut old = encoded; strip_empty_named_struct_section(&mut old); @@ -1089,7 +1089,7 @@ fn call_script_no_script_program_code_bytes_unchanged_by_version_bump() { bc.ret(); let program = Program::new(vec![Value::Int(1), Value::Int(2)], bc.finish()); let encoded = encode_program(&program).expect("encode should succeed"); - assert_eq!(u16::from_le_bytes([encoded[4], encoded[5]]), 13); + assert_eq!(u16::from_le_bytes([encoded[4], encoded[5]]), 14); let decoded = decode_program(&encoded).expect("decode should succeed"); assert_eq!(decoded.code, program.code); assert_eq!(decoded.constants, program.constants); @@ -1112,7 +1112,7 @@ fn v12_trailing_zero_count_is_not_a_named_struct_table() { } #[test] -fn v13_roundtrip_preserves_guest_named_struct_payload() { +fn v14_roundtrip_preserves_guest_named_struct_payload() { let compiled = compile_source( r#" struct Point { x: int, y: int } @@ -1126,10 +1126,23 @@ fn v13_roundtrip_preserves_guest_named_struct_payload() { "codegen should attach guest struct decls" ); let encoded = encode_program(&compiled.program).expect("struct-bearing program should encode"); - assert_eq!(u16::from_le_bytes([encoded[4], encoded[5]]), 13); - let decoded = decode_program(&encoded).expect("v13 named-struct section should decode"); + assert_eq!(u16::from_le_bytes([encoded[4], encoded[5]]), 14); + assert_eq!(vm::bytecode::BYTECODE_ABI_VERSION, 14); + let decoded = decode_program(&encoded).expect("v14 named-struct section should decode"); assert!( decoded.named_struct_decls().contains_key("Point"), - "VMBC v13 should preserve guest struct decls" + "VMBC v14 should preserve guest struct decls" ); } + +#[test] +fn v13_artifact_requires_recompilation_after_catalog_revision() { + let program = Program::new(Vec::new(), vec![vm::OpCode::Ret as u8]); + let mut encoded = encode_program(&program).expect("current program should encode"); + encoded[4..6].copy_from_slice(&13u16.to_le_bytes()); + + assert!(matches!( + decode_program(&encoded), + Err(WireError::UnsupportedVersion(13)) + )); +} From dfdf16dfb9951a0ad70f367becef8f4619c6bdab Mon Sep 17 00:00:00 2001 From: fffonion Date: Sat, 19 Sep 2026 16:49:32 +0800 Subject: [PATCH 03/23] docs(vmbc): update current-version wording --- pd-vm-nostd/tests/call_script_tests.rs | 2 +- src/compiler/frontends/mod.rs | 2 +- src/compiler/parser/mod.rs | 6 +++--- tests/wire/wire_tests.rs | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pd-vm-nostd/tests/call_script_tests.rs b/pd-vm-nostd/tests/call_script_tests.rs index 97aa9c4c..880fd703 100644 --- a/pd-vm-nostd/tests/call_script_tests.rs +++ b/pd-vm-nostd/tests/call_script_tests.rs @@ -1,6 +1,6 @@ //! Milestone 7: `CallScript` parity in the no_std + alloc runtime. //! -//! Programs are produced by the std VMBC encoder (V13) or hand-built with +//! Programs are produced by the current std VMBC encoder or hand-built with //! `CallScript` bytecode (0x1A, prototype_id:u32 LE, argc:u8) so the wire //! contract and the typed validation/execution failures are pinned //! independently of the compiler. diff --git a/src/compiler/frontends/mod.rs b/src/compiler/frontends/mod.rs index cb3bd0e2..efa726ea 100644 --- a/src/compiler/frontends/mod.rs +++ b/src/compiler/frontends/mod.rs @@ -191,7 +191,7 @@ pub(super) fn parse_rustscript_repl_source( } /// REPL parse with an optional catalog snapshot: when `Some`, the parsed IR -/// carries `host_api_metadata` so standard host calls compile to exact V13 +/// carries `host_api_metadata` so standard host calls compile to exact /// `HostImport` schemas (never a name-only fallback). pub(super) fn parse_rustscript_repl_source_with_catalog( source: &str, diff --git a/src/compiler/parser/mod.rs b/src/compiler/parser/mod.rs index ea792b26..6f4bd0a4 100644 --- a/src/compiler/parser/mod.rs +++ b/src/compiler/parser/mod.rs @@ -316,9 +316,9 @@ impl Parser { } /// Catalog-aware REPL constructor: combines the predeclared-locals path - /// with an optional [`HostApiCatalog`] snapshot so REPL compiles emit - /// exact V13 `HostImport` schemas against the standard snapshot (when a - /// catalog is supplied) instead of name-only imports. + /// with an optional [`HostApiCatalog`] snapshot so REPL compiles emit exact + /// `HostImport` schemas against the standard snapshot (when a catalog is + /// supplied) instead of name-only imports. #[allow(clippy::too_many_arguments)] pub(super) fn new_with_predeclared_locals_and_host_catalog( source: &str, diff --git a/tests/wire/wire_tests.rs b/tests/wire/wire_tests.rs index 3e30d2ae..3b06786c 100644 --- a/tests/wire/wire_tests.rs +++ b/tests/wire/wire_tests.rs @@ -81,7 +81,7 @@ fn wire_v11_legacy_imports_decode_without_schema_metadata() { vec![import.clone()], None, ); - let encoded = encode_program(&program).expect("v13 encoding should succeed"); + let encoded = encode_program(&program).expect("current encoding should succeed"); let marker_offset = 8 + 4 + 4 + program.code.len() + 4 + 4 + import.name.len() + 2; assert_eq!(encoded[marker_offset], 0); let mut legacy = encoded; @@ -97,7 +97,7 @@ fn wire_v11_legacy_imports_decode_without_schema_metadata() { #[test] fn wire_v11_zero_import_program_decodes_by_version() { let program = Program::new(Vec::new(), vec![vm::OpCode::Ret as u8]); - let mut encoded = encode_program(&program).expect("v13 encoding should succeed"); + let mut encoded = encode_program(&program).expect("current encoding should succeed"); strip_empty_named_struct_section(&mut encoded); encoded[4..6].copy_from_slice(&11u16.to_le_bytes()); @@ -1098,7 +1098,7 @@ fn call_script_no_script_program_code_bytes_unchanged_by_version_bump() { #[test] fn v12_trailing_zero_count_is_not_a_named_struct_table() { let program = Program::new(Vec::new(), vec![vm::OpCode::Ret as u8]); - let mut encoded = encode_program(&program).expect("v13 encoding should succeed"); + let mut encoded = encode_program(&program).expect("current encoding should succeed"); strip_empty_named_struct_section(&mut encoded); encoded[4..6].copy_from_slice(&12u16.to_le_bytes()); decode_program(&encoded).expect("clean v12 without a named-struct section should decode"); From ffe21437af20bb728a7dd4db7e69c2f663ef23be Mon Sep 17 00:00:00 2001 From: fffonion Date: Sat, 19 Sep 2026 14:14:17 +0800 Subject: [PATCH 04/23] refactor(io): simplify async host lifecycles --- build.rs | 7 +- src/builtins/runtime/io/async_io.rs | 677 ++----- src/builtins/runtime/io/blocking.rs | 1863 +++----------------- src/builtins/runtime/io/mod.rs | 25 +- tests/builtins/io_async_tests.rs | 38 +- tests/builtins/io_builtin_edge_tests.rs | 33 +- tests/builtins/io_scope_lifecycle_tests.rs | 350 +--- tests/io_descriptor_install_tests.rs | 26 +- 8 files changed, 629 insertions(+), 2390 deletions(-) diff --git a/build.rs b/build.rs index 71a71477..ae0c8a3c 100644 --- a/build.rs +++ b/build.rs @@ -2351,7 +2351,12 @@ mod tests { .iter() .find(|callable| callable.name == "io::open") .expect("selected IO source must contain io::open"); - assert_eq!(open.host_execution, HostExecutionKind::MaySuspend); + let expected_execution = if async_enabled || target_arch == "wasm32" { + HostExecutionKind::MaySuspend + } else { + HostExecutionKind::Sync + }; + assert_eq!(open.host_execution, expected_execution); } } } diff --git a/src/builtins/runtime/io/async_io.rs b/src/builtins/runtime/io/async_io.rs index 859d4828..bd2342e0 100644 --- a/src/builtins/runtime/io/async_io.rs +++ b/src/builtins/runtime/io/async_io.rs @@ -1,22 +1,15 @@ -//! Feature-selected async IO host implementation. +//! Tokio-backed IO hosts for builds with the `async` feature. //! -//! This is the `async`-feature counterpart of the worker-thread -//! [`blocking`](super::blocking) implementation. Live handles are typed -//! [`IoResource`]s owned by the VM's execution scope (exactly like the -//! blocking path) and in-flight IO work runs through tokio; the guest-facing -//! builtins are async host functions that capture owned host context and -//! submit a future through the generic async host bridge. -//! -//! The guest-visible handle id is the raw resource token, so handles opened -//! on one path can be closed/read on the other. +//! Each call is an ordinary annotated async function. Open file and process +//! handles remain typed execution-scope resources because they span guest +//! calls; transient reads, writes, flushes, and closes rely on the generic +//! submitted-future lifecycle. -use std::future::Future; use std::path::{Path, PathBuf}; -use std::pin::Pin; use std::process::Stdio; -use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex as StdMutex}; -use std::task::{Context, Poll, Waker}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::task::{Context, Poll}; use pd_host_function::pd_host_function; use tokio::fs::{File, OpenOptions}; @@ -44,311 +37,128 @@ pub(crate) enum IoHandle { }, } -type CloseFuture = Pin> + Send + 'static>>; - -/// The typed resource stored in the execution scope for one async IO handle. -/// -/// Mirrors the blocking path: the handle lives behind an `Arc>` -/// so the async builtin can take/restore it while the resource stays in the -/// scope table. Closing is exact-once. -struct IoResource { - handle: Arc>>, - closed: Arc, - process_id: Arc, - active_operations: Arc, - close_waker: Arc>>, - close_scheduled: Arc, - close_future: Option, - owner: bool, - owner_alive: Arc, -} - -impl crate::host_extension::HostResourceType for IoResource { - const KEY: &'static str = super::IO_FILE_KEY; - const DESCRIPTION: &'static str = super::IO_FILE_DESCRIPTION; +impl Drop for IoHandle { + fn drop(&mut self) { + match self { + Self::PopenRead { child, .. } | Self::PopenWrite { child, .. } => { + terminate_process_id(child.id().unwrap_or(0)); + let _ = child.start_kill(); + } + Self::File(_) => {} + } + } } -/// The canonical declaration for the `io.file` resource type. -pub(crate) fn io_file_resource() -> crate::host_extension::HostResourceTypeMeta { - crate::host_extension::HostResourceTypeMeta::of::() +/// Shared handle state captured by async calls. +struct IoResourceState { + handle: Mutex>, + closed: AtomicBool, + process_id: AtomicU32, } -impl IoResource { +impl IoResourceState { fn new(handle: IoHandle) -> Self { - let process_id = match &handle { - IoHandle::PopenRead { child, .. } | IoHandle::PopenWrite { child, .. } => { - child.id().unwrap_or(0) - } - IoHandle::File(_) => 0, - }; - Self { - handle: Arc::new(Mutex::new(Some(handle))), - closed: Arc::new(AtomicBool::new(false)), - process_id: Arc::new(AtomicU32::new(process_id)), - active_operations: Arc::new(AtomicUsize::new(0)), - close_waker: Arc::new(StdMutex::new(None)), - close_scheduled: Arc::new(AtomicBool::new(false)), - close_future: None, - owner: true, - owner_alive: Arc::new(AtomicBool::new(true)), - } - } - - fn new_shared(cells: &IoResource) -> Self { + let process_id = process_id(&handle); Self { - handle: Arc::clone(&cells.handle), - closed: Arc::clone(&cells.closed), - process_id: Arc::clone(&cells.process_id), - active_operations: Arc::clone(&cells.active_operations), - close_waker: Arc::clone(&cells.close_waker), - close_scheduled: Arc::clone(&cells.close_scheduled), - close_future: None, - owner: false, - owner_alive: Arc::clone(&cells.owner_alive), + handle: Mutex::new(Some(handle)), + closed: AtomicBool::new(false), + process_id: AtomicU32::new(process_id), } } - fn begin_operation(&self, operation: &'static str) -> VmResult { + fn ensure_open(&self, operation: &str) -> VmResult<()> { if self.closed.load(Ordering::Acquire) { - return Err(VmError::HostError(format!("{operation} handle is closed"))); + Err(VmError::HostError(format!("{operation} handle is closed"))) + } else { + Ok(()) } - self.active_operations.fetch_add(1, Ordering::AcqRel); - if self.closed.load(Ordering::Acquire) { - self.active_operations.fetch_sub(1, Ordering::AcqRel); - wake_close_waker(&self.close_waker); - return Err(VmError::HostError(format!("{operation} handle is closed"))); - } - Ok(IoOperationLease { - active_operations: Arc::clone(&self.active_operations), - close_waker: Arc::clone(&self.close_waker), - handle: Arc::clone(&self.handle), - closed: Arc::clone(&self.closed), - owner_alive: Arc::clone(&self.owner_alive), - close_scheduled: Arc::clone(&self.close_scheduled), - process_id: Arc::clone(&self.process_id), - completed: false, - }) } +} - fn schedule_close(&mut self, reason: ResourceCloseReason) { - if self.close_future.is_some() { - return; - } - self.close_scheduled.store(true, Ordering::Release); - let handle = Arc::clone(&self.handle); - let process_id = Arc::clone(&self.process_id); - self.close_future = Some(Box::pin(async move { - let handle = handle.lock().await.take(); - let result = match handle { - Some(handle) => close_io_handle(handle, reason).await, - None => Ok(()), - }; - if result.is_ok() { - process_id.store(0, Ordering::Release); - } - result - })); - } +/// The typed resource stored in the execution scope for one async IO handle. +struct IoResource { + state: Arc, +} - fn wait_for_operations(&self, cx: &Context<'_>) -> bool { - if self.active_operations.load(Ordering::Acquire) == 0 { - return false; - } - let mut wake = None; - let pending = { - let mut slot = self - .close_waker - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - if self.active_operations.load(Ordering::Acquire) == 0 { - false - } else { - *slot = Some(cx.waker().clone()); - if self.active_operations.load(Ordering::Acquire) == 0 { - wake = slot.take(); - false - } else { - true - } - } - }; - if let Some(waker) = wake { - waker.wake(); +impl IoResource { + fn new(handle: IoHandle) -> Self { + Self { + state: Arc::new(IoResourceState::new(handle)), } - pending } - fn take_handle(&self) -> impl Future> + Send + 'static { - let handle = Arc::clone(&self.handle); - async move { - handle - .lock() - .await - .take() - .ok_or_else(|| VmError::HostError("io handle is closed".to_string())) + fn close_nonblocking(&mut self) -> ResourceResult { + self.state.closed.store(true, Ordering::Release); + terminate_process_id(self.state.process_id.load(Ordering::Acquire)); + let Ok(mut slot) = self.state.handle.try_lock() else { + return Ok(CloseProgress::Pending); + }; + if let Some(mut handle) = slot.take() { + start_close_io_handle(&mut handle)?; } + self.state.process_id.store(0, Ordering::Release); + Ok(CloseProgress::Ready) } } -struct IoOperationLease { - active_operations: Arc, - close_waker: Arc>>, - handle: Arc>>, - closed: Arc, - owner_alive: Arc, - close_scheduled: Arc, - process_id: Arc, - completed: bool, -} - -impl Drop for IoOperationLease { +impl Drop for IoResource { fn drop(&mut self) { - if !self.completed { - self.closed.store(true, Ordering::Release); - terminate_process_id( - self.process_id.load(Ordering::Acquire), - ResourceCloseReason::ResourceClosed, - ); - if !self.close_scheduled.load(Ordering::Acquire) { - self.process_id.store(0, Ordering::Release); - } - } - let previous = self.active_operations.fetch_sub(1, Ordering::AcqRel); - debug_assert!(previous > 0, "IO operation lease count underflowed"); - if previous != 1 { - return; - } - if self.closed.load(Ordering::Acquire) - && (!self.owner_alive.load(Ordering::Acquire) - || !self.close_scheduled.load(Ordering::Acquire)) - && let Ok(mut guard) = self.handle.try_lock() + self.state.closed.store(true, Ordering::Release); + terminate_process_id(self.state.process_id.swap(0, Ordering::AcqRel)); + if let Ok(mut slot) = self.state.handle.try_lock() + && let Some(mut handle) = slot.take() { - drop(guard.take()); + let _ = start_close_io_handle(&mut handle); } - wake_close_waker(&self.close_waker); } } -impl IoOperationLease { - fn complete(&mut self) { - self.completed = true; - } -} - -fn wake_close_waker(close_waker: &StdMutex>) { - if let Some(waker) = close_waker - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .take() - { - waker.wake(); - } +impl crate::host_extension::HostResourceType for IoResource { + const KEY: &'static str = super::IO_FILE_KEY; + const DESCRIPTION: &'static str = super::IO_FILE_DESCRIPTION; } -impl Drop for IoHandle { - fn drop(&mut self) { - match self { - Self::PopenRead { child, .. } | Self::PopenWrite { child, .. } => { - reap_child_now(child, ResourceCloseReason::VmDrop); - } - Self::File(_) => {} - } - } +/// The canonical declaration for the `io.file` resource type. +pub(crate) fn io_file_resource() -> crate::host_extension::HostResourceTypeMeta { + crate::host_extension::HostResourceTypeMeta::of::() } -fn reap_child_now(child: &mut Child, reason: ResourceCloseReason) { - let Some(pid) = child.id() else { - return; - }; - terminate_process_id(pid, reason); - let _ = child.start_kill(); - for _ in 0..200 { - match child.try_wait() { - Ok(Some(_)) => return, - Ok(None) => std::thread::sleep(std::time::Duration::from_millis(1)), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return, - Err(_) => return, - } +impl HostResource for IoResource { + fn begin_close(&mut self, _reason: ResourceCloseReason) -> ResourceResult { + self.close_nonblocking() } -} -impl Drop for IoResource { - fn drop(&mut self) { - if !self.owner { - return; - } - self.owner_alive.store(false, Ordering::Release); - self.closed.store(true, Ordering::Release); - let pid = self.process_id.load(Ordering::Acquire); - terminate_process_id(pid, ResourceCloseReason::VmDrop); - if let Ok(mut guard) = self.handle.try_lock() { - drop(guard.take()); - } - self.close_waker - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .take(); - } -} - -impl HostResource for IoResource { - fn begin_close(&mut self, reason: ResourceCloseReason) -> ResourceResult { - self.closed.store(true, Ordering::Release); - let pid = self.process_id.load(Ordering::Acquire); - if pid != 0 { - terminate_process_id(pid, reason); - } - self.schedule_close(reason); - if self.active_operations.load(Ordering::Acquire) != 0 { - return Ok(CloseProgress::Pending); - } - match self.handle.try_lock() { - Ok(guard) if guard.is_none() => { - self.close_future = None; - self.process_id.store(0, Ordering::Release); - Ok(CloseProgress::Ready) + fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll> { + match self.close_nonblocking() { + Ok(CloseProgress::Ready) => Poll::Ready(Ok(())), + Ok(CloseProgress::Pending) => { + cx.waker().wake_by_ref(); + Poll::Pending } - Ok(_) | Err(_) => Ok(CloseProgress::Pending), + Err(error) => Poll::Ready(Err(error)), } } +} - fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll> { - if self.wait_for_operations(cx) { - return Poll::Pending; - } - if tokio::runtime::Handle::try_current().is_err() { - // A close future must not be discarded while an operation is still - // active. Once operations are quiescent, there is no reactor in - // which to flush/finish the future, so report a concrete cleanup - // error and let the resource table decide how to retire the slot. - return Poll::Ready(Err(ResourceError::new( - ResourceErrorCode::ResourceCleanupFailed, - "io::resource", - "async IO close requires a Tokio runtime", - ))); - } - let Some(close_future) = self.close_future.as_mut() else { - return Poll::Ready(Ok(())); - }; - match close_future.as_mut().poll(cx) { - Poll::Pending => Poll::Pending, - Poll::Ready(Ok(())) => { - self.close_future = None; - Poll::Ready(Ok(())) - } - Poll::Ready(Err(error)) => { - self.close_future = None; - Poll::Ready(Err(ResourceError::new( +fn start_close_io_handle(handle: &mut IoHandle) -> ResourceResult<()> { + match handle { + IoHandle::File(_) => Ok(()), + IoHandle::PopenRead { child, .. } | IoHandle::PopenWrite { child, .. } => { + terminate_process_id(child.id().unwrap_or(0)); + match child.start_kill() { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::InvalidInput => Ok(()), + Err(error) => Err(ResourceError::new( ResourceErrorCode::ResourceCleanupFailed, "io::resource", - error.to_string(), - ))) + format!("io_close popen terminate failed: {error}"), + )), } } } } -async fn close_io_handle(mut handle: IoHandle, reason: ResourceCloseReason) -> VmResult<()> { +async fn close_io_handle(mut handle: IoHandle) -> VmResult<()> { match &mut handle { IoHandle::File(file) => { file.get_mut() @@ -357,32 +167,47 @@ async fn close_io_handle(mut handle: IoHandle, reason: ResourceCloseReason) -> V .map_err(|error| VmError::HostError(format!("io_close flush failed: {error}")))?; } IoHandle::PopenRead { child, .. } => { - terminate_process_id(child.id().unwrap_or(0), reason); - child.kill().await.map_err(|error| { - VmError::HostError(format!("io_close popen wait failed: {error}")) - })?; + terminate_process_id(child.id().unwrap_or(0)); + kill_and_reap_child(child).await?; } IoHandle::PopenWrite { child, stdin } => { let _ = stdin.shutdown().await; - terminate_process_id(child.id().unwrap_or(0), reason); - child.kill().await.map_err(|error| { - VmError::HostError(format!("io_close popen wait failed: {error}")) - })?; + terminate_process_id(child.id().unwrap_or(0)); + kill_and_reap_child(child).await?; } } Ok(()) } -fn terminate_process_id(pid: u32, reason: ResourceCloseReason) { +async fn kill_and_reap_child(child: &mut Child) -> VmResult<()> { + match child.kill().await { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::InvalidInput => { + child.wait().await.map(|_| ()).map_err(|wait_error| { + VmError::HostError(format!("io_close popen wait failed: {wait_error}")) + }) + } + Err(error) => Err(VmError::HostError(format!( + "io_close popen wait failed: {error}" + ))), + } +} + +fn process_id(handle: &IoHandle) -> u32 { + match handle { + IoHandle::PopenRead { child, .. } | IoHandle::PopenWrite { child, .. } => { + child.id().unwrap_or(0) + } + IoHandle::File(_) => 0, + } +} + +fn terminate_process_id(pid: u32) { if pid == 0 { return; } - let _ = reason; #[cfg(unix)] - { - let Ok(pid) = libc::pid_t::try_from(pid) else { - return; - }; + if let Ok(pid) = libc::pid_t::try_from(pid) { unsafe { libc::kill(-pid, libc::SIGKILL); } @@ -411,11 +236,10 @@ impl CaptureAsyncHostContext for IoPolicyContext { } } -/// The per-call captured handle context: shared resource cells plus the -/// policy byte limits, captured before the future is submitted. +/// Shared handle state and byte limits captured before an async call starts. pub(crate) struct IoHandleContext { handle: ResourceHandle, - resource: IoResource, + state: Arc, max_read_bytes: Option, max_write_bytes: Option, } @@ -434,10 +258,10 @@ impl CaptureAsyncHostContext for IoHandleContext { None => return Err(VmError::HostError("missing io handle argument".to_string())), }; let handle = io_parse_handle(handle_id)?; - let resource = io_resource_for_handle(vm, handle)?; + let state = io_state_for_handle(vm, handle)?; Ok(Self { handle, - resource, + state, max_read_bytes: io_policy(vm).map(|policy| policy.max_read_bytes), max_write_bytes: io_policy(vm).map(|policy| policy.max_write_bytes), }) @@ -456,7 +280,7 @@ pub(crate) async fn builtin_io_open( "w" | "a" | "r+" | "w+" | "a+" => true, other => { return Err(VmError::HostError(format!( - "io_open unsupported mode '{other}'" + "unsupported io_open mode '{other}', expected r/w/a/r+/w+/a+" ))); } }; @@ -481,7 +305,7 @@ pub(crate) async fn builtin_io_open( "a+" => { options.read(true).write(true).create(true).append(true); } - _ => unreachable!(), + _ => unreachable!("mode validated above"), } let file = options .open(path) @@ -506,16 +330,16 @@ pub(crate) async fn builtin_io_popen( ) -> VmResult> { if mode != "r" && mode != "w" { return Err(VmError::HostError(format!( - "io_popen unsupported mode '{mode}'" + "unsupported io_popen mode '{mode}', expected r or w" ))); } - if !context + if context .policy .as_ref() - .is_none_or(|policy| policy.allow_process) + .is_some_and(|policy| !policy.allow_process) { return Err(VmError::HostError( - "io_popen requires the command capability".to_string(), + "io_popen requires the process capability".to_string(), )); } let handle = spawn_shell_command(&command, &mode)?; @@ -534,14 +358,10 @@ pub(crate) async fn builtin_io_read_all( #[pd_host_context] context: IoHandleContext, _handle_id: i64, ) -> VmResult> { - let mut lease = context.resource.begin_operation("io_read_all")?; - let mut guard = context.resource.handle.lock().await; - if context.resource.closed.load(Ordering::Acquire) { - return Err(VmError::HostError( - "io_read_all handle is closed".to_string(), - )); - } - let handle = guard + context.state.ensure_open("io_read_all")?; + let mut slot = context.state.handle.lock().await; + context.state.ensure_open("io_read_all")?; + let handle = slot .as_mut() .ok_or_else(|| VmError::HostError("io handle is closed".to_string()))?; let mut out = String::new(); @@ -550,16 +370,11 @@ pub(crate) async fn builtin_io_read_all( IoHandle::PopenRead { stdout, .. } => stdout.read_to_string(&mut out).await, IoHandle::PopenWrite { .. } => { return Err(VmError::HostError( - "io_read_all cannot read from a write handle".to_string(), + "io_read_all requires a readable handle".to_string(), )); } } .map_err(|error| VmError::HostError(format!("io_read_all failed: {error}")))?; - if context.resource.closed.load(Ordering::Acquire) { - return Err(VmError::HostError( - "io_read_all handle is closed".to_string(), - )); - } if context .max_read_bytes .is_some_and(|limit| out.len() > limit) @@ -568,7 +383,6 @@ pub(crate) async fn builtin_io_read_all( "io_read_all exceeded read limit".to_string(), )); } - lease.complete(); Ok(HostFutureOutput::returning(out)) } @@ -578,14 +392,10 @@ pub(crate) async fn builtin_io_read_line( #[pd_host_context] context: IoHandleContext, _handle_id: i64, ) -> VmResult> { - let mut lease = context.resource.begin_operation("io_read_line")?; - let mut guard = context.resource.handle.lock().await; - if context.resource.closed.load(Ordering::Acquire) { - return Err(VmError::HostError( - "io_read_line handle is closed".to_string(), - )); - } - let handle = guard + context.state.ensure_open("io_read_line")?; + let mut slot = context.state.handle.lock().await; + context.state.ensure_open("io_read_line")?; + let handle = slot .as_mut() .ok_or_else(|| VmError::HostError("io handle is closed".to_string()))?; let mut line = String::new(); @@ -594,16 +404,11 @@ pub(crate) async fn builtin_io_read_line( IoHandle::PopenRead { stdout, .. } => stdout.read_line(&mut line).await, IoHandle::PopenWrite { .. } => { return Err(VmError::HostError( - "io_read_line cannot read from a write handle".to_string(), + "io_read_line requires a readable handle".to_string(), )); } } .map_err(|error| VmError::HostError(format!("io_read_line failed: {error}")))?; - if context.resource.closed.load(Ordering::Acquire) { - return Err(VmError::HostError( - "io_read_line handle is closed".to_string(), - )); - } if context .max_read_bytes .is_some_and(|limit| line.len() > limit) @@ -612,7 +417,6 @@ pub(crate) async fn builtin_io_read_line( "io_read_line exceeded read limit".to_string(), )); } - lease.complete(); Ok(HostFutureOutput::returning(line)) } @@ -631,12 +435,10 @@ pub(crate) async fn builtin_io_write( "io_write exceeded write limit".to_string(), )); } - let mut lease = context.resource.begin_operation("io_write")?; - let mut guard = context.resource.handle.lock().await; - if context.resource.closed.load(Ordering::Acquire) { - return Err(VmError::HostError("io_write handle is closed".to_string())); - } - let handle = guard + context.state.ensure_open("io_write")?; + let mut slot = context.state.handle.lock().await; + context.state.ensure_open("io_write")?; + let handle = slot .as_mut() .ok_or_else(|| VmError::HostError("io handle is closed".to_string()))?; let written = match handle { @@ -644,15 +446,11 @@ pub(crate) async fn builtin_io_write( IoHandle::PopenWrite { stdin, .. } => stdin.write(text.as_bytes()).await, IoHandle::PopenRead { .. } => { return Err(VmError::HostError( - "io_write cannot write to a read handle".to_string(), + "io_write requires a writable handle".to_string(), )); } } .map_err(|error| VmError::HostError(format!("io_write failed: {error}")))?; - if context.resource.closed.load(Ordering::Acquire) { - return Err(VmError::HostError("io_write handle is closed".to_string())); - } - lease.complete(); Ok(HostFutureOutput::returning(written as i64)) } @@ -662,12 +460,10 @@ pub(crate) async fn builtin_io_flush( #[pd_host_context] context: IoHandleContext, _handle_id: i64, ) -> VmResult> { - let mut lease = context.resource.begin_operation("io_flush")?; - let mut guard = context.resource.handle.lock().await; - if context.resource.closed.load(Ordering::Acquire) { - return Err(VmError::HostError("io_flush handle is closed".to_string())); - } - let handle = guard + context.state.ensure_open("io_flush")?; + let mut slot = context.state.handle.lock().await; + context.state.ensure_open("io_flush")?; + let handle = slot .as_mut() .ok_or_else(|| VmError::HostError("io handle is closed".to_string()))?; match handle { @@ -676,10 +472,6 @@ pub(crate) async fn builtin_io_flush( IoHandle::PopenRead { .. } => Ok(()), } .map_err(|error| VmError::HostError(format!("io_flush failed: {error}")))?; - if context.resource.closed.load(Ordering::Acquire) { - return Err(VmError::HostError("io_flush handle is closed".to_string())); - } - lease.complete(); Ok(HostFutureOutput::returning(true)) } @@ -689,15 +481,21 @@ pub(crate) async fn builtin_io_close( #[pd_host_context] context: IoHandleContext, _handle_id: i64, ) -> VmResult> { - let mut lease = context.resource.begin_operation("io_close")?; - let resource = IoResource::new_shared(&context.resource); - let handle = context.handle; - let owned_handle = resource.take_handle().await?; - let close_result = close_io_handle(owned_handle, ResourceCloseReason::Requested).await; + if context.state.closed.swap(true, Ordering::AcqRel) { + return Err(VmError::HostError("io_close handle is closed".to_string())); + } + let owned = context + .state + .handle + .lock() + .await + .take() + .ok_or_else(|| VmError::HostError("io handle is closed".to_string()))?; + let close_result = close_io_handle(owned).await; if close_result.is_ok() { - context.resource.process_id.store(0, Ordering::Release); + context.state.process_id.store(0, Ordering::Release); } - lease.complete(); + let handle = context.handle; Ok(HostFutureOutput::complete(move |vm| { let progress = vm .execution_scope() @@ -777,8 +575,6 @@ async fn canonicalize_io_target(path: &Path) -> VmResult { .await .map_err(|error| VmError::HostError(format!("io path resolution failed: {error}"))); } - // The target does not exist yet (e.g. a create-mode open): canonicalize - // the parent and append the final component. let parent = path.parent().unwrap_or_else(|| Path::new(".")); let canonical_parent = tokio::fs::canonicalize(parent) .await @@ -789,10 +585,7 @@ async fn canonicalize_io_target(path: &Path) -> VmResult { Ok(canonical_parent.join(name)) } -/// Looks up the shared cells of a live IO handle resource in the execution -/// scope, cloning them so the async builtin can take/restore the handle -/// while the resource stays in the scope table. -fn io_resource_for_handle(vm: &mut Vm, handle: ResourceHandle) -> VmResult { +fn io_state_for_handle(vm: &mut Vm, handle: ResourceHandle) -> VmResult> { let token = vm .execution_scope() .resources() @@ -813,7 +606,7 @@ fn io_resource_for_handle(vm: &mut Vm, handle: ResourceHandle) -> VmResult VmResult { @@ -836,11 +629,9 @@ fn spawn_shell_command(shell_command: &str, mode: &str) -> VmResult { command.arg("-c").arg(shell_command); command }; - #[cfg(unix)] process.process_group(0); process.kill_on_drop(true); - match mode { "r" => { process.stdout(Stdio::piped()).stdin(Stdio::null()); @@ -848,16 +639,14 @@ fn spawn_shell_command(shell_command: &str, mode: &str) -> VmResult { "w" => { process.stdin(Stdio::piped()).stdout(Stdio::null()); } - _ => {} + _ => unreachable!("mode validated above"), } - let mut child = process .spawn() - .map_err(|error| VmError::HostError(format!("io_popen spawn failed: {error}")))?; - + .map_err(|error| VmError::HostError(format!("io_popen failed: {error}")))?; if mode == "r" { let Some(stdout) = child.stdout.take() else { - terminate_process_id(child.id().unwrap_or(0), ResourceCloseReason::VmDrop); + terminate_process_id(child.id().unwrap_or(0)); let _ = child.start_kill(); return Err(VmError::HostError( "io_popen('r') did not provide stdout pipe".to_string(), @@ -869,7 +658,7 @@ fn spawn_shell_command(shell_command: &str, mode: &str) -> VmResult { }) } else { let Some(stdin) = child.stdin.take() else { - terminate_process_id(child.id().unwrap_or(0), ResourceCloseReason::VmDrop); + terminate_process_id(child.id().unwrap_or(0)); let _ = child.start_kill(); return Err(VmError::HostError( "io_popen('w') did not provide stdin pipe".to_string(), @@ -878,143 +667,3 @@ fn spawn_shell_command(shell_command: &str, mode: &str) -> VmResult { Ok(IoHandle::PopenWrite { child, stdin }) } } - -#[cfg(test)] -mod tests { - use std::task::{Context, Poll, Waker}; - - use super::*; - use crate::return_one; - - fn file_resource() -> IoResource { - let file = std::fs::File::open("Cargo.toml").expect("test fixture should exist"); - IoResource::new(IoHandle::File(BufReader::new(File::from_std(file)))) - } - - async fn assert_close_waits_for_busy_handle_lock() { - let mut resource = file_resource(); - let handle = Arc::clone(&resource.handle); - let guard = handle.lock().await; - let lease = resource - .begin_operation("test") - .expect("test operation should start"); - let reason = ResourceCloseReason::Requested; - - assert_eq!( - resource.begin_close(reason).expect("close should start"), - CloseProgress::Pending, - "close must stay pending while an async operation owns the handle lock" - ); - - let mut cx = Context::from_waker(Waker::noop()); - assert!(matches!(resource.poll_close(&mut cx), Poll::Pending)); - - drop(guard); - drop(lease); - assert!(matches!(resource.poll_close(&mut cx), Poll::Ready(Ok(())))); - } - - #[tokio::test] - async fn async_io_close_while_read_lock_is_busy_stays_pending() { - assert_close_waits_for_busy_handle_lock().await; - } - - #[tokio::test] - async fn async_io_close_while_write_lock_is_busy_stays_pending() { - assert_close_waits_for_busy_handle_lock().await; - } - - #[test] - fn async_io_close_without_runtime_waits_for_active_operations_then_reports_error() { - let mut resource = file_resource(); - let lease = resource - .begin_operation("test") - .expect("test operation should start"); - assert_eq!( - resource - .begin_close(ResourceCloseReason::Requested) - .expect("close should start"), - CloseProgress::Pending - ); - - let mut cx = Context::from_waker(Waker::noop()); - assert!( - matches!(resource.poll_close(&mut cx), Poll::Pending), - "a no-runtime close must not release an active IO handle" - ); - - drop(lease); - match resource.poll_close(&mut cx) { - Poll::Ready(Err(error)) => { - assert_eq!(error.code(), ResourceErrorCode::ResourceCleanupFailed); - } - other => panic!("no-runtime close should surface a concrete error, got {other:?}"), - } - } - - #[cfg(unix)] - #[tokio::test] - async fn async_io_child_close_polls_until_child_is_reaped() { - let mut resource = IoResource::new(spawn_shell_command("sleep 30", "r").expect("spawn")); - let pid = resource.process_id.load(Ordering::Acquire); - assert_ne!(pid, 0); - - assert_eq!( - resource - .begin_close(ResourceCloseReason::Requested) - .expect("close should start"), - CloseProgress::Pending - ); - - std::future::poll_fn(|cx| resource.poll_close(cx)) - .await - .expect("child close should succeed"); - assert!( - !std::path::Path::new(&format!("/proc/{pid}")).exists(), - "poll_close must wait for the child to be reaped" - ); - } - - #[tokio::test] - async fn async_io_close_propagates_scope_retirement_errors() { - let compiled = crate::compile_source("0;").expect("test program should compile"); - let mut vm = Vm::new(compiled.program); - let resource = IoResource::new(spawn_shell_command("sleep 30", "r").expect("spawn")); - let shared = IoResource::new_shared(&resource); - let token = vm - .execution_scope() - .push_resource(resource) - .expect("resource should insert"); - let context = IoHandleContext { - handle: token.handle(), - resource: shared, - max_read_bytes: None, - max_write_bytes: None, - }; - let mut close_future = - Box::pin(builtin_io_close_impl(context, token.handle().raw() as i64)); - let mut cx = Context::from_waker(Waker::noop()); - - assert!(matches!(close_future.as_mut().poll(&mut cx), Poll::Pending)); - assert_eq!( - vm.execution_scope() - .close_resource::(token.handle(), ResourceCloseReason::Requested) - .expect("concurrent close should start"), - CloseProgress::Pending - ); - - let output = close_future - .await - .expect("close future should complete") - .map(return_one); - let error = output - .finish(&mut vm) - .expect_err("scope retirement failure must reach the guest"); - assert!( - error.to_string().contains("already closed") - || error.to_string().contains("closing") - || error.to_string().contains("resource"), - "unexpected scope retirement error: {error}" - ); - } -} diff --git a/src/builtins/runtime/io/blocking.rs b/src/builtins/runtime/io/blocking.rs index bc16faf0..373ae994 100644 --- a/src/builtins/runtime/io/blocking.rs +++ b/src/builtins/runtime/io/blocking.rs @@ -1,24 +1,14 @@ use std::fs::OpenOptions; use std::io::{Read, Write}; -use std::ops::{Deref, DerefMut}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex}; -use std::task::{Context, Poll, Waker}; -use std::thread::JoinHandle; use pd_host_function::pd_host_function; -use super::HostCallResult; -use crate::vm::operation::driver::HostOperation; -use crate::vm::operation::error::{OperationError, OperationErrorCode, OperationResult}; -use crate::vm::operation::reason::OperationCancelReason; -use crate::vm::operation::{OperationId, OperationOutcome, OperationSpec}; use crate::vm::resource::close::{CloseProgress, HostResource}; use crate::vm::resource::error::{ResourceError, ResourceErrorCode, ResourceResult}; -use crate::vm::resource::{ResourceCloseReason, ResourceHandle}; -use crate::vm::{CallReturn, HostOpId, Value, Vm, VmError, VmResult}; +use crate::vm::resource::{Resource, ResourceCloseReason, ResourceHandle}; +use crate::vm::{Vm, VmError, VmResult}; /// A file / child-process backed IO handle. pub(super) enum IoHandle { @@ -27,230 +17,27 @@ pub(super) enum IoHandle { PopenWrite { child: Child }, } -/// Shared lifecycle state for one typed IO resource. -/// -/// The handle cell is also the admission lock for workers: a worker increments -/// `active_workers` while holding the cell lock before taking the handle, and a -/// close marks the resource closed before inspecting that same cell. This -/// makes a close racing with a worker either reject the worker or observe it as -/// active; it can never mistake an owned handle for an idle resource. -struct IoResourceState { - handle: Mutex>, - closed: AtomicBool, - active_workers: AtomicUsize, - close_waker: Mutex>, - close_error: Mutex>, +/// The typed resource stored in the execution scope for one IO handle. +struct IoResource { + handle: Option, } -impl IoResourceState { +impl IoResource { fn new(handle: IoHandle) -> Self { Self { - handle: Mutex::new(Some(handle)), - closed: AtomicBool::new(false), - active_workers: AtomicUsize::new(0), - close_waker: Mutex::new(None), - close_error: Mutex::new(None), - } - } - - /// Takes the handle for one worker and records its ownership before - /// releasing the admission lock. - fn take_handle(self: &Arc) -> Option { - let mut slot = self - .handle - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - if self.closed.load(Ordering::Acquire) { - return None; - } - let handle = slot.take()?; - self.active_workers.fetch_add(1, Ordering::AcqRel); - Some(IoHandleLease { - state: Arc::clone(self), handle: Some(handle), - active: true, - }) - } - - fn mark_closed(&self) { - let _guard = self - .handle - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - self.closed.store(true, Ordering::Release); - } - - fn register_close_waker(&self, waker: &Waker) { - let mut guard = self - .close_waker - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - if self.active_workers.load(Ordering::Acquire) == 0 { - return; - } - *guard = Some(waker.clone()); - if self.active_workers.load(Ordering::Acquire) == 0 - && let Some(waker) = guard.take() - { - waker.wake(); } } - - fn release_worker(&self) { - let previous = self.active_workers.fetch_sub(1, Ordering::AcqRel); - debug_assert!(previous > 0, "IO worker release without an active worker"); - if previous == 1 - && let Some(waker) = self - .close_waker - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .take() - { - waker.wake(); - } - } - - fn record_close_error(&self, error: &VmError) { - let mut guard = self - .close_error - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - if guard.is_none() { - *guard = Some(error.to_string()); - } - } - - fn cleanup_error(&self) -> Option { - self.close_error - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .as_ref() - .map(|message| { - ResourceError::new( - ResourceErrorCode::ResourceCleanupFailed, - "io::resource", - message.clone(), - ) - }) - } } -impl Drop for IoResourceState { +impl Drop for IoResource { fn drop(&mut self) { - let handle = self - .handle - .get_mut() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .take(); - if let Some(handle) = handle { + if let Some(handle) = self.handle.take() { let _ = close_io_handle(handle); } } } -/// A worker-owned handle lease. Normal completion explicitly restores the -/// handle to an open resource, while close/cancellation/unwind paths close it -/// instead. In either case the active-worker count is decremented and a -/// pending resource close is woken. -struct IoHandleLease { - state: Arc, - handle: Option, - active: bool, -} - -impl Deref for IoHandleLease { - type Target = IoHandle; - - fn deref(&self) -> &Self::Target { - self.handle.as_ref().expect("active IO lease has a handle") - } -} - -impl DerefMut for IoHandleLease { - fn deref_mut(&mut self) -> &mut Self::Target { - self.handle.as_mut().expect("active IO lease has a handle") - } -} - -impl IoHandleLease { - fn restore(mut self) -> VmResult<()> { - self.release_inner(false) - } - - fn close(mut self) -> VmResult<()> { - self.release_inner(true) - } - - fn release_inner(&mut self, force_close: bool) -> VmResult<()> { - if !self.active { - return Ok(()); - } - let Some(handle) = self.handle.take() else { - self.active = false; - self.state.release_worker(); - return Ok(()); - }; - - let mut handle = Some(handle); - let should_close = if force_close { - true - } else { - let mut slot = self - .state - .handle - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - if self.state.closed.load(Ordering::Acquire) { - true - } else { - *slot = handle.take(); - false - } - }; - - let result = if should_close { - close_io_handle(handle.expect("IO lease close owns its handle")) - } else { - Ok(()) - }; - if let Err(error) = &result { - self.state.record_close_error(error); - } - self.active = false; - self.state.release_worker(); - result - } -} - -impl Drop for IoHandleLease { - fn drop(&mut self) { - if self.active { - // A normal worker calls `restore`/`close` explicitly. Reaching this - // guard means an unwind or failed handoff, so never return a live - // process handle to the resource table implicitly. - let _ = self.release_inner(true); - } - } -} - -/// The typed resource stored in the execution scope for one IO handle. -struct IoResource { - state: Arc, -} - -impl IoResource { - fn new(handle: IoHandle) -> Self { - Self { - state: Arc::new(IoResourceState::new(handle)), - } - } - - /// Takes the inner handle for a worker thread and records its lease. - fn take_handle(&self) -> Option { - self.state.take_handle() - } -} - impl crate::host_extension::HostResourceType for IoResource { const KEY: &'static str = super::IO_FILE_KEY; const DESCRIPTION: &'static str = super::IO_FILE_DESCRIPTION; @@ -263,27 +50,8 @@ pub(crate) fn io_file_resource() -> crate::host_extension::HostResourceTypeMeta impl HostResource for IoResource { fn begin_close(&mut self, _reason: ResourceCloseReason) -> ResourceResult { - // Marking closed while holding the same admission lock used by worker - // leases makes the close boundary linearizable: a worker either - // restores before close begins, or observes closed and cleans up. - let mut slot = self - .state - .handle - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - self.state.closed.store(true, Ordering::Release); - if self.state.active_workers.load(Ordering::Acquire) != 0 { - return Ok(CloseProgress::Pending); - } - let handle = slot.take(); - drop(slot); - - if let Some(error) = self.state.cleanup_error() { - return Err(error); - } - if let Some(handle) = handle { + if let Some(handle) = self.handle.take() { close_io_handle(handle).map_err(|error| { - self.state.record_close_error(&error); ResourceError::new( ResourceErrorCode::ResourceCleanupFailed, "io::resource", @@ -293,561 +61,56 @@ impl HostResource for IoResource { } Ok(CloseProgress::Ready) } - - fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll> { - if self.state.active_workers.load(Ordering::Acquire) != 0 { - self.state.register_close_waker(cx.waker()); - if self.state.active_workers.load(Ordering::Acquire) != 0 { - return Poll::Pending; - } - } - - if let Some(error) = self.state.cleanup_error() { - return Poll::Ready(Err(error)); - } - let handle = self - .state - .handle - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .take(); - if let Some(handle) = handle - && let Err(error) = close_io_handle(handle) - { - self.state.record_close_error(&error); - return Poll::Ready(Err(ResourceError::new( - ResourceErrorCode::ResourceCleanupFailed, - "io::resource", - error.to_string(), - ))); - } - Poll::Ready(Ok(())) - } -} - -/// Shared state between one IO worker thread, its [`IoOpDriver`] operation, -/// and the adapter-owned completion hook on the VM thread. -/// -/// The worker writes the terminal [`signal`](IoOpShared::signal), the -/// guest-visible [`value`](IoOpShared::value), and any opened handle or -/// close target; the driver reflects the signal into the operation registry -/// and the VM wrapper reads the value out of the mailbox after the registry -/// drive returns terminal. -struct IoOpShared { - cancelled: AtomicBool, - worker_done: AtomicBool, - signal: Mutex>>, - value: Mutex>>, - opened: Mutex>, - target: Mutex>, - waker: Mutex>, - quiescence_waker: Mutex>, - worker: Mutex>>, - cancel_hook: Mutex>>, } -impl IoOpShared { - fn new() -> Self { - Self { - cancelled: AtomicBool::new(false), - worker_done: AtomicBool::new(false), - signal: Mutex::new(None), - value: Mutex::new(None), - opened: Mutex::new(None), - target: Mutex::new(None), - waker: Mutex::new(None), - quiescence_waker: Mutex::new(None), - worker: Mutex::new(None), - cancel_hook: Mutex::new(None), - } - } - - fn mark_worker_done(&self) { - self.worker_done.store(true, Ordering::Release); - if let Some(waker) = self - .quiescence_waker - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .take() - { - waker.wake(); - } - } - - fn is_quiescent(&self) -> bool { - self.worker_done.load(Ordering::Acquire) - } - - fn register_quiescence_waker(&self, waker: &Waker) { - let mut guard = self - .quiescence_waker - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - if self.is_quiescent() { - return; - } - *guard = Some(waker.clone()); - if self.is_quiescent() - && let Some(waker) = guard.take() - { - waker.wake(); - } - } - - fn install_cancel_hook(&self, hook: impl FnOnce() + Send + 'static) { - let mut hook = Some(Box::new(hook) as Box); - { - let mut guard = self - .cancel_hook - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - if !self.cancelled.load(Ordering::Acquire) { - *guard = hook.take(); - } - } - if let Some(hook) = hook { - hook(); - } - } - - fn cancel_work(&self) { - if let Some(hook) = self - .cancel_hook - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .take() - { - hook(); - } - } - - fn set_worker(&self, worker: JoinHandle<()>) { - *self - .worker - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(worker); - } - - fn join_worker(&self) -> bool { - let worker = self - .worker - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .take(); - worker.is_some_and(|worker| worker.join().is_err()) - } - - /// The worker's terminal publish: stores the signal and wakes any - /// registered waker (check-register-double-check in the driver's poll). - fn publish(&self, signal: Result<(), String>) { - *self - .signal - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(signal); - if let Some(waker) = self - .waker - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .take() - { - waker.wake(); - } - } - - fn take_signal(&self) -> Option> { - self.signal - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .take() - } - - fn register_waker(&self, waker: &Waker) { - *self - .waker - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(waker.clone()); - } - - /// The worker's failure path: records the guest-visible `VmError` in the - /// value mailbox and publishes a textual signal for the operation driver. - fn fail(&self, error: VmError) { - let message = error.to_string(); - *self - .value - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(Err(error)); - self.publish(Err(message)); - } - - /// Publishes a terminal operation while preserving a guest-visible value - /// (including an error that must still retire a close target). - fn complete(&self, value: VmResult) { - *self - .value - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(value); - self.publish(Ok(())); - } - - /// The worker's success path: records the guest-visible value and - /// publishes a success signal. - fn succeed(&self, value: CallReturn) { - self.complete(Ok(value)); - } -} - -impl Drop for IoOpShared { - fn drop(&mut self) { - let handle = self - .opened - .get_mut() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .take(); - if let Some(handle) = handle { - let _ = close_io_handle(handle); - } - } -} - -/// A concrete [`HostOperation`] driver for one pending IO operation. -/// -/// The worker thread performs the actual IO; this driver reflects the -/// worker's terminal signal into the operation registry and honours -/// cancellation by flagging the shared state so the worker aborts promptly. -struct IoOpDriver { - shared: Arc, - name: String, -} - -impl IoOpDriver { - fn new(shared: Arc, name: impl Into) -> Self { - Self { - shared, - name: name.into(), - } - } - - fn worker_failed(&self, message: impl Into) -> Poll> { - Poll::Ready(Err(OperationError::new( - OperationErrorCode::OperationDriverFailed, - "io::operation", - message, - ))) - } -} - -impl HostOperation for IoOpDriver { - fn poll(&mut self, cx: &mut Context<'_>) -> Poll> { - if !self.shared.is_quiescent() { - self.shared.register_waker(cx.waker()); - self.shared.register_quiescence_waker(cx.waker()); - if !self.shared.is_quiescent() { - return Poll::Pending; - } +/// Opens a file handle for runtime I/O inline on non-async builds. +#[pd_host_function(name = "io::open", contract = super::io_open_contract)] +pub(super) fn builtin_io_open(vm: &mut Vm, path: &str, mode: &str) -> VmResult { + let writes = match mode { + "r" => false, + "w" | "a" | "r+" | "w+" | "a+" => true, + other => { + return Err(VmError::HostError(format!( + "unsupported io_open mode '{other}', expected r/w/a/r+/w+/a+" + ))); } - if self.shared.cancelled.load(Ordering::Acquire) { - return self.worker_failed(format!("{} was cancelled", self.name)); + }; + let path = authorize_blocking_io_path(vm, path, writes)?; + let mut options = OpenOptions::new(); + match mode { + "r" => { + options.read(true); } - match self.shared.take_signal() { - Some(Ok(())) => Poll::Ready(Ok(())), - Some(Err(message)) => self.worker_failed(message), - None => self.worker_failed(format!( - "{} worker terminated without a completion signal", - self.name - )), + "w" => { + options.write(true).create(true).truncate(true); } - } - - fn cancel(&mut self, _reason: OperationCancelReason) -> OperationResult<()> { - self.shared.cancelled.store(true, Ordering::Release); - self.shared.cancel_work(); - Ok(()) - } - - fn is_quiescent(&self) -> bool { - self.shared.is_quiescent() - } - - fn register_quiescence_waker(&mut self, cx: &Context<'_>) { - self.shared.register_quiescence_waker(cx.waker()); - } - - fn cancel_and_wait(&mut self, reason: OperationCancelReason) -> OperationResult<()> { - self.cancel(reason)?; - if self.shared.join_worker() { - return Err(OperationError::new( - OperationErrorCode::OperationDriverFailed, - "io::operation", - format!("{} worker panicked while cancelling", self.name), - )); + "a" => { + options.write(true).create(true).append(true); } - Ok(()) - } -} - -impl Drop for IoOpDriver { - fn drop(&mut self) { - if !self.shared.is_quiescent() { - self.shared.cancelled.store(true, Ordering::Release); - self.shared.cancel_work(); + "r+" => { + options.read(true).write(true); } - let _ = self.shared.join_worker(); - } -} - -/// Completes one operation after the generic scope registry reports a -/// terminal outcome. The adapter owns the mailbox and any resource-table -/// mutation; the VM only invokes this opaque completion hook. -fn finish_io_operation( - vm: &mut Vm, - op_id: OperationId, - outcome: OperationOutcome, - shared: Arc, -) -> VmResult { - if matches!(outcome, OperationOutcome::Cancelled(_)) || shared.cancelled.load(Ordering::Acquire) - { - // The completion hook can be discarded after cancellation. Clean up - // an opened child here as well as in `IoOpShared::drop`, so ownership - // is released as soon as the worker has quiesced. - if let Some(handle) = shared - .opened - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .take() - { - let _ = close_io_handle(handle); + "w+" => { + options.read(true).write(true).create(true).truncate(true); } - return Err(VmError::HostError("IO operation cancelled".to_string())); - } - - // An opened handle (io::open / io::popen) becomes a typed IO resource in - // the scope; the script-visible handle is its raw resource token. The - // resource state's drop guard closes the handle if table admission fails. - if let Some(handle) = shared - .opened - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .take() - { - let resource = IoResource::new(handle); - let token = vm - .execution_scope() - .push_resource(resource) - .map_err(|error| { - VmError::HostError(format!( - "scoped operation {} resource insert failed: {error}", - op_id.raw() - )) - })?; - *shared - .value - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = - Some(Ok(CallReturn::one(Value::Int(token.handle().raw() as i64)))); - } - - // A closed handle (io::close) retires the exact resource entry through - // the generic scope close (exact-once). A close operation is successful - // only once both the underlying handle and the scope entry are retired. - if let Some(target) = shared - .target - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .take() - { - let progress = vm - .execution_scope() - .close_resource::(target, ResourceCloseReason::Requested) - .map_err(VmError::ExecutionScope)?; - if progress != CloseProgress::Ready { - return Err(VmError::HostError(format!( - "scoped operation {} resource retirement remained pending", - op_id.raw() - ))); + "a+" => { + options.read(true).write(true).create(true).append(true); } + _ => unreachable!("mode validated above"), } - - let value = shared - .value - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .take(); - value.ok_or_else(|| { - VmError::HostError(format!( - "scoped operation {} completed without a result", - op_id.raw() - )) - })? -} - -/// Maximum UTF-8 byte length passed to `thread::Builder::name` for an IO -/// worker. The sanitized ASCII name also avoids embedded NULs and platform -/// surprises from an operation name supplied by a future caller. -const IO_WORKER_THREAD_NAME_MAX_LEN: usize = 32; - -fn io_worker_thread_name(operation: &str) -> String { - let mut name = String::from("pd-vm-io-"); - for byte in operation.bytes() { - if name.len() == IO_WORKER_THREAD_NAME_MAX_LEN { - break; - } - let safe = match byte { - b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'_' | b'-' => byte, - _ => b'_', - }; - name.push(safe as char); - } - name -} - -struct WorkerCompletion { - shared: Arc, -} - -impl Drop for WorkerCompletion { - fn drop(&mut self) { - self.shared.mark_worker_done(); - } -} - -/// Spawns a worker thread for an IO operation and registers its driver in -/// the VM's execution scope. Returns the packed [`OperationId`] raw value -/// to hand to the guest as the pending op id. -fn schedule_io_task( - vm: &mut Vm, - name: &str, - work: impl FnOnce(&IoOpShared) + Send + 'static, -) -> VmResult { - let name = name.to_string(); - let shared = Arc::new(IoOpShared::new()); - let driver_shared = Arc::clone(&shared); - let worker_shared = Arc::clone(&shared); - let worker_name = name.clone(); - - let op_id = vm + let file = options + .open(path) + .map_err(|error| VmError::HostError(format!("io_open failed: {error}")))?; + let token = vm .execution_scope() - .start_operation(OperationSpec::new(IoOpDriver::new(driver_shared, name))) - .map_err(|error| { - VmError::HostError(format!( - "failed to start io operation '{}': {error}", - worker_name - )) - })?; - if let Err(error) = vm.register_scoped_operation_completion(op_id, { - let completion_shared = Arc::clone(&shared); - move |vm, outcome| finish_io_operation(vm, op_id, outcome, completion_shared) - }) { - let _ = vm - .execution_scope() - .abort_operation(op_id, OperationCancelReason::Requested); - return Err(error); - } - let raw = op_id.raw(); - let thread_name = io_worker_thread_name(&worker_name); - - std::thread::Builder::new() - .name(thread_name) - .spawn(move || { - let _completion = WorkerCompletion { - shared: Arc::clone(&worker_shared), - }; - if worker_shared.cancelled.load(Ordering::Acquire) { - worker_shared.publish(Err(format!("io operation '{worker_name}' was cancelled"))); - return; - } - work(&worker_shared); - }) - .map(|worker| { - shared.set_worker(worker); - }) - .map_err(|error| { - // Roll back the registered operation so no orphaned op lingers. - shared.mark_worker_done(); - vm.discard_scoped_operation_completion(op_id); - let _ = vm - .execution_scope() - .abort_operation(op_id, OperationCancelReason::Requested); - VmError::HostError(format!("failed to spawn io task: {error}")) - })?; - - Ok(raw) -} - -fn finish_io_worker(shared: &IoOpShared, handle: IoHandleLease, result: VmResult) { - let result = match handle.restore() { - Ok(()) => result, - Err(error) => Err(error), - }; - match result { - Ok(value) => shared.succeed(value), - Err(error) => shared.fail(error), - } -} - -/// Opens a file handle for runtime I/O. -#[pd_host_function(name = "io::open", contract = super::io_open_contract)] -pub(super) fn builtin_io_open( - vm: &mut Vm, - path: &str, - mode: &str, -) -> VmResult> { - let writes = matches!(mode, "w" | "a" | "r+" | "w+" | "a+"); - let path = authorize_blocking_io_path(vm, path, writes)? - .display() - .to_string(); - let mode = mode.to_string(); - let op_id = schedule_io_task(vm, "io::open", move |shared| { - let mut options = OpenOptions::new(); - match mode.as_str() { - "r" => { - options.read(true); - } - "w" => { - options.write(true).create(true).truncate(true); - } - "a" => { - options.write(true).create(true).append(true); - } - "r+" => { - options.read(true).write(true); - } - "w+" => { - options.read(true).write(true).create(true).truncate(true); - } - "a+" => { - options.read(true).write(true).create(true).append(true); - } - other => { - shared.fail(VmError::HostError(format!( - "unsupported io_open mode '{other}', expected r/w/a/r+/w+/a+", - ))); - return; - } - } - - match options.open(path) { - Ok(file) => { - *shared - .opened - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(IoHandle::File(file)); - shared.publish(Ok(())); - } - Err(err) => { - shared.fail(VmError::HostError(format!("io_open failed: {err}"))); - } - } - })?; - Ok(HostCallResult::Pending(op_id)) + .push_resource(IoResource::new(IoHandle::File(file))) + .map_err(|error| VmError::HostError(format!("io resource insert failed: {error}")))?; + Ok(token.into_handle().raw() as i64) } /// Starts a child process and returns a process-backed handle. #[pd_host_function(name = "io::popen")] -pub(super) fn builtin_io_popen( - vm: &mut Vm, - command: &str, - mode: &str, -) -> VmResult> { +pub(super) fn builtin_io_popen(vm: &mut Vm, command: &str, mode: &str) -> VmResult { if mode != "r" && mode != "w" { return Err(VmError::HostError(format!( "unsupported io_popen mode '{mode}', expected r or w" @@ -861,136 +124,91 @@ pub(super) fn builtin_io_popen( "io_popen requires the process capability".to_string(), )); } - let command = command.to_string(); - let mode = mode.to_string(); - let op_id = schedule_io_task(vm, "io::popen", move |shared| { - let child = match spawn_shell_command(command.as_str(), mode.as_str()) { - Ok(child) => child, - Err(err) => { - shared.fail(err); - return; - } - }; - let child_guard = SpawnedChildGuard::new(child); - let child_pid = child_guard.id(); - shared.install_cancel_hook(move || terminate_process_tree(child_pid)); - let handle = match mode.as_str() { - "r" => { - if child_guard.stdout_is_none() { - let err = - VmError::HostError("io_popen('r') did not provide stdout pipe".to_string()); - shared.fail(err); - return; - } - IoHandle::PopenRead { - child: child_guard.into_child(), - } - } - "w" => { - if child_guard.stdin_is_none() { - let err = - VmError::HostError("io_popen('w') did not provide stdin pipe".to_string()); - shared.fail(err); - return; - } - IoHandle::PopenWrite { - child: child_guard.into_child(), - } - } - _ => unreachable!("mode validated above"), - }; - *shared - .opened - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(handle); - shared.publish(Ok(())); - })?; - Ok(HostCallResult::Pending(op_id)) + let handle = spawn_shell_command(command, mode)?; + let token = vm + .execution_scope() + .push_resource(IoResource::new(handle)) + .map_err(|error| VmError::HostError(format!("io resource insert failed: {error}")))?; + Ok(token.into_handle().raw() as i64) } /// Reads all remaining text from an I/O handle. #[pd_host_function(name = "io::read_all", contract = super::io_read_all_contract)] -pub(super) fn builtin_io_read_all(vm: &mut Vm, handle_id: i64) -> VmResult> { - let (_handle, resource) = io_resource_for_handle(vm, handle_id)?; - let op_id = schedule_io_task(vm, "io::read_all", move |shared| { - let mut handle = match resource.take_handle() { - Some(handle) => handle, - None => { - let err = VmError::HostError("io_read_all handle is already closing".to_string()); - shared.fail(err); - return; - } - }; - install_process_cancel_hook(shared, &handle, &resource.state); - let mut out = String::new(); - let result = match &mut *handle { - IoHandle::File(file) => file - .read_to_string(&mut out) - .map_err(|err| VmError::HostError(format!("io_read_all failed: {err}"))) - .map(|_| CallReturn::one(Value::string(out))), - IoHandle::PopenRead { child } => match child.stdout.as_mut() { - Some(stdout) => stdout - .read_to_string(&mut out) - .map_err(|err| VmError::HostError(format!("io_read_all failed: {err}"))) - .map(|_| CallReturn::one(Value::string(out))), - None => Err(VmError::HostError( - "io_read_all popen handle missing stdout".to_string(), - )), - }, - IoHandle::PopenWrite { .. } => Err(VmError::HostError( +pub(super) fn builtin_io_read_all(vm: &mut Vm, handle_id: i64) -> VmResult { + let limit = super::io_policy(vm).map(|policy| policy.max_read_bytes); + let token = io_resource_for_handle(vm, handle_id)?; + let mut resource = vm + .execution_scope() + .resources_mut() + .get_mut(&token) + .map_err(|error| io_borrow_error(handle_id, error))?; + let handle = resource + .handle + .as_mut() + .ok_or_else(|| VmError::HostError("io handle is closed".to_string()))?; + let mut out = String::new(); + match handle { + IoHandle::File(file) => file.read_to_string(&mut out), + IoHandle::PopenRead { child } => child + .stdout + .as_mut() + .ok_or_else(|| { + VmError::HostError("io_read_all popen handle missing stdout".to_string()) + })? + .read_to_string(&mut out), + IoHandle::PopenWrite { .. } => { + return Err(VmError::HostError( "io_read_all requires a readable handle".to_string(), - )), - }; - finish_io_worker(shared, handle, result); - })?; - Ok(HostCallResult::Pending(op_id)) + )); + } + } + .map_err(|error| VmError::HostError(format!("io_read_all failed: {error}")))?; + if limit.is_some_and(|limit| out.len() > limit) { + return Err(VmError::HostError( + "io_read_all exceeded read limit".to_string(), + )); + } + Ok(out) } /// Reads a single line of text from an I/O handle. #[pd_host_function(name = "io::read_line")] -pub(super) fn builtin_io_read_line( - vm: &mut Vm, - handle_id: i64, -) -> VmResult> { - let (_handle, resource) = io_resource_for_handle(vm, handle_id)?; - let op_id = schedule_io_task(vm, "io::read_line", move |shared| { - let mut handle = match resource.take_handle() { - Some(handle) => handle, - None => { - let err = VmError::HostError("io_read_line handle is already closing".to_string()); - shared.fail(err); - return; - } - }; - install_process_cancel_hook(shared, &handle, &resource.state); - let result = match &mut *handle { - IoHandle::File(file) => { - read_line_from_reader(file).map(|line| CallReturn::one(Value::string(line))) - } - IoHandle::PopenRead { child } => match child.stdout.as_mut() { - Some(stdout) => { - read_line_from_reader(stdout).map(|line| CallReturn::one(Value::string(line))) - } - None => Err(VmError::HostError( - "io_read_line popen handle missing stdout".to_string(), - )), - }, - IoHandle::PopenWrite { .. } => Err(VmError::HostError( +pub(super) fn builtin_io_read_line(vm: &mut Vm, handle_id: i64) -> VmResult { + let limit = super::io_policy(vm).map(|policy| policy.max_read_bytes); + let token = io_resource_for_handle(vm, handle_id)?; + let mut resource = vm + .execution_scope() + .resources_mut() + .get_mut(&token) + .map_err(|error| io_borrow_error(handle_id, error))?; + let handle = resource + .handle + .as_mut() + .ok_or_else(|| VmError::HostError("io handle is closed".to_string()))?; + let line = match handle { + IoHandle::File(file) => read_line_from_reader(file)?, + IoHandle::PopenRead { child } => { + read_line_from_reader(child.stdout.as_mut().ok_or_else(|| { + VmError::HostError("io_read_line popen handle missing stdout".to_string()) + })?)? + } + IoHandle::PopenWrite { .. } => { + return Err(VmError::HostError( "io_read_line requires a readable handle".to_string(), - )), - }; - finish_io_worker(shared, handle, result); - })?; - Ok(HostCallResult::Pending(op_id)) + )); + } + }; + if limit.is_some_and(|limit| line.len() > limit) { + return Err(VmError::HostError( + "io_read_line exceeded read limit".to_string(), + )); + } + Ok(line) } /// Writes text to an I/O handle. #[pd_host_function(name = "io::write")] -pub(super) fn builtin_io_write( - vm: &mut Vm, - handle_id: i64, - text: &str, -) -> VmResult> { +pub(super) fn builtin_io_write(vm: &mut Vm, handle_id: i64, text: &str) -> VmResult { if super::io_policy(vm) .as_ref() .is_some_and(|policy| text.len() > policy.max_write_bytes) @@ -999,338 +217,111 @@ pub(super) fn builtin_io_write( "io_write exceeded write limit".to_string(), )); } - let bytes = text.as_bytes().to_vec(); - let (_handle, resource) = io_resource_for_handle(vm, handle_id)?; - let op_id = schedule_io_task(vm, "io::write", move |shared| { - let mut handle = match resource.take_handle() { - Some(handle) => handle, - None => { - let err = VmError::HostError("io_write handle is already closing".to_string()); - shared.fail(err); - return; - } - }; - install_process_cancel_hook(shared, &handle, &resource.state); - let result = match &mut *handle { - IoHandle::File(file) => file - .write(&bytes) - .map_err(|err| VmError::HostError(format!("io_write failed: {err}"))) - .map(|written| CallReturn::one(Value::Int(written as i64))), - IoHandle::PopenWrite { child } => match child.stdin.as_mut() { - Some(stdin) => stdin - .write(&bytes) - .map_err(|err| VmError::HostError(format!("io_write failed: {err}"))) - .map(|written| CallReturn::one(Value::Int(written as i64))), - None => Err(VmError::HostError( - "io_write popen handle missing stdin".to_string(), - )), - }, - IoHandle::PopenRead { .. } => Err(VmError::HostError( + let token = io_resource_for_handle(vm, handle_id)?; + let mut resource = vm + .execution_scope() + .resources_mut() + .get_mut(&token) + .map_err(|error| io_borrow_error(handle_id, error))?; + let handle = resource + .handle + .as_mut() + .ok_or_else(|| VmError::HostError("io handle is closed".to_string()))?; + let written = match handle { + IoHandle::File(file) => file.write(text.as_bytes()), + IoHandle::PopenWrite { child } => child + .stdin + .as_mut() + .ok_or_else(|| VmError::HostError("io_write popen handle missing stdin".to_string()))? + .write(text.as_bytes()), + IoHandle::PopenRead { .. } => { + return Err(VmError::HostError( "io_write requires a writable handle".to_string(), - )), - }; - finish_io_worker(shared, handle, result); - })?; - Ok(HostCallResult::Pending(op_id)) + )); + } + } + .map_err(|error| VmError::HostError(format!("io_write failed: {error}")))?; + Ok(written as i64) } /// Flushes buffered output for an I/O handle. #[pd_host_function(name = "io::flush")] -pub(super) fn builtin_io_flush(vm: &mut Vm, handle_id: i64) -> VmResult> { - let (_handle, resource) = io_resource_for_handle(vm, handle_id)?; - let op_id = schedule_io_task(vm, "io::flush", move |shared| { - let mut handle = match resource.take_handle() { - Some(handle) => handle, - None => { - let err = VmError::HostError("io_flush handle is already closing".to_string()); - shared.fail(err); - return; - } - }; - install_process_cancel_hook(shared, &handle, &resource.state); - let result = match &mut *handle { - IoHandle::File(file) => file - .flush() - .map_err(|err| VmError::HostError(format!("io_flush failed: {err}"))) - .map(|_| CallReturn::one(Value::Bool(true))), - IoHandle::PopenWrite { child } => match child.stdin.as_mut() { - Some(stdin) => stdin - .flush() - .map_err(|err| VmError::HostError(format!("io_flush failed: {err}"))) - .map(|_| CallReturn::one(Value::Bool(true))), - None => Err(VmError::HostError( - "io_flush popen handle missing stdin".to_string(), - )), - }, - IoHandle::PopenRead { .. } => Ok(CallReturn::one(Value::Bool(true))), - }; - finish_io_worker(shared, handle, result); - })?; - Ok(HostCallResult::Pending(op_id)) -} - -/// Closes an I/O handle. -#[pd_host_function(name = "io::close", contract = super::io_close_contract)] -pub(super) fn builtin_io_close(vm: &mut Vm, handle_id: i64) -> VmResult> { - let (target, resource) = io_resource_for_handle(vm, handle_id)?; - let op_id = schedule_io_task(vm, "io::close", move |shared| { - // Close the underlying handle exactly once on the worker thread. - let result = match resource.take_handle() { - Some(handle) => { - install_process_cancel_hook(shared, &handle, &resource.state); - handle.close() - } - None => Err(VmError::HostError( - "io_close handle is already closing".to_string(), - )), - }; - *shared - .target - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(target); - match result { - Ok(()) => shared.succeed(CallReturn::one(Value::Bool(true))), - Err(error) => shared.complete(Err(error)), - } - })?; - Ok(HostCallResult::Pending(op_id)) -} - -/// Returns whether a file system path exists. -#[pd_host_function(name = "io::exists")] -pub(super) fn builtin_io_exists(vm: &mut Vm, path: &str) -> VmResult> { - let path = authorize_blocking_io_path(vm, path, false)? - .display() - .to_string(); - let op_id = schedule_io_task(vm, "io::exists", move |shared| { - shared.succeed(CallReturn::one(Value::Bool( - std::path::Path::new(path.as_str()).exists(), - ))); - })?; - Ok(HostCallResult::Pending(op_id)) -} - -struct SpawnedChildGuard { - child: Option, -} - -impl SpawnedChildGuard { - fn new(child: Child) -> Self { - Self { child: Some(child) } - } - - fn id(&self) -> u32 { - self.child.as_ref().expect("child guard owns a child").id() - } - - fn stdout_is_none(&self) -> bool { - self.child - .as_ref() - .expect("child guard owns a child") - .stdout - .is_none() - } - - fn stdin_is_none(&self) -> bool { - self.child - .as_ref() - .expect("child guard owns a child") +pub(super) fn builtin_io_flush(vm: &mut Vm, handle_id: i64) -> VmResult { + let token = io_resource_for_handle(vm, handle_id)?; + let mut resource = vm + .execution_scope() + .resources_mut() + .get_mut(&token) + .map_err(|error| io_borrow_error(handle_id, error))?; + let handle = resource + .handle + .as_mut() + .ok_or_else(|| VmError::HostError("io handle is closed".to_string()))?; + match handle { + IoHandle::File(file) => file.flush(), + IoHandle::PopenWrite { child } => child .stdin - .is_none() - } - - fn into_child(mut self) -> Child { - self.child.take().expect("child guard owns a child") - } -} - -impl Drop for SpawnedChildGuard { - fn drop(&mut self) { - if let Some(child) = self.child.as_mut() { - let _ = terminate_child_tree(child); - } + .as_mut() + .ok_or_else(|| VmError::HostError("io_flush popen handle missing stdin".to_string()))? + .flush(), + IoHandle::PopenRead { .. } => Ok(()), } + .map_err(|error| VmError::HostError(format!("io_flush failed: {error}")))?; + Ok(true) } -fn install_process_cancel_hook( - shared: &IoOpShared, - handle: &IoHandle, - state: &Arc, -) { - let pid = match handle { - IoHandle::PopenRead { child } | IoHandle::PopenWrite { child } => child.id(), - IoHandle::File(_) => return, +/// Closes an I/O handle. +#[pd_host_function(name = "io::close", contract = super::io_close_contract)] +pub(super) fn builtin_io_close(vm: &mut Vm, handle_id: i64) -> VmResult { + let token = io_resource_for_handle(vm, handle_id)?; + let handle = token.handle(); + let owned = { + let mut resource = vm + .execution_scope() + .resources_mut() + .get_mut(&token) + .map_err(|error| io_borrow_error(handle_id, error))?; + resource + .handle + .take() + .ok_or_else(|| VmError::HostError("io handle is closed".to_string()))? }; - let state = Arc::clone(state); - shared.install_cancel_hook(move || { - // A cancelled process operation has already invalidated the process - // stream. Marking the resource closed makes the worker lease reap the - // child instead of restoring a killed, unreaped Child. - state.mark_closed(); - terminate_process_tree(pid); - }); -} - -fn terminate_process_tree(pid: u32) { - let _ = terminate_process_tree_result(pid); -} - -fn terminate_process_tree_result(pid: u32) -> std::io::Result<()> { - #[cfg(unix)] - { - let pid = libc::pid_t::try_from(pid).map_err(|_| { - std::io::Error::new(std::io::ErrorKind::InvalidInput, "invalid child pid") + let close_result = close_io_handle(owned); + let progress = vm + .execution_scope() + .close_resource::(handle, ResourceCloseReason::Requested) + .map_err(|error| { + VmError::HostError(format!("io_close scope retirement failed: {error}")) })?; - // `spawn_shell_command` puts the shell in its own process group, so a - // negative pid terminates the shell and descendants without touching - // the VM process group. - let result = unsafe { libc::kill(-pid, libc::SIGKILL) }; - if result == 0 { - Ok(()) - } else { - let error = std::io::Error::last_os_error(); - if error.raw_os_error() == Some(libc::ESRCH) { - Ok(()) - } else { - Err(error) - } - } - } - #[cfg(windows)] - { - let status = Command::new("taskkill") - .args(["/T", "/F", "/PID", &pid.to_string()]) - .status()?; - if status.success() { - Ok(()) - } else { - Err(std::io::Error::new( - std::io::ErrorKind::Other, - format!("taskkill exited with {status}"), - )) - } - } - #[cfg(not(any(unix, windows)))] - { - let _ = pid; - Ok(()) - } -} - -/// Terminates a child and reaps it. The only `wait` below is reached after a -/// tree termination signal and a direct leader kill have been attempted; an -/// already exited child is reaped by `try_wait` instead. -fn terminate_child_tree(child: &mut Child) -> VmResult<()> { - let tree_error = terminate_process_tree_result(child.id()).err(); - let status = child - .try_wait() - .map_err(|error| VmError::HostError(format!("io_close popen status failed: {error}")))?; - if status.is_some() { - return Ok(()); - } - - let mut reaped = false; - let direct_error = match child.kill() { - Ok(()) => None, - Err(error) if error.kind() == std::io::ErrorKind::InvalidInput => { - if child - .try_wait() - .map_err(|status_error| { - VmError::HostError(format!("io_close popen status failed: {status_error}")) - })? - .is_some() - { - reaped = true; - None - } else { - Some(error) - } - } - Err(error) => Some(error), - }; - if let Some(error) = direct_error { - return Err(VmError::HostError(format!( - "io_close popen terminate failed: {error}" - ))); - } - - if !reaped { - child - .wait() - .map_err(|error| VmError::HostError(format!("io_close popen wait failed: {error}")))?; - } - if let Some(error) = tree_error { - return Err(VmError::HostError(format!( - "io_close popen process-tree terminate failed: {error}" - ))); + if progress != CloseProgress::Ready { + return Err(VmError::HostError( + "io_close scope retirement is still pending".to_string(), + )); } - Ok(()) + close_result?; + Ok(true) } -fn spawn_shell_command(command: &str, mode: &str) -> VmResult { - let mut process = if cfg!(windows) { - let mut cmd = Command::new("cmd"); - cmd.arg("/C").arg(command); - cmd - } else { - let mut cmd = Command::new("sh"); - cmd.arg("-c").arg(command); - cmd - }; - - #[cfg(unix)] - { - use std::os::unix::process::CommandExt as _; - process.process_group(0); - } - - match mode { - "r" => { - process.stdout(Stdio::piped()).stdin(Stdio::null()); - } - "w" => { - process.stdin(Stdio::piped()).stdout(Stdio::null()); - } - _ => {} - } - - process - .spawn() - .map_err(|err| VmError::HostError(format!("io_popen failed: {err}"))) +/// Returns whether a file system path exists. +#[pd_host_function(name = "io::exists")] +pub(super) fn builtin_io_exists(vm: &mut Vm, path: &str) -> VmResult { + Ok(authorize_blocking_io_path(vm, path, false)?.exists()) } -/// Parses a script-visible integer handle into a typed scope token and -/// returns the raw scope handle plus shared resource cells, validating -/// staleness and type through the generic typed table. -fn io_resource_for_handle( - vm: &mut Vm, - handle_id: i64, -) -> VmResult<(ResourceHandle, Arc)> { +fn io_resource_for_handle(vm: &mut Vm, handle_id: i64) -> VmResult> { let handle = io_parse_handle(handle_id)?; - let token = vm - .execution_scope() + vm.execution_scope() .resources() .typed::(handle) .map_err(|error| { VmError::HostError(format!( "io handle {handle_id} is not a live IO handle: {error}" )) - })?; - let resource = vm - .execution_scope() - .resources() - .get::(&token) - .map_err(|error| { - VmError::HostError(format!("io handle {handle_id} borrow failed: {error}")) - })?; - // Clone the shared resource state so the worker can take/restore the - // handle while the resource itself stays in the scope table. - Ok(( - handle, - Arc::new(IoResource { - state: Arc::clone(&resource.state), - }), - )) + }) +} + +fn io_borrow_error(handle_id: i64, error: impl std::fmt::Display) -> VmError { + VmError::HostError(format!("io handle {handle_id} borrow failed: {error}")) } fn io_parse_handle(handle_id: i64) -> VmResult { @@ -1343,8 +334,6 @@ fn io_parse_handle(handle_id: i64) -> VmResult { .map_err(|error| VmError::HostError(format!("invalid io handle id {handle_id}: {error}"))) } -/// Authorizes one IO path against the configured policy, mirroring the -/// async path: a policy with no matching allowed root denies the path. fn authorize_blocking_io_path(vm: &Vm, path: &str, writes: bool) -> VmResult { let requested = PathBuf::from(path); let Some(policy) = super::io_policy(vm) else { @@ -1393,11 +382,57 @@ fn canonicalize_blocking_target(path: &Path) -> VmResult { Ok(canonical_parent.join(name)) } +fn spawn_shell_command(command: &str, mode: &str) -> VmResult { + let mut process = if cfg!(windows) { + let mut shell = Command::new("cmd"); + shell.arg("/C").arg(command); + shell + } else { + let mut shell = Command::new("sh"); + shell.arg("-c").arg(command); + shell + }; + #[cfg(unix)] + { + use std::os::unix::process::CommandExt as _; + process.process_group(0); + } + match mode { + "r" => { + process.stdout(Stdio::piped()).stdin(Stdio::null()); + } + "w" => { + process.stdin(Stdio::piped()).stdout(Stdio::null()); + } + _ => unreachable!("mode validated above"), + } + let mut child = process + .spawn() + .map_err(|error| VmError::HostError(format!("io_popen failed: {error}")))?; + match mode { + "r" if child.stdout.is_some() => Ok(IoHandle::PopenRead { child }), + "w" if child.stdin.is_some() => Ok(IoHandle::PopenWrite { child }), + "r" => { + let _ = terminate_child_tree(&mut child); + Err(VmError::HostError( + "io_popen('r') did not provide stdout pipe".to_string(), + )) + } + "w" => { + let _ = terminate_child_tree(&mut child); + Err(VmError::HostError( + "io_popen('w') did not provide stdin pipe".to_string(), + )) + } + _ => unreachable!("mode validated above"), + } +} + fn close_io_handle(mut handle: IoHandle) -> VmResult<()> { match &mut handle { IoHandle::File(file) => file .flush() - .map_err(|err| VmError::HostError(format!("io_close flush failed: {err}"))), + .map_err(|error| VmError::HostError(format!("io_close flush failed: {error}"))), IoHandle::PopenRead { child } => terminate_child_tree(child), IoHandle::PopenWrite { child } => { let _ = child.stdin.take(); @@ -1406,13 +441,57 @@ fn close_io_handle(mut handle: IoHandle) -> VmResult<()> { } } +fn terminate_child_tree(child: &mut Child) -> VmResult<()> { + terminate_process_tree(child.id()); + match child.try_wait() { + Ok(Some(_)) => return Ok(()), + Ok(None) => {} + Err(error) => { + return Err(VmError::HostError(format!( + "io_close popen status failed: {error}" + ))); + } + } + child + .kill() + .or_else(|error| { + if error.kind() == std::io::ErrorKind::InvalidInput { + Ok(()) + } else { + Err(error) + } + }) + .map_err(|error| VmError::HostError(format!("io_close popen terminate failed: {error}")))?; + child + .wait() + .map_err(|error| VmError::HostError(format!("io_close popen wait failed: {error}")))?; + Ok(()) +} + +fn terminate_process_tree(pid: u32) { + #[cfg(unix)] + if let Ok(pid) = libc::pid_t::try_from(pid) { + unsafe { + libc::kill(-pid, libc::SIGKILL); + } + } + #[cfg(windows)] + { + let _ = Command::new("taskkill") + .args(["/T", "/F", "/PID", &pid.to_string()]) + .status(); + } + #[cfg(not(any(unix, windows)))] + let _ = pid; +} + fn read_line_from_reader(reader: &mut impl Read) -> VmResult { let mut bytes = Vec::new(); let mut one = [0u8; 1]; loop { let read = reader .read(&mut one) - .map_err(|err| VmError::HostError(format!("io_read_line failed: {err}")))?; + .map_err(|error| VmError::HostError(format!("io_read_line failed: {error}")))?; if read == 0 { break; } @@ -1423,367 +502,3 @@ fn read_line_from_reader(reader: &mut impl Read) -> VmResult { } Ok(String::from_utf8_lossy(&bytes).into_owned()) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn io_worker_thread_name_is_sanitized_and_bounded() { - assert_eq!( - io_worker_thread_name("io::read_all"), - "pd-vm-io-io__read_all" - ); - let name = io_worker_thread_name("io::operation/with spaces\0 and a very long suffix"); - assert!(name.len() <= IO_WORKER_THREAD_NAME_MAX_LEN); - assert!(name.is_ascii()); - assert!( - name.bytes() - .all(|byte| byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'-') - ); - assert!(!name.contains('\0')); - } - - #[test] - fn io_driver_waits_for_worker_completion_before_reporting_ready() { - let shared = Arc::new(IoOpShared::new()); - let release = Arc::new(AtomicBool::new(false)); - let worker_shared = Arc::clone(&shared); - let worker_release = Arc::clone(&release); - let worker = std::thread::spawn(move || { - while !worker_release.load(Ordering::Acquire) { - std::thread::yield_now(); - } - worker_shared.publish(Ok(())); - worker_shared.mark_worker_done(); - }); - shared.set_worker(worker); - let mut driver = IoOpDriver::new(Arc::clone(&shared), "io::test"); - let mut cx = Context::from_waker(Waker::noop()); - - assert!(matches!(driver.poll(&mut cx), Poll::Pending)); - assert!(!driver.is_quiescent()); - - release.store(true, Ordering::Release); - while !driver.is_quiescent() { - std::thread::yield_now(); - } - assert!(matches!(driver.poll(&mut cx), Poll::Ready(Ok(())))); - } - - #[test] - fn io_resource_close_stays_pending_while_worker_owns_handle() { - let path = std::env::temp_dir().join(format!( - "pd-vm-blocking-io-resource-close-{}", - std::process::id() - )); - let file = std::fs::File::create(&path).expect("test file should open"); - let mut resource = IoResource::new(IoHandle::File(file)); - let worker_handle = resource.take_handle().expect("worker should take handle"); - let close = resource - .begin_close(ResourceCloseReason::Requested) - .expect("begin close should succeed"); - assert_eq!(close, CloseProgress::Pending); - - let wake_count = Arc::new(AtomicUsize::new(0)); - struct CloseWake(Arc); - impl std::task::Wake for CloseWake { - fn wake(self: Arc) { - self.0.fetch_add(1, Ordering::SeqCst); - } - - fn wake_by_ref(self: &Arc) { - self.0.fetch_add(1, Ordering::SeqCst); - } - } - let waker = Waker::from(Arc::new(CloseWake(Arc::clone(&wake_count)))); - let mut cx = Context::from_waker(&waker); - assert!(matches!( - HostResource::poll_close(&mut resource, &mut cx), - Poll::Pending - )); - drop(worker_handle); - assert_eq!(wake_count.load(Ordering::SeqCst), 1); - - assert!(matches!( - HostResource::poll_close(&mut resource, &mut cx), - Poll::Ready(Ok(())) - )); - let _ = std::fs::remove_file(path); - } - - #[test] - fn io_resource_worker_release_after_close_does_not_restore_handle() { - let path = std::env::temp_dir().join(format!( - "pd-vm-blocking-io-resource-worker-close-{}", - std::process::id() - )); - let file = std::fs::File::create(&path).expect("test file should open"); - let mut resource = IoResource::new(IoHandle::File(file)); - let worker_handle = resource.take_handle().expect("worker should take handle"); - assert_eq!( - resource - .begin_close(ResourceCloseReason::Requested) - .expect("begin close should succeed"), - CloseProgress::Pending - ); - - worker_handle - .restore() - .expect("worker cleanup after close should succeed"); - assert_eq!(resource.state.active_workers.load(Ordering::Acquire), 0); - assert!( - resource - .state - .handle - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .is_none() - ); - let mut cx = Context::from_waker(Waker::noop()); - assert!(matches!( - HostResource::poll_close(&mut resource, &mut cx), - Poll::Ready(Ok(())) - )); - let _ = std::fs::remove_file(path); - } - - #[cfg(unix)] - struct ProcessTreeCleanup { - leader: u32, - descendant: i32, - marker: PathBuf, - } - - #[cfg(unix)] - impl Drop for ProcessTreeCleanup { - fn drop(&mut self) { - terminate_process_tree(self.leader); - unsafe { - libc::kill(self.descendant, libc::SIGKILL); - } - let _ = std::fs::remove_file(&self.marker); - } - } - - #[cfg(unix)] - fn live_popen_for_test() -> (SpawnedChildGuard, PathBuf, i32) { - static TEST_PROCESS_COUNTER: AtomicUsize = AtomicUsize::new(0); - let suffix = TEST_PROCESS_COUNTER.fetch_add(1, Ordering::Relaxed); - let marker = std::env::temp_dir().join(format!( - "pd-vm-blocking-io-popen-{0}-{suffix}.marker", - std::process::id() - )); - let command = format!( - r#"sleep 30 & child=$!; printf '%s\n' "$child" > '{}'; wait "$child""#, - marker.display() - ); - let child = spawn_shell_command(&command, "r").expect("test popen should spawn"); - let guard = SpawnedChildGuard::new(child); - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); - let descendant = loop { - if let Ok(contents) = std::fs::read_to_string(&marker) - && let Ok(pid) = contents.trim().parse::() - { - break pid; - } - assert!( - std::time::Instant::now() < deadline, - "popen test child did not publish its descendant marker" - ); - std::thread::yield_now(); - }; - (guard, marker, descendant) - } - - #[cfg(unix)] - fn process_is_running(pid: i32) -> bool { - let path = format!("/proc/{pid}/stat"); - let Ok(stat) = std::fs::read_to_string(path) else { - return false; - }; - let Some((_, state)) = stat.split_once(") ") else { - return true; - }; - !state.starts_with('Z') - } - - #[cfg(unix)] - fn wait_for_process_exit(pid: i32) { - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); - while process_is_running(pid) { - assert!( - std::time::Instant::now() < deadline, - "popen descendant remained alive after process-tree close" - ); - std::thread::yield_now(); - } - } - - #[cfg(unix)] - #[test] - fn closing_live_popen_terminates_and_reaps_the_process_tree() { - let (child, marker, descendant) = live_popen_for_test(); - let _cleanup = ProcessTreeCleanup { - leader: child.id(), - descendant, - marker: marker.clone(), - }; - close_io_handle(IoHandle::PopenRead { - child: child.into_child(), - }) - .expect("closing a live popen must terminate and reap it"); - wait_for_process_exit(descendant); - let _ = std::fs::remove_file(marker); - } - - #[cfg(unix)] - #[test] - fn failed_worker_lease_drop_terminates_and_reaps_process_tree() { - let (child, marker, descendant) = live_popen_for_test(); - let leader = child.id(); - let _cleanup = ProcessTreeCleanup { - leader, - descendant, - marker: marker.clone(), - }; - let resource = IoResource::new(IoHandle::PopenRead { - child: child.into_child(), - }); - let worker_handle = resource.take_handle().expect("worker should take handle"); - drop(worker_handle); - wait_for_process_exit(descendant); - let _ = std::fs::remove_file(marker); - } - - #[cfg(unix)] - #[test] - fn failed_resource_handoff_terminates_and_reaps_opened_process_tree() { - let (child, marker, descendant) = live_popen_for_test(); - let _cleanup = ProcessTreeCleanup { - leader: child.id(), - descendant, - marker: marker.clone(), - }; - let mut vm = Vm::new(crate::Program::new( - Vec::new(), - vec![crate::OpCode::Ret as u8], - )); - let shared = Arc::new(IoOpShared::new()); - *shared - .opened - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(IoHandle::PopenRead { - child: child.into_child(), - }); - let op_id = vm - .execution_scope() - .start_operation(OperationSpec::new(IoOpDriver::new( - Arc::clone(&shared), - "io::test-handoff", - ))) - .expect("test operation should start"); - vm.execution_scope() - .begin_close(ResourceCloseReason::Requested) - .expect("scope should start closing"); - let error = finish_io_operation(&mut vm, op_id, OperationOutcome::Completed, shared) - .expect_err("resource insertion into a closing scope must fail"); - assert!(error.to_string().contains("resource insert failed")); - wait_for_process_exit(descendant); - let _ = std::fs::remove_file(marker); - } - - #[test] - fn close_completion_does_not_report_success_while_resource_retirement_is_pending() { - let path = std::env::temp_dir().join(format!( - "pd-vm-blocking-io-close-pending-{}", - std::process::id() - )); - let file = std::fs::File::create(&path).expect("test file should open"); - let resource = IoResource::new(IoHandle::File(file)); - let worker_resource = IoResource { - state: Arc::clone(&resource.state), - }; - let mut vm = Vm::new(crate::Program::new( - Vec::new(), - vec![crate::OpCode::Ret as u8], - )); - let token = vm - .execution_scope() - .push_resource(resource) - .expect("resource should insert"); - let worker_handle = worker_resource - .take_handle() - .expect("worker should take handle"); - let shared = Arc::new(IoOpShared::new()); - *shared - .target - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(token.handle()); - *shared - .value - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = - Some(Ok(CallReturn::one(Value::Bool(true)))); - let op_id = vm - .execution_scope() - .start_operation(OperationSpec::new(IoOpDriver::new( - Arc::clone(&shared), - "io::test-close-pending", - ))) - .expect("test operation should start"); - - let error = finish_io_operation(&mut vm, op_id, OperationOutcome::Completed, shared) - .expect_err("pending resource retirement must not report success"); - assert!(error.to_string().contains("remained pending")); - - worker_handle - .restore() - .expect("worker cleanup after close should succeed"); - let mut cx = Context::from_waker(Waker::noop()); - assert!(matches!( - vm.execution_scope() - .resources_mut() - .poll_close(token, &mut cx), - Poll::Ready(Ok(())) - )); - let _ = std::fs::remove_file(path); - } - - #[test] - fn close_completion_reports_scope_retirement_errors() { - let mut vm = Vm::new(crate::Program::new( - Vec::new(), - vec![crate::OpCode::Ret as u8], - )); - let file = std::fs::File::open("Cargo.toml").expect("test file should open"); - let token = vm - .execution_scope() - .push_resource(IoResource::new(IoHandle::File(file))) - .expect("resource should insert"); - vm.execution_scope() - .close_resource::(token.handle(), ResourceCloseReason::Requested) - .expect("initial close should retire resource"); - - let shared = Arc::new(IoOpShared::new()); - *shared - .target - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(token.handle()); - *shared - .value - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = - Some(Ok(CallReturn::one(Value::Bool(true)))); - let op_id = vm - .execution_scope() - .start_operation(OperationSpec::new(IoOpDriver::new( - Arc::clone(&shared), - "io::test-close", - ))) - .expect("test operation should start"); - let error = finish_io_operation(&mut vm, op_id, OperationOutcome::Completed, shared) - .expect_err("stale scope retirement must be visible to the caller"); - assert!(error.to_string().contains("execution scope")); - } -} diff --git a/src/builtins/runtime/io/mod.rs b/src/builtins/runtime/io/mod.rs index 78ddf579..3778b961 100644 --- a/src/builtins/runtime/io/mod.rs +++ b/src/builtins/runtime/io/mod.rs @@ -5,20 +5,17 @@ //! metadata, and [`io_host_module`]. Only the function *implementations* are //! selected per target and per feature: //! -//! - `async` (non-wasm32): `async_io` drives IO through tokio and submits async -//! host functions via the generic async host bridge. -//! - default (non-wasm32): `blocking` drives IO through worker threads -//! registered as concrete operation drivers in the execution scope. +//! - `async` (non-wasm32): `async_io` awaits Tokio file/process operations +//! through ordinary annotated async host functions. +//! - default (non-wasm32): `blocking` performs synchronous IO inline without +//! worker threads or private operation machinery. //! - wasm32: `wasm` (the `io_wasm.rs` backend) keeps the catalog surface and //! rejects every IO call with a host error; the target has no file system. //! -//! The non-wasm32 implementations share the same execution-scope resource -//! model: live handles are `IoResource`s owned by the VM's execution scope and -//! in-flight IO work is driven by concrete operation drivers registered in the -//! same scope. Only the concurrency mechanism differs. Every backend declares -//! the same [`IO_FILE_KEY`] and [`IO_FILE_DESCRIPTION`] for its own concrete -//! handle type, so the guest contract and the target it compiles for cannot -//! drift. +//! Native backends retain only script-visible file/process handles as typed +//! execution-scope resources. Every backend declares the same [`IO_FILE_KEY`] +//! and [`IO_FILE_DESCRIPTION`] for its concrete handle type, so the guest +//! contract and the target it compiles for cannot drift. use super::borrow_arg; #[cfg(all(feature = "async", not(target_arch = "wasm32")))] @@ -26,10 +23,8 @@ use super::{CallOutcome, CaptureAsyncHostContext, return_one}; #[cfg(not(target_arch = "wasm32"))] use crate::vm::Vm; -/// The synchronous host-call channel that hands a pending operation id back to -/// the VM. The blocking and wasm backends schedule a concrete operation driver; -/// the async backend submits a future through the generic bridge instead. -#[cfg(any(not(feature = "async"), target_arch = "wasm32"))] +/// The synchronous pending-call channel used only by the wasm32 stub backend. +#[cfg(target_arch = "wasm32")] pub(super) use super::HostCallResult; /// The canonical catalog key of the `io.file` resource type. diff --git a/tests/builtins/io_async_tests.rs b/tests/builtins/io_async_tests.rs index e1a7ba59..2754f0b1 100644 --- a/tests/builtins/io_async_tests.rs +++ b/tests/builtins/io_async_tests.rs @@ -97,18 +97,38 @@ fn async_io_popen_reads_through_tokio_process_pipe() { } #[test] -fn io_implementations_do_not_create_private_threads_or_runtimes() { +fn io_implementations_use_only_generic_async_and_inline_sync_lifecycles() { let async_source = include_str!("../../src/builtins/runtime/io/async_io.rs"); let blocking_source = include_str!("../../src/builtins/runtime/io/blocking.rs"); - // The async implementation must run on the bridge's executor: it must - // not spawn its own threads or build its own tokio runtime. - assert!(!async_source.contains("thread::Builder")); - assert!(!async_source.contains("runtime::Builder")); - assert!(!async_source.contains("spawn_blocking")); - // The blocking implementation must not create a private runtime either; - // per-op worker threads are driven by the blocking path itself. - assert!(!blocking_source.contains("runtime::Builder")); + for forbidden in [ + "std::thread", + "thread::Builder", + "JoinHandle", + "runtime::Builder", + "spawn_blocking", + "submit_host_future", + "HostAsyncBridge", + "HostOperation", + "IoOperationLease", + "active_operations", + "close_waker", + "close_scheduled", + "close_future", + "owner_alive", + "OperationSpec", + "schedule_io_task", + "worker_done", + ] { + assert!( + !async_source.contains(forbidden), + "async IO must not contain `{forbidden}` lifecycle machinery" + ); + assert!( + !blocking_source.contains(forbidden), + "blocking IO must be synchronous inline code without `{forbidden}`" + ); + } } #[cfg(unix)] diff --git a/tests/builtins/io_builtin_edge_tests.rs b/tests/builtins/io_builtin_edge_tests.rs index f92241fd..90309cc3 100644 --- a/tests/builtins/io_builtin_edge_tests.rs +++ b/tests/builtins/io_builtin_edge_tests.rs @@ -28,6 +28,31 @@ fn run_source_host_error(source: &str) -> String { } } +fn run_vm_to_error(vm: &mut Vm) -> VmError { + let mut status = match vm.run() { + Ok(status) => status, + Err(error) => return error, + }; + loop { + status = match status { + VmStatus::Halted => panic!("expected host error, VM halted"), + VmStatus::Yielded => match vm.resume() { + Ok(status) => status, + Err(error) => return error, + }, + VmStatus::Waiting(_) => { + if let Err(error) = vm.wait_for_host_op_blocking() { + return error; + } + match vm.resume() { + Ok(status) => status, + Err(error) => return error, + } + } + }; + } +} + #[test] fn io_open_rejects_unsupported_mode() { let err = run_source_host_error( @@ -235,13 +260,7 @@ fn io_policy_limits_write_size() { .bind_vm_cached(&mut vm) .expect("profile should bind"); - assert!(matches!( - vm.run().expect("open should start"), - VmStatus::Waiting(_) - )); - vm.wait_for_host_op_blocking() - .expect("open should complete"); - let error = vm.resume().expect_err("oversized write should be denied"); + let error = run_vm_to_error(&mut vm); assert!(matches!(error, VmError::HostError(message) if message.contains("write limit"))); let _ = std::fs::remove_file(path); } diff --git a/tests/builtins/io_scope_lifecycle_tests.rs b/tests/builtins/io_scope_lifecycle_tests.rs index 78ce8260..69c8cebf 100644 --- a/tests/builtins/io_scope_lifecycle_tests.rs +++ b/tests/builtins/io_scope_lifecycle_tests.rs @@ -1,48 +1,26 @@ -//! Focused TDD tests for migrating baseline IO onto the generic -//! [`ExecutionScope`] lifecycle (PR16 commit 3). +//! Lifecycle coverage for the inline non-async IO backend. //! -//! File/process handles are typed resources stored in the VM's execution -//! scope; read/write/flush/close/open/popen/exists pending work is driven by -//! concrete [`HostOperation`] drivers registered in the same scope. These -//! tests verify the scope-backed behaviour through the public VM + IO API: -//! stale-handle and type-mismatch rejection, exact-once close, pending -//! operation cancellation, and reset/drop retirement through the generic -//! scope. +//! Calls execute synchronously, while file/process handles remain typed +//! execution-scope resources retired by explicit close, reset, or VM drop. use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::time::{Duration, Instant}; -use vm::operation::OperationCancelReason; -use vm::operation::OperationId; use vm::resource::close::{CloseProgress, HostResource}; use vm::resource::{ResourceCloseReason, ResourceResult}; use vm::{Value, Vm, VmError, VmStatus, compile_source}; use super::vm_reset::reset_for_reuse_to_ready; -/// Helper: run an IO source to completion, returning the final stack. fn run_source(source: &str) -> Result, VmError> { - let wrapped = format!("use io;\n{source}"); - let compiled = compile_source(&wrapped).expect("source should compile"); + let compiled = compile_source(&format!("use io;\n{source}")).expect("source should compile"); let mut vm = Vm::new(compiled.program); - - let mut status = vm.run()?; - loop { - match status { - VmStatus::Halted => return Ok(vm.stack().to_vec()), - VmStatus::Yielded => { - status = vm.resume()?; - } - VmStatus::Waiting(_) => { - vm.wait_for_host_op_blocking()?; - status = vm.resume()?; - } - } + match vm.run()? { + VmStatus::Halted => Ok(vm.stack().to_vec()), + status => panic!("inline IO must not suspend the VM, got {status:?}"), } } -/// Helper: run an IO source expecting a host error, returning its message. fn run_source_host_error(source: &str) -> String { match run_source(source) { Ok(stack) => panic!("expected host error, got stack: {stack:?}"), @@ -51,7 +29,6 @@ fn run_source_host_error(source: &str) -> String { } } -// A foreign (non-IO) resource used to exercise type-mismatch rejection. struct ForeignResource { closes: Arc, } @@ -63,234 +40,127 @@ impl HostResource for ForeignResource { } } -/// Compiles and runs an IO source to a VM whose scope reflects the result. fn vm_for(source: &str) -> Vm { - let wrapped = format!("use io;\n{source}"); - let compiled = compile_source(&wrapped).expect("source should compile"); + let compiled = compile_source(&format!("use io;\n{source}")).expect("source should compile"); let mut vm = Vm::new(compiled.program); - let mut status = vm.run().expect("run should start"); - loop { - match status { - VmStatus::Halted => break, - VmStatus::Yielded => { - status = vm.resume().expect("resume should continue"); - } - VmStatus::Waiting(_) => { - vm.wait_for_host_op_blocking() - .expect("waiting host op should complete"); - status = vm.resume().expect("resume should continue"); - } - } - } + assert!(matches!( + vm.run().expect("inline IO program should run"), + VmStatus::Halted + )); vm } -fn host_error(err: VmError) -> String { - match err { - VmError::HostError(message) => message, - other => panic!("expected host error, got: {other:?}"), - } +#[test] +fn inline_io_calls_complete_without_pending_operations() { + let mut vm = vm_for("let h = io::open(\"Cargo.toml\", \"r\"); io::close(h);"); + assert!(vm.execution_scope().operations().is_empty()); + assert!(vm.execution_scope().resources().is_empty()); } -// ------------------------------------------------------------------ handles - #[test] fn io_close_returns_true_and_closed_handle_is_stale() { - // The first close is exact-once and returns `true`; a second use of the - // closed handle (a stale handle) is rejected with a host error rather - // than silently succeeding. - let err = run_source_host_error( + let error = run_source_host_error( r#" let handle = io::open("Cargo.toml", "r"); io::close(handle); io::close(handle); - "#, + "#, ); assert!( - err.contains("stale") - || err.contains("not found") - || err.contains("closed") - || err.contains("invalid"), - "double close of a closed IO handle should be rejected; got: {err}" + error.contains("stale") + || error.contains("not found") + || error.contains("closed") + || error.contains("invalid"), + "double close must be rejected: {error}" ); } #[test] fn io_close_then_read_rejects_stale_handle() { - let err = run_source_host_error( + let error = run_source_host_error( r#" let handle = io::open("Cargo.toml", "r"); io::close(handle); io::read_all(handle); - "#, + "#, ); assert!( - err.contains("stale") - || err.contains("not found") - || err.contains("closed") - || err.contains("invalid"), - "reading a closed IO handle should be rejected; got: {err}" + error.contains("stale") + || error.contains("not found") + || error.contains("closed") + || error.contains("invalid"), + "reading a closed handle must be rejected: {error}" ); } #[test] fn io_close_on_non_positive_handle_is_rejected() { - let err = run_source_host_error( - r#" - io::close(0); - "#, - ); - assert!( - err.contains("invalid io handle"), - "non-positive handles must be rejected; got: {err}" - ); + let error = run_source_host_error("io::close(0);"); + assert!(error.contains("invalid io handle"), "{error}"); } #[test] fn io_open_read_mode_reports_missing_file() { - let err = run_source_host_error( - r#" - io::open("__pd_vm_missing_file_for_test__.txt", "r"); - "#, - ); - assert!( - err.contains("io_open failed"), - "unexpected error message: {err}" - ); + let error = run_source_host_error("io::open(\"__pd_vm_missing_file_for_test__.txt\", \"r\");"); + assert!(error.contains("io_open failed"), "{error}"); } #[test] fn io_open_rejects_unsupported_mode() { - let err = run_source_host_error( - r#" - io::open("Cargo.toml", "bad"); - "#, - ); - assert!( - err.contains("unsupported io_open mode"), - "unexpected error message: {err}" - ); + let error = run_source_host_error("io::open(\"Cargo.toml\", \"bad\");"); + assert!(error.contains("unsupported io_open mode"), "{error}"); } -// ------------------------------------------------------------- type mismatch - #[test] fn io_rejects_foreign_scope_handles() { - // IO handles are scope-scoped typed tokens: a handle minted by one VM's - // execution scope must be rejected when used against another VM's scope - // (wrong table / stale / invalid), never interpreted as a live handle. let foreign_handle = { - let wrapped = "use io;\nlet h = io::open(\"Cargo.toml\", \"r\");\nh;"; - let compiled = compile_source(wrapped).expect("compile"); - let mut vm = Vm::new(compiled.program); - // Run and drain any waiting IO op. - let mut status = vm.run().expect("run"); - loop { - match status { - VmStatus::Waiting(_) => { - vm.wait_for_host_op_blocking().expect("wait"); - status = vm.resume().expect("resume"); - } - VmStatus::Yielded => { - status = vm.resume().expect("resume"); - } - VmStatus::Halted => break, - } - } - let handle = vm.stack().last().cloned().expect("handle on stack"); - let Value::Int(raw) = handle else { + let vm = vm_for("let h = io::open(\"Cargo.toml\", \"r\"); h;"); + let Value::Int(raw) = vm.stack().last().cloned().expect("handle on stack") else { panic!("io::open must return an integer handle"); }; raw }; - - let wrapped = format!("use io;\nio::close({foreign_handle});"); - let compiled = compile_source(&wrapped).expect("compile"); - let mut vm2 = Vm::new(compiled.program); - let err = host_error( - vm2.run() - .expect_err("foreign handle close must be rejected"), - ); + let error = run_source_host_error(&format!("io::close({foreign_handle});")); assert!( - err.contains("mismatch") - || err.contains("type") - || err.contains("stale") - || err.contains("invalid") - || err.contains("table"), - "foreign-scope IO handle access must be rejected; got: {err}" + error.contains("mismatch") + || error.contains("type") + || error.contains("stale") + || error.contains("invalid") + || error.contains("table"), + "foreign-scope handle must be rejected: {error}" ); } #[test] -fn io_resources_are_typed_and_never_cross_interpreted() { - // A foreign (non-IO) resource sharing the same execution scope is a - // distinct typed resource: the generic typed-table access rejects a - // wrong-typed token before any IO interpretation can happen. This is the - // generic guarantee IO handles rely on (TypeId-checked borrows). +fn generic_resource_types_remain_distinct() { let closes = Arc::new(AtomicUsize::new(0)); - let wrapped = "use io;\nio::open(\"Cargo.toml\", \"r\");"; - let compiled = compile_source(wrapped).expect("compile"); - let mut vm = Vm::new(compiled.program); + let mut vm = vm_for("io::open(\"Cargo.toml\", \"r\");"); let foreign = vm .execution_scope() .push_resource(ForeignResource { closes: Arc::clone(&closes), }) .expect("foreign resource must insert"); - // The foreign token is a valid live resource in this scope: its own - // close (typed correctly) succeeds and runs exactly once. - let _ = vm - .execution_scope() + vm.execution_scope() .close_resource::(foreign.handle(), ResourceCloseReason::Requested) - .expect("typed close of the foreign resource must succeed"); - assert_eq!(closes.load(Ordering::SeqCst), 1, "close runs exactly once"); + .expect("typed close must succeed"); + assert_eq!(closes.load(Ordering::SeqCst), 1); } -// ----------------------------------------------------- pending cancellation - #[test] -fn pending_io_operation_can_be_cancelled_through_scope() { - // `read_all` on a child that produces no output and does not exit keeps - // the operation genuinely pending. Cancelling it through the VM's - // execution scope must mark it terminal and retire it from the registry. - let wrapped = "use io;\nlet h = io::popen(\"sleep 30\", \"r\");\nio::read_all(h);"; - let compiled = compile_source(wrapped).expect("compile"); - let mut vm = Vm::new(compiled.program); - - let status = vm.run().expect("run should start pending"); - let waiting = match status { - VmStatus::Waiting(op_id) => op_id, - other => panic!("expected a waiting host op, got: {other:?}"), - }; - let id = OperationId::from_raw(waiting).expect("waiting op id must be a valid operation id"); - assert_eq!( - vm.execution_scope().operations().len(), - 1, - "the pending IO op must occupy a scope operation slot" - ); - - let can_cancel = vm - .execution_scope() - .cancel_operation(id, OperationCancelReason::Requested) - .expect("pending op must be cancellable"); - assert!(can_cancel, "cancel on a pending op must report success"); - assert_eq!( - vm.execution_scope().operations().len(), - 1, - "cancellation must retain the operation until its worker exits" - ); +fn reset_for_reuse_retires_io_resources_through_scope() { + let mut vm = vm_for("let h = io::open(\"Cargo.toml\", \"r\"); h;"); + assert!(!vm.execution_scope().resources().is_empty()); + reset_for_reuse_to_ready(&mut vm).expect("reset should reach quiescence"); + assert!(vm.execution_scope().resources().is_empty()); + assert!(vm.execution_scope().operations().is_empty()); +} - let error = vm - .wait_for_host_op_blocking() - .expect_err("cancelled IO operation should report cancellation"); - assert!( - matches!(error, VmError::HostError(ref message) if message.contains("cancelled")), - "unexpected cancellation error: {error:?}" - ); - assert!( - vm.execution_scope().operations().is_empty(), - "polling the cancelled operation must release it exactly once" - ); +#[test] +fn drop_retires_io_resources_through_scope() { + let mut vm = vm_for("io::open(\"Cargo.toml\", \"r\");"); + assert!(!vm.execution_scope().resources().is_empty()); + drop(vm); } #[cfg(unix)] @@ -344,92 +214,32 @@ fn read_process_marker(path: &std::path::Path) -> (i32, i32) { .split_whitespace() .map(str::parse::) .collect::, _>>() - .expect("popen marker should contain process ids"); + .expect("marker should contain process ids"); if values.len() == 2 { return (values[0], values[1]); } } assert!( std::time::Instant::now() < deadline, - "popen test child did not publish its process marker" + "popen child did not publish its process marker" ); std::thread::yield_now(); } } -// ------------------------------------------------ reset / drop retirement - -#[test] -fn reset_for_reuse_joins_pending_io_worker() { - let compiled = - compile_source("use io;\nlet h = io::popen(\"sleep 30\", \"r\");\nio::read_all(h);") - .expect("source should compile"); - let mut vm = Vm::new(compiled.program); - assert!(matches!( - vm.run().expect("run should start"), - VmStatus::Waiting(_) - )); - assert_eq!(vm.execution_scope().operations().len(), 1); - - reset_for_reuse_to_ready(&mut vm).expect("reset should reach quiescence"); - assert!(vm.execution_scope().operations().is_empty()); - assert!(vm.execution_scope().resources().is_empty()); -} - -#[test] -fn reset_for_reuse_retires_io_resources_through_scope() { - let mut vm = vm_for("let h = io::open(\"Cargo.toml\", \"r\");\nh;"); - assert!( - !vm.execution_scope().resources().is_empty(), - "open leaves a live IO resource in the scope" - ); - - reset_for_reuse_to_ready(&mut vm).expect("reset should reach quiescence"); - - assert!( - vm.execution_scope().resources().is_empty() && vm.execution_scope().operations().is_empty(), - "reset for reuse must retire IO resources and operations through the scope" - ); -} - -#[test] -fn drop_retires_io_resources_through_scope() { - // Dropping a VM with a live IO handle must retire the handle through the - // generic scope (no custom close-all side channel). The scope's own Drop - // runs the closing sweep; this test guards that path stays wired. - let mut vm = vm_for("io::open(\"Cargo.toml\", \"r\");"); - assert!(!vm.execution_scope().resources().is_empty()); - drop(vm); -} - #[cfg(unix)] #[test] fn reset_for_reuse_terminates_live_popen_process_tree() { - static TEST_COUNTER: AtomicUsize = AtomicUsize::new(0); - let suffix = TEST_COUNTER.fetch_add(1, Ordering::Relaxed); let marker = std::env::temp_dir().join(format!( - "pd-vm-blocking-io-reset-{0}-{suffix}.marker", - std::process::id() + "pd-vm-blocking-io-reset-{}-{}.marker", + std::process::id(), + SystemTimeNonce::new() )); let command = format!( "parent=$$; sleep 30 & child=$!; printf '%s %s' $parent $child > {}; wait $child", marker.display() ); - let source = format!("use io;\nlet h = io::popen(\"{command}\", \"r\");\nh;"); - let compiled = compile_source(&source).expect("source should compile"); - let mut vm = Vm::new(compiled.program); - let mut status = vm.run().expect("run should start"); - loop { - match status { - VmStatus::Halted => break, - VmStatus::Yielded => status = vm.resume().expect("resume should continue"), - VmStatus::Waiting(_) => { - vm.wait_for_host_op_blocking() - .expect("waiting op should finish"); - status = vm.resume().expect("resume should continue"); - } - } - } + let mut vm = vm_for(&format!("let h = io::popen(\"{command}\", \"r\"); h;")); let (leader, descendant) = read_process_marker(&marker); let _cleanup = ProcessTreeCleanup { leader, @@ -442,3 +252,25 @@ fn reset_for_reuse_terminates_live_popen_process_tree() { wait_for_process_exit(descendant); let _ = std::fs::remove_file(marker); } + +#[cfg(unix)] +struct SystemTimeNonce(u128); + +#[cfg(unix)] +impl SystemTimeNonce { + fn new() -> Self { + Self( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock should follow Unix epoch") + .as_nanos(), + ) + } +} + +#[cfg(unix)] +impl std::fmt::Display for SystemTimeNonce { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(formatter) + } +} diff --git a/tests/io_descriptor_install_tests.rs b/tests/io_descriptor_install_tests.rs index 99f8534c..a2a2cfdc 100644 --- a/tests/io_descriptor_install_tests.rs +++ b/tests/io_descriptor_install_tests.rs @@ -287,10 +287,11 @@ fn io_module_install_rejects_a_descriptor_list_whose_adapters_disagree() { /// The IO calls the round-trip script makes. Each one is dispatched through the /// installed descriptor and must be driven to completion as a pending host /// operation. +#[cfg(feature = "async")] const ROUND_TRIP_IO_CALLS: usize = 7; -/// Submitted-future count of the async host driver; stays zero on the blocking -/// backend, which drives registered operation drivers instead. +/// Submitted-future count of the async host driver; stays zero on the inline +/// synchronous backend. type SubmittedOps = std::sync::Arc; /// One driven run: the final stack plus the number of pending host operations @@ -418,8 +419,8 @@ fn install_async_driver(vm: &mut Vm) -> SubmittedOps { count } -/// The blocking backend schedules concrete operation drivers in the execution -/// scope, so it submits no futures and needs no bridge. +/// The synchronous backend completes inline, so it submits no futures and +/// needs no bridge. #[cfg(not(feature = "async"))] fn install_async_driver(_vm: &mut Vm) -> SubmittedOps { SubmittedOps::default() @@ -427,8 +428,8 @@ fn install_async_driver(_vm: &mut Vm) -> SubmittedOps { /// Asserts that the pending operations the run loop drove were resolved by the /// backend that is compiled in: the async backend resolves each one through the -/// submitted-future bridge, the blocking backend through its registered -/// operation driver. +/// submitted-future bridge, while the synchronous backend yields no pending +/// operation. fn assert_pending_ops_resolved_by_backend(submitted: &SubmittedOps, driven_ops: usize) { use std::sync::atomic::Ordering; let submitted = submitted.load(Ordering::SeqCst); @@ -445,10 +446,13 @@ fn assert_pending_ops_resolved_by_backend(submitted: &SubmittedOps, driven_ops: } #[cfg(not(feature = "async"))] { - assert_eq!(submitted, 0, "the blocking backend must not submit futures"); assert_eq!( - driven_ops, ROUND_TRIP_IO_CALLS, - "the blocking backend must drive one registered operation driver per IO call" + submitted, 0, + "the synchronous backend must not submit futures" + ); + assert_eq!( + driven_ops, 0, + "the synchronous backend must complete every IO call inline" ); } } @@ -479,7 +483,7 @@ fn assert_rejected_call_left_no_io_state(vm: &mut Vm, submitted: &SubmittedOps) assert_eq!( submitted.load(std::sync::atomic::Ordering::SeqCst), 0, - "the blocking adapter must reject a policy violation before scheduling work" + "the synchronous adapter must reject a policy violation inline" ); } @@ -578,7 +582,7 @@ fn installed_io_descriptors_reject_a_stale_handle_after_close() { assert_eq!( submitted.load(std::sync::atomic::Ordering::SeqCst), 0, - "the blocking backend must not submit futures" + "the synchronous backend must not submit futures" ); let _ = fs::remove_dir_all(&dir); } From 6c6607d2b309a7c00baf1ac9d793096078896ef3 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sat, 19 Sep 2026 15:59:23 +0800 Subject: [PATCH 05/23] fix(io): make process cleanup cancellation-safe --- src/builtins/runtime/io/async_io.rs | 529 ++++++++++++++++++++-------- src/builtins/runtime/io/blocking.rs | 240 ++++++++++--- tests/builtins/io_async_tests.rs | 78 +++- 3 files changed, 663 insertions(+), 184 deletions(-) diff --git a/src/builtins/runtime/io/async_io.rs b/src/builtins/runtime/io/async_io.rs index bd2342e0..445d3502 100644 --- a/src/builtins/runtime/io/async_io.rs +++ b/src/builtins/runtime/io/async_io.rs @@ -5,10 +5,12 @@ //! calls; transient reads, writes, flushes, and closes rely on the generic //! submitted-future lifecycle. +use std::future::Future; +use std::io; use std::path::{Path, PathBuf}; +use std::pin::Pin; use std::process::Stdio; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::task::{Context, Poll}; use pd_host_function::pd_host_function; @@ -41,72 +43,89 @@ impl Drop for IoHandle { fn drop(&mut self) { match self { Self::PopenRead { child, .. } | Self::PopenWrite { child, .. } => { - terminate_process_id(child.id().unwrap_or(0)); - let _ = child.start_kill(); + let _ = start_terminate_child_tree(child); } Self::File(_) => {} } } } -/// Shared handle state captured by async calls. -struct IoResourceState { - handle: Mutex>, - closed: AtomicBool, - process_id: AtomicU32, +/// The typed resource stored in the execution scope for one async IO handle. +struct IoResource { + handle: Arc>>, } -impl IoResourceState { +impl IoResource { fn new(handle: IoHandle) -> Self { - let process_id = process_id(&handle); Self { - handle: Mutex::new(Some(handle)), - closed: AtomicBool::new(false), - process_id: AtomicU32::new(process_id), + handle: Arc::new(Mutex::new(Some(handle))), } } - fn ensure_open(&self, operation: &str) -> VmResult<()> { - if self.closed.load(Ordering::Acquire) { - Err(VmError::HostError(format!("{operation} handle is closed"))) - } else { - Ok(()) - } + fn exclusive_handle_slot(&mut self) -> ResourceResult<&mut Option> { + Arc::get_mut(&mut self.handle) + .map(Mutex::get_mut) + .ok_or_else(|| { + ResourceError::new( + ResourceErrorCode::ResourceCleanupFailed, + "io::resource", + "async IO handle remained borrowed after host operations quiesced", + ) + }) } -} - -/// The typed resource stored in the execution scope for one async IO handle. -struct IoResource { - state: Arc, -} -impl IoResource { - fn new(handle: IoHandle) -> Self { - Self { - state: Arc::new(IoResourceState::new(handle)), + fn begin_close_after_operations_quiesce(&mut self) -> ResourceResult { + let slot = self.exclusive_handle_slot()?; + match slot.as_mut() { + None => Ok(CloseProgress::Ready), + Some(IoHandle::File(_)) => { + slot.take(); + Ok(CloseProgress::Ready) + } + Some(IoHandle::PopenRead { child, .. }) | Some(IoHandle::PopenWrite { child, .. }) => { + start_terminate_child_tree(child).map_err(process_resource_error)?; + Ok(CloseProgress::Pending) + } } } - fn close_nonblocking(&mut self) -> ResourceResult { - self.state.closed.store(true, Ordering::Release); - terminate_process_id(self.state.process_id.load(Ordering::Acquire)); - let Ok(mut slot) = self.state.handle.try_lock() else { - return Ok(CloseProgress::Pending); + fn poll_process_close(&mut self, cx: &mut Context<'_>) -> Poll> { + let slot = match self.exclusive_handle_slot() { + Ok(slot) => slot, + Err(error) => return Poll::Ready(Err(error)), + }; + let poll = match slot.as_mut() { + None | Some(IoHandle::File(_)) => Poll::Ready(Ok(())), + Some(IoHandle::PopenRead { child, .. }) | Some(IoHandle::PopenWrite { child, .. }) => { + match child.try_wait() { + Ok(Some(_)) => Poll::Ready(Ok(())), + Err(error) => Poll::Ready(Err(error)), + Ok(None) if tokio::runtime::Handle::try_current().is_err() => Poll::Pending, + Ok(None) => { + let mut wait = Box::pin(child.wait()); + match wait.as_mut().poll(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(result) => Poll::Ready(result.map(|_| ())), + } + } + } + } }; - if let Some(mut handle) = slot.take() { - start_close_io_handle(&mut handle)?; + match poll { + Poll::Pending => Poll::Pending, + Poll::Ready(Ok(_)) => { + slot.take(); + Poll::Ready(Ok(())) + } + Poll::Ready(Err(error)) => Poll::Ready(Err(process_resource_error(error))), } - self.state.process_id.store(0, Ordering::Release); - Ok(CloseProgress::Ready) } } impl Drop for IoResource { fn drop(&mut self) { - self.state.closed.store(true, Ordering::Release); - terminate_process_id(self.state.process_id.swap(0, Ordering::AcqRel)); - if let Ok(mut slot) = self.state.handle.try_lock() - && let Some(mut handle) = slot.take() + if let Some(mutex) = Arc::get_mut(&mut self.handle) + && let Some(mut handle) = mutex.get_mut().take() { let _ = start_close_io_handle(&mut handle); } @@ -125,41 +144,52 @@ pub(crate) fn io_file_resource() -> crate::host_extension::HostResourceTypeMeta impl HostResource for IoResource { fn begin_close(&mut self, _reason: ResourceCloseReason) -> ResourceResult { - self.close_nonblocking() + self.begin_close_after_operations_quiesce() } fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll> { - match self.close_nonblocking() { - Ok(CloseProgress::Ready) => Poll::Ready(Ok(())), - Ok(CloseProgress::Pending) => { - cx.waker().wake_by_ref(); - Poll::Pending - } - Err(error) => Poll::Ready(Err(error)), - } + self.poll_process_close(cx) } } +fn process_resource_error(error: io::Error) -> ResourceError { + ResourceError::new( + ResourceErrorCode::ResourceCleanupFailed, + "io::resource", + format!("io_close popen terminate failed: {error}"), + ) +} + fn start_close_io_handle(handle: &mut IoHandle) -> ResourceResult<()> { match handle { IoHandle::File(_) => Ok(()), IoHandle::PopenRead { child, .. } | IoHandle::PopenWrite { child, .. } => { - terminate_process_id(child.id().unwrap_or(0)); - match child.start_kill() { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::InvalidInput => Ok(()), - Err(error) => Err(ResourceError::new( - ResourceErrorCode::ResourceCleanupFailed, - "io::resource", - format!("io_close popen terminate failed: {error}"), - )), - } + start_terminate_child_tree(child).map_err(process_resource_error) } } } -async fn close_io_handle(mut handle: IoHandle) -> VmResult<()> { - match &mut handle { +type CloseIoFuture<'a> = Pin> + Send + 'a>>; + +fn close_io_handle_future(handle: &mut IoHandle) -> CloseIoFuture<'_> { + Box::pin(close_io_handle(handle)) +} + +async fn close_shared_io_handle_with( + shared: Arc>>, + close: impl for<'a> FnOnce(&'a mut IoHandle) -> CloseIoFuture<'a>, +) -> VmResult<()> { + let mut slot = shared.lock().await; + let handle = slot + .as_mut() + .ok_or_else(|| VmError::HostError("io handle is closed".to_string()))?; + close(handle).await?; + slot.take(); + Ok(()) +} + +async fn close_io_handle(handle: &mut IoHandle) -> VmResult<()> { + match handle { IoHandle::File(file) => { file.get_mut() .flush() @@ -167,59 +197,138 @@ async fn close_io_handle(mut handle: IoHandle) -> VmResult<()> { .map_err(|error| VmError::HostError(format!("io_close flush failed: {error}")))?; } IoHandle::PopenRead { child, .. } => { - terminate_process_id(child.id().unwrap_or(0)); - kill_and_reap_child(child).await?; + terminate_child_tree(child).await?; } IoHandle::PopenWrite { child, stdin } => { let _ = stdin.shutdown().await; - terminate_process_id(child.id().unwrap_or(0)); - kill_and_reap_child(child).await?; + terminate_child_tree(child).await?; } } Ok(()) } -async fn kill_and_reap_child(child: &mut Child) -> VmResult<()> { - match child.kill().await { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::InvalidInput => { - child.wait().await.map(|_| ()).map_err(|wait_error| { - VmError::HostError(format!("io_close popen wait failed: {wait_error}")) - }) - } - Err(error) => Err(VmError::HostError(format!( - "io_close popen wait failed: {error}" - ))), - } +async fn terminate_child_tree(child: &mut Child) -> VmResult<()> { + let pid = child.id().unwrap_or(0); + terminate_process_tree_and_leader(|| terminate_process_id(pid), || kill_and_reap_child(child)) + .await + .map_err(|error| VmError::HostError(format!("io_close popen terminate failed: {error}"))) } -fn process_id(handle: &IoHandle) -> u32 { - match handle { - IoHandle::PopenRead { child, .. } | IoHandle::PopenWrite { child, .. } => { - child.id().unwrap_or(0) - } - IoHandle::File(_) => 0, +async fn kill_and_reap_child(child: &mut Child) -> io::Result<()> { + match child.kill().await { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::InvalidInput => child.wait().await.map(|_| ()), + Err(error) => Err(error), + } +} + +async fn terminate_process_tree_and_leader( + terminate_tree: Tree, + terminate_leader: Leader, +) -> io::Result<()> +where + Tree: FnOnce() -> io::Result<()>, + Leader: FnOnce() -> LeaderFuture, + LeaderFuture: Future>, +{ + let tree_result = terminate_tree(); + let leader_result = terminate_leader().await; + combine_process_cleanup_results(tree_result, leader_result) +} + +fn combine_process_cleanup_results( + tree_result: io::Result<()>, + leader_result: io::Result<()>, +) -> io::Result<()> { + match (tree_result, leader_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(tree_error), Ok(())) => Err(tree_error), + (Ok(()), Err(leader_error)) => Err(leader_error), + (Err(tree_error), Err(leader_error)) => Err(io::Error::new( + tree_error.kind(), + format!( + "process-tree termination failed: {tree_error}; direct leader cleanup also failed: {leader_error}" + ), + )), + } +} + +fn start_terminate_child_tree(child: &mut Child) -> io::Result<()> { + let pid = child.id().unwrap_or(0); + let tree_result = terminate_process_id(pid); + let leader_result = child.start_kill().or_else(ignore_already_exited); + combine_process_cleanup_results(tree_result, leader_result) +} + +fn ignore_already_exited(error: io::Error) -> io::Result<()> { + if error.kind() == io::ErrorKind::InvalidInput { + Ok(()) + } else { + Err(error) } } -fn terminate_process_id(pid: u32) { +fn terminate_process_id(pid: u32) -> io::Result<()> { if pid == 0 { - return; + return Ok(()); } #[cfg(unix)] - if let Ok(pid) = libc::pid_t::try_from(pid) { - unsafe { - libc::kill(-pid, libc::SIGKILL); - } + { + terminate_unix_process_group_with( + pid, + |process_group, signal| unsafe { libc::kill(process_group, signal) }, + io::Error::last_os_error, + ) } #[cfg(windows)] { - let _ = std::process::Command::new("taskkill") - .args(["/T", "/F", "/PID", &pid.to_string()]) - .status(); + run_taskkill_with(pid, std::process::Command::status) } #[cfg(not(any(unix, windows)))] - let _ = pid; + { + let _ = pid; + Ok(()) + } +} + +#[cfg(unix)] +fn terminate_unix_process_group_with( + pid: u32, + kill: Kill, + last_error: LastError, +) -> io::Result<()> +where + Kill: FnOnce(libc::pid_t, libc::c_int) -> libc::c_int, + LastError: FnOnce() -> io::Error, +{ + let pid = libc::pid_t::try_from(pid) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "process id exceeds pid_t"))?; + if kill(-pid, libc::SIGKILL) == 0 { + return Ok(()); + } + let error = last_error(); + if error.raw_os_error() == Some(libc::ESRCH) { + Ok(()) + } else { + Err(error) + } +} + +#[cfg(any(windows, test))] +fn run_taskkill_with(pid: u32, run: Run) -> io::Result<()> +where + Run: FnOnce(&mut std::process::Command) -> io::Result, +{ + let mut command = std::process::Command::new("taskkill"); + command.args(["/T", "/F", "/PID", &pid.to_string()]); + let status = run(&mut command)?; + if status.success() { + Ok(()) + } else { + Err(io::Error::other(format!( + "taskkill exited with status {status}" + ))) + } } /// The per-call captured policy context. @@ -236,10 +345,10 @@ impl CaptureAsyncHostContext for IoPolicyContext { } } -/// Shared handle state and byte limits captured before an async call starts. +/// Shared handle ownership and byte limits captured before an async call starts. pub(crate) struct IoHandleContext { - handle: ResourceHandle, - state: Arc, + resource: ResourceHandle, + handle: Arc>>, max_read_bytes: Option, max_write_bytes: Option, } @@ -257,11 +366,11 @@ impl CaptureAsyncHostContext for IoHandleContext { Some(_) => return Err(VmError::TypeMismatch("int")), None => return Err(VmError::HostError("missing io handle argument".to_string())), }; - let handle = io_parse_handle(handle_id)?; - let state = io_state_for_handle(vm, handle)?; + let resource = io_parse_handle(handle_id)?; + let handle = io_handle_for_resource(vm, resource)?; Ok(Self { + resource, handle, - state, max_read_bytes: io_policy(vm).map(|policy| policy.max_read_bytes), max_write_bytes: io_policy(vm).map(|policy| policy.max_write_bytes), }) @@ -358,9 +467,7 @@ pub(crate) async fn builtin_io_read_all( #[pd_host_context] context: IoHandleContext, _handle_id: i64, ) -> VmResult> { - context.state.ensure_open("io_read_all")?; - let mut slot = context.state.handle.lock().await; - context.state.ensure_open("io_read_all")?; + let mut slot = context.handle.lock().await; let handle = slot .as_mut() .ok_or_else(|| VmError::HostError("io handle is closed".to_string()))?; @@ -392,9 +499,7 @@ pub(crate) async fn builtin_io_read_line( #[pd_host_context] context: IoHandleContext, _handle_id: i64, ) -> VmResult> { - context.state.ensure_open("io_read_line")?; - let mut slot = context.state.handle.lock().await; - context.state.ensure_open("io_read_line")?; + let mut slot = context.handle.lock().await; let handle = slot .as_mut() .ok_or_else(|| VmError::HostError("io handle is closed".to_string()))?; @@ -435,9 +540,7 @@ pub(crate) async fn builtin_io_write( "io_write exceeded write limit".to_string(), )); } - context.state.ensure_open("io_write")?; - let mut slot = context.state.handle.lock().await; - context.state.ensure_open("io_write")?; + let mut slot = context.handle.lock().await; let handle = slot .as_mut() .ok_or_else(|| VmError::HostError("io handle is closed".to_string()))?; @@ -460,9 +563,7 @@ pub(crate) async fn builtin_io_flush( #[pd_host_context] context: IoHandleContext, _handle_id: i64, ) -> VmResult> { - context.state.ensure_open("io_flush")?; - let mut slot = context.state.handle.lock().await; - context.state.ensure_open("io_flush")?; + let mut slot = context.handle.lock().await; let handle = slot .as_mut() .ok_or_else(|| VmError::HostError("io handle is closed".to_string()))?; @@ -481,25 +582,12 @@ pub(crate) async fn builtin_io_close( #[pd_host_context] context: IoHandleContext, _handle_id: i64, ) -> VmResult> { - if context.state.closed.swap(true, Ordering::AcqRel) { - return Err(VmError::HostError("io_close handle is closed".to_string())); - } - let owned = context - .state - .handle - .lock() - .await - .take() - .ok_or_else(|| VmError::HostError("io handle is closed".to_string()))?; - let close_result = close_io_handle(owned).await; - if close_result.is_ok() { - context.state.process_id.store(0, Ordering::Release); - } - let handle = context.handle; + close_shared_io_handle_with(Arc::clone(&context.handle), close_io_handle_future).await?; + let resource = context.resource; Ok(HostFutureOutput::complete(move |vm| { let progress = vm .execution_scope() - .close_resource::(handle, ResourceCloseReason::Requested) + .close_resource::(resource, ResourceCloseReason::Requested) .map_err(|error| { VmError::HostError(format!("io_close scope retirement failed: {error}")) })?; @@ -508,7 +596,6 @@ pub(crate) async fn builtin_io_close( "io_close scope retirement is still pending".to_string(), )); } - close_result?; Ok(true) })) } @@ -585,7 +672,10 @@ async fn canonicalize_io_target(path: &Path) -> VmResult { Ok(canonical_parent.join(name)) } -fn io_state_for_handle(vm: &mut Vm, handle: ResourceHandle) -> VmResult> { +fn io_handle_for_resource( + vm: &mut Vm, + handle: ResourceHandle, +) -> VmResult>>> { let token = vm .execution_scope() .resources() @@ -606,7 +696,7 @@ fn io_state_for_handle(vm: &mut Vm, handle: ResourceHandle) -> VmResult VmResult { @@ -646,11 +736,13 @@ fn spawn_shell_command(shell_command: &str, mode: &str) -> VmResult { .map_err(|error| VmError::HostError(format!("io_popen failed: {error}")))?; if mode == "r" { let Some(stdout) = child.stdout.take() else { - terminate_process_id(child.id().unwrap_or(0)); - let _ = child.start_kill(); - return Err(VmError::HostError( - "io_popen('r') did not provide stdout pipe".to_string(), - )); + let cleanup = start_terminate_child_tree(&mut child); + return Err(VmError::HostError(match cleanup { + Ok(()) => "io_popen('r') did not provide stdout pipe".to_string(), + Err(error) => format!( + "io_popen('r') did not provide stdout pipe; process cleanup failed: {error}" + ), + })); }; Ok(IoHandle::PopenRead { child, @@ -658,12 +750,165 @@ fn spawn_shell_command(shell_command: &str, mode: &str) -> VmResult { }) } else { let Some(stdin) = child.stdin.take() else { - terminate_process_id(child.id().unwrap_or(0)); - let _ = child.start_kill(); - return Err(VmError::HostError( - "io_popen('w') did not provide stdin pipe".to_string(), - )); + let cleanup = start_terminate_child_tree(&mut child); + return Err(VmError::HostError(match cleanup { + Ok(()) => "io_popen('w') did not provide stdin pipe".to_string(), + Err(error) => format!( + "io_popen('w') did not provide stdin pipe; process cleanup failed: {error}" + ), + })); }; Ok(IoHandle::PopenWrite { child, stdin }) } } + +#[cfg(test)] +mod tests { + use std::future::{Future, pending}; + use std::io; + use std::pin::Pin; + use std::sync::{Arc, Mutex as StdMutex}; + use std::task::{Context, Poll, Waker}; + + use super::*; + + fn file_resource() -> IoResource { + let file = std::fs::File::open("Cargo.toml").expect("test fixture should exist"); + IoResource::new(IoHandle::File(BufReader::new(File::from_std(file)))) + } + + fn pending_close<'a>( + _handle: &'a mut IoHandle, + started: Arc>, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + *started.lock().expect("started lock") = true; + pending().await + }) + } + + #[tokio::test] + async fn cancelling_explicit_close_retains_the_handle_in_the_resource() { + let resource = file_resource(); + let shared = Arc::clone(&resource.handle); + let started = Arc::new(StdMutex::new(false)); + let close_started = Arc::clone(&started); + let mut future = Box::pin(close_shared_io_handle_with( + Arc::clone(&shared), + move |handle| pending_close(handle, close_started), + )); + let mut cx = Context::from_waker(Waker::noop()); + + assert!(matches!(future.as_mut().poll(&mut cx), Poll::Pending)); + assert!(*started.lock().expect("started lock")); + drop(future); + + assert!( + shared.lock().await.is_some(), + "cancelling io::close must leave the real handle owned by the resource" + ); + } + + #[tokio::test] + async fn process_resource_close_polls_until_the_leader_is_reaped() { + let mut resource = + IoResource::new(spawn_shell_command("sleep 30", "r").expect("process should spawn")); + let pid = { + let slot = resource.handle.lock().await; + match slot.as_ref().expect("resource handle") { + IoHandle::PopenRead { child, .. } => child.id().expect("child pid"), + other => panic!("expected popen read handle, got {other:?}"), + } + }; + + assert_eq!( + resource + .begin_close(ResourceCloseReason::VmReset) + .expect("close should start"), + CloseProgress::Pending + ); + std::future::poll_fn(|cx| resource.poll_close(cx)) + .await + .expect("leader cleanup should complete"); + assert!( + !std::path::Path::new(&format!("/proc/{pid}")).exists(), + "resource close must reap the direct child" + ); + } + + #[tokio::test] + async fn process_tree_failure_still_attempts_direct_async_leader_cleanup() { + let leader_attempted = Arc::new(StdMutex::new(false)); + let attempted = Arc::clone(&leader_attempted); + let error = terminate_process_tree_and_leader( + || { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "tree denied", + )) + }, + move || async move { + *attempted.lock().expect("attempt lock") = true; + Ok(()) + }, + ) + .await + .expect_err("tree failure must propagate"); + + assert!(*leader_attempted.lock().expect("attempt lock")); + assert_eq!(error.kind(), io::ErrorKind::PermissionDenied); + assert!(error.to_string().contains("tree denied")); + } + + #[cfg(unix)] + #[test] + fn unix_process_group_signal_treats_esrch_as_success() { + terminate_unix_process_group_with( + 42, + |_, _| -1, + || io::Error::from_raw_os_error(libc::ESRCH), + ) + .expect("an already absent process group is successfully terminated"); + + let error = terminate_unix_process_group_with( + 42, + |_, _| -1, + || io::Error::from_raw_os_error(libc::EPERM), + ) + .expect_err("other process group failures must propagate"); + assert_eq!(error.raw_os_error(), Some(libc::EPERM)); + } + + #[test] + fn taskkill_launch_failure_is_an_error() { + let launch = run_taskkill_with(42, |_| { + Err(io::Error::new(io::ErrorKind::NotFound, "taskkill missing")) + }) + .expect_err("taskkill launch failure must propagate"); + assert_eq!(launch.kind(), io::ErrorKind::NotFound); + } + + #[cfg(unix)] + #[test] + fn unsuccessful_taskkill_status_is_an_error() { + let status = run_taskkill_with(42, |_| { + std::process::Command::new("sh") + .args(["-c", "exit 7"]) + .status() + }) + .expect_err("unsuccessful taskkill status must propagate"); + assert!(status.to_string().contains("status")); + } + + #[cfg(windows)] + #[test] + fn unsuccessful_taskkill_status_is_an_error() { + let status = run_taskkill_with(42, |_| { + std::process::Command::new("cmd") + .args(["/C", "exit", "7"]) + .status() + }) + .expect_err("unsuccessful taskkill status must propagate"); + assert!(status.to_string().contains("status")); + } +} diff --git a/src/builtins/runtime/io/blocking.rs b/src/builtins/runtime/io/blocking.rs index 373ae994..49170dc8 100644 --- a/src/builtins/runtime/io/blocking.rs +++ b/src/builtins/runtime/io/blocking.rs @@ -1,5 +1,5 @@ use std::fs::OpenOptions; -use std::io::{Read, Write}; +use std::io::{self, Read, Write}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; @@ -413,16 +413,22 @@ fn spawn_shell_command(command: &str, mode: &str) -> VmResult { "r" if child.stdout.is_some() => Ok(IoHandle::PopenRead { child }), "w" if child.stdin.is_some() => Ok(IoHandle::PopenWrite { child }), "r" => { - let _ = terminate_child_tree(&mut child); - Err(VmError::HostError( - "io_popen('r') did not provide stdout pipe".to_string(), - )) + let cleanup = terminate_child_tree(&mut child); + Err(VmError::HostError(match cleanup { + Ok(()) => "io_popen('r') did not provide stdout pipe".to_string(), + Err(error) => format!( + "io_popen('r') did not provide stdout pipe; process cleanup failed: {error}" + ), + })) } "w" => { - let _ = terminate_child_tree(&mut child); - Err(VmError::HostError( - "io_popen('w') did not provide stdin pipe".to_string(), - )) + let cleanup = terminate_child_tree(&mut child); + Err(VmError::HostError(match cleanup { + Ok(()) => "io_popen('w') did not provide stdin pipe".to_string(), + Err(error) => format!( + "io_popen('w') did not provide stdin pipe; process cleanup failed: {error}" + ), + })) } _ => unreachable!("mode validated above"), } @@ -442,47 +448,117 @@ fn close_io_handle(mut handle: IoHandle) -> VmResult<()> { } fn terminate_child_tree(child: &mut Child) -> VmResult<()> { - terminate_process_tree(child.id()); - match child.try_wait() { - Ok(Some(_)) => return Ok(()), - Ok(None) => {} - Err(error) => { - return Err(VmError::HostError(format!( - "io_close popen status failed: {error}" - ))); - } + let pid = child.id(); + terminate_process_tree_and_leader( + || terminate_process_tree(pid), + || kill_and_reap_child(child), + ) + .map_err(|error| VmError::HostError(format!("io_close popen terminate failed: {error}"))) +} + +fn kill_and_reap_child(child: &mut Child) -> io::Result<()> { + if child.try_wait()?.is_some() { + return Ok(()); + } + match child.kill() { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::InvalidInput => {} + Err(error) => return Err(error), + } + child.wait().map(|_| ()) +} + +fn terminate_process_tree_and_leader( + terminate_tree: Tree, + terminate_leader: Leader, +) -> io::Result<()> +where + Tree: FnOnce() -> io::Result<()>, + Leader: FnOnce() -> io::Result<()>, +{ + let tree_result = terminate_tree(); + let leader_result = terminate_leader(); + combine_process_cleanup_results(tree_result, leader_result) +} + +fn combine_process_cleanup_results( + tree_result: io::Result<()>, + leader_result: io::Result<()>, +) -> io::Result<()> { + match (tree_result, leader_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(tree_error), Ok(())) => Err(tree_error), + (Ok(()), Err(leader_error)) => Err(leader_error), + (Err(tree_error), Err(leader_error)) => Err(io::Error::new( + tree_error.kind(), + format!( + "process-tree termination failed: {tree_error}; direct leader cleanup also failed: {leader_error}" + ), + )), } - child - .kill() - .or_else(|error| { - if error.kind() == std::io::ErrorKind::InvalidInput { - Ok(()) - } else { - Err(error) - } - }) - .map_err(|error| VmError::HostError(format!("io_close popen terminate failed: {error}")))?; - child - .wait() - .map_err(|error| VmError::HostError(format!("io_close popen wait failed: {error}")))?; - Ok(()) } -fn terminate_process_tree(pid: u32) { +fn terminate_process_tree(pid: u32) -> io::Result<()> { + if pid == 0 { + return Ok(()); + } #[cfg(unix)] - if let Ok(pid) = libc::pid_t::try_from(pid) { - unsafe { - libc::kill(-pid, libc::SIGKILL); - } + { + terminate_unix_process_group_with( + pid, + |process_group, signal| unsafe { libc::kill(process_group, signal) }, + io::Error::last_os_error, + ) } #[cfg(windows)] { - let _ = Command::new("taskkill") - .args(["/T", "/F", "/PID", &pid.to_string()]) - .status(); + run_taskkill_with(pid, Command::status) } #[cfg(not(any(unix, windows)))] - let _ = pid; + { + let _ = pid; + Ok(()) + } +} + +#[cfg(unix)] +fn terminate_unix_process_group_with( + pid: u32, + kill: Kill, + last_error: LastError, +) -> io::Result<()> +where + Kill: FnOnce(libc::pid_t, libc::c_int) -> libc::c_int, + LastError: FnOnce() -> io::Error, +{ + let pid = libc::pid_t::try_from(pid) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "process id exceeds pid_t"))?; + if kill(-pid, libc::SIGKILL) == 0 { + return Ok(()); + } + let error = last_error(); + if error.raw_os_error() == Some(libc::ESRCH) { + Ok(()) + } else { + Err(error) + } +} + +#[cfg(any(windows, test))] +fn run_taskkill_with(pid: u32, run: Run) -> io::Result<()> +where + Run: FnOnce(&mut Command) -> io::Result, +{ + let mut command = Command::new("taskkill"); + command.args(["/T", "/F", "/PID", &pid.to_string()]); + let status = run(&mut command)?; + if status.success() { + Ok(()) + } else { + Err(io::Error::other(format!( + "taskkill exited with status {status}" + ))) + } } fn read_line_from_reader(reader: &mut impl Read) -> VmResult { @@ -502,3 +578,85 @@ fn read_line_from_reader(reader: &mut impl Read) -> VmResult { } Ok(String::from_utf8_lossy(&bytes).into_owned()) } + +#[cfg(test)] +mod tests { + use std::cell::Cell; + use std::io; + + use super::*; + + #[test] + fn process_tree_failure_still_attempts_direct_blocking_leader_cleanup() { + let leader_attempted = Cell::new(false); + let error = terminate_process_tree_and_leader( + || { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "tree denied", + )) + }, + || { + leader_attempted.set(true); + Ok(()) + }, + ) + .expect_err("tree failure must propagate"); + + assert!(leader_attempted.get()); + assert_eq!(error.kind(), io::ErrorKind::PermissionDenied); + assert!(error.to_string().contains("tree denied")); + } + + #[cfg(unix)] + #[test] + fn unix_process_group_signal_treats_esrch_as_success() { + terminate_unix_process_group_with( + 42, + |_, _| -1, + || io::Error::from_raw_os_error(libc::ESRCH), + ) + .expect("an already absent process group is successfully terminated"); + + let error = terminate_unix_process_group_with( + 42, + |_, _| -1, + || io::Error::from_raw_os_error(libc::EPERM), + ) + .expect_err("other process group failures must propagate"); + assert_eq!(error.raw_os_error(), Some(libc::EPERM)); + } + + #[test] + fn taskkill_launch_failure_is_an_error() { + let launch = run_taskkill_with(42, |_| { + Err(io::Error::new(io::ErrorKind::NotFound, "taskkill missing")) + }) + .expect_err("taskkill launch failure must propagate"); + assert_eq!(launch.kind(), io::ErrorKind::NotFound); + } + + #[cfg(unix)] + #[test] + fn unsuccessful_taskkill_status_is_an_error() { + let status = run_taskkill_with(42, |_| { + std::process::Command::new("sh") + .args(["-c", "exit 7"]) + .status() + }) + .expect_err("unsuccessful taskkill status must propagate"); + assert!(status.to_string().contains("status")); + } + + #[cfg(windows)] + #[test] + fn unsuccessful_taskkill_status_is_an_error() { + let status = run_taskkill_with(42, |_| { + std::process::Command::new("cmd") + .args(["/C", "exit", "7"]) + .status() + }) + .expect_err("unsuccessful taskkill status must propagate"); + assert!(status.to_string().contains("status")); + } +} diff --git a/tests/builtins/io_async_tests.rs b/tests/builtins/io_async_tests.rs index 2754f0b1..9d5f3f4e 100644 --- a/tests/builtins/io_async_tests.rs +++ b/tests/builtins/io_async_tests.rs @@ -102,6 +102,8 @@ fn io_implementations_use_only_generic_async_and_inline_sync_lifecycles() { let blocking_source = include_str!("../../src/builtins/runtime/io/blocking.rs"); for forbidden in [ + "AtomicBool", + "AtomicU32", "std::thread", "thread::Builder", "JoinHandle", @@ -116,6 +118,9 @@ fn io_implementations_use_only_generic_async_and_inline_sync_lifecycles() { "close_scheduled", "close_future", "owner_alive", + "process_id:", + "try_lock()", + "wake_by_ref()", "OperationSpec", "schedule_io_task", "worker_done", @@ -176,7 +181,13 @@ fn pid_is_running(pid: u32) -> bool { return false; }; let stat_path = std::path::PathBuf::from(format!("/proc/{pid}/stat")); - if std::fs::read_to_string(stat_path).is_err() { + let Ok(stat) = std::fs::read_to_string(stat_path) else { + return false; + }; + if stat + .split_once(") ") + .is_some_and(|(_, state)| state.starts_with('Z')) + { return false; } let result = unsafe { libc::kill(pid, 0) }; @@ -283,6 +294,71 @@ fn async_io_reset_kills_and_reaps_the_entire_popen_process_group() { let _ = std::fs::remove_file(marker_path); } +#[cfg(unix)] +#[test] +fn async_io_vm_drop_terminates_live_popen_process_tree() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should follow Unix epoch") + .as_nanos(); + let base = std::env::temp_dir().join(format!( + "pd-vm-async-drop-tree-{}-{nonce}", + std::process::id() + )); + let parent_path = base.with_extension("parent"); + let descendant_path = base.with_extension("descendant"); + let marker_path = base.with_extension("marker"); + let command = process_tree_command(&parent_path, &descendant_path, &marker_path); + let source = guest_popen_program(&command, "h;"); + + let compiled = compile_source(&format!("use io;\n{source}")).expect("source should compile"); + let mut vm = Vm::new(compiled.program); + super::async_test_bridge::install(&mut vm); + assert!(matches!( + vm.run().expect("run should start"), + VmStatus::Waiting(_) + )); + vm.wait_for_host_op_blocking() + .expect("popen should complete"); + assert!(matches!( + vm.resume().expect("program should complete"), + VmStatus::Halted + )); + + let parent_pid = wait_for_file(&parent_path) + .trim() + .parse::() + .expect("parent pid"); + let descendant_pid = wait_for_file(&descendant_path) + .trim() + .parse::() + .expect("descendant pid"); + let _guard = ProcessGroupGuard { + parent: Some(parent_pid), + descendant: Some(descendant_pid), + }; + + drop(vm); + + assert!( + wait_for_pid_exit(parent_pid), + "the popen parent must be gone" + ); + assert!( + wait_for_pid_exit(descendant_pid), + "the popen descendant must be gone" + ); + std::thread::sleep(std::time::Duration::from_millis(1_200)); + assert!( + !marker_path.exists(), + "VM drop must prevent descendants from continuing" + ); + + let _ = std::fs::remove_file(parent_path); + let _ = std::fs::remove_file(descendant_path); + let _ = std::fs::remove_file(marker_path); +} + #[cfg(unix)] #[test] fn async_io_failed_resource_handoff_cleans_up_the_popen_process_group() { From 8b5a409b8819a5a9816784ed1c28921f0e1c5887 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sat, 19 Sep 2026 17:04:39 +0800 Subject: [PATCH 06/23] fix(io): reap process leader before cleanup error --- src/builtins/runtime/io/async_io.rs | 104 ++++++++++++++++++++++++---- 1 file changed, 91 insertions(+), 13 deletions(-) diff --git a/src/builtins/runtime/io/async_io.rs b/src/builtins/runtime/io/async_io.rs index 445d3502..b60ec0a2 100644 --- a/src/builtins/runtime/io/async_io.rs +++ b/src/builtins/runtime/io/async_io.rs @@ -53,12 +53,28 @@ impl Drop for IoHandle { /// The typed resource stored in the execution scope for one async IO handle. struct IoResource { handle: Arc>>, + process_tree_terminator: fn(u32) -> io::Result<()>, + deferred_process_cleanup_error: Option, } impl IoResource { fn new(handle: IoHandle) -> Self { Self { handle: Arc::new(Mutex::new(Some(handle))), + process_tree_terminator: terminate_process_id, + deferred_process_cleanup_error: None, + } + } + + #[cfg(test)] + fn new_with_process_tree_terminator( + handle: IoHandle, + process_tree_terminator: fn(u32) -> io::Result<()>, + ) -> Self { + Self { + handle: Arc::new(Mutex::new(Some(handle))), + process_tree_terminator, + deferred_process_cleanup_error: None, } } @@ -75,24 +91,39 @@ impl IoResource { } fn begin_close_after_operations_quiesce(&mut self) -> ResourceResult { + let process_tree_terminator = self.process_tree_terminator; let slot = self.exclusive_handle_slot()?; - match slot.as_mut() { - None => Ok(CloseProgress::Ready), + let (progress, cleanup_error) = match slot.as_mut() { + None => (CloseProgress::Ready, None), Some(IoHandle::File(_)) => { slot.take(); - Ok(CloseProgress::Ready) + (CloseProgress::Ready, None) } Some(IoHandle::PopenRead { child, .. }) | Some(IoHandle::PopenWrite { child, .. }) => { - start_terminate_child_tree(child).map_err(process_resource_error)?; - Ok(CloseProgress::Pending) + let cleanup_error = + start_terminate_child_tree_with(child, process_tree_terminator).err(); + (CloseProgress::Pending, cleanup_error) } - } + }; + self.deferred_process_cleanup_error = cleanup_error; + Ok(progress) } fn poll_process_close(&mut self, cx: &mut Context<'_>) -> Poll> { - let slot = match self.exclusive_handle_slot() { - Ok(slot) => slot, - Err(error) => return Poll::Ready(Err(error)), + let Self { + handle, + deferred_process_cleanup_error, + .. + } = self; + let slot = match Arc::get_mut(handle).map(Mutex::get_mut) { + Some(slot) => slot, + None => { + return Poll::Ready(Err(ResourceError::new( + ResourceErrorCode::ResourceCleanupFailed, + "io::resource", + "async IO handle remained borrowed after host operations quiesced", + ))); + } }; let poll = match slot.as_mut() { None | Some(IoHandle::File(_)) => Poll::Ready(Ok(())), @@ -113,11 +144,17 @@ impl IoResource { }; match poll { Poll::Pending => Poll::Pending, - Poll::Ready(Ok(_)) => { + Poll::Ready(leader_result) => { slot.take(); - Poll::Ready(Ok(())) + let prior_result = match deferred_process_cleanup_error.take() { + Some(error) => Err(error), + None => Ok(()), + }; + Poll::Ready( + combine_process_cleanup_results(prior_result, leader_result) + .map_err(process_resource_error), + ) } - Poll::Ready(Err(error)) => Poll::Ready(Err(process_resource_error(error))), } } } @@ -254,8 +291,15 @@ fn combine_process_cleanup_results( } fn start_terminate_child_tree(child: &mut Child) -> io::Result<()> { + start_terminate_child_tree_with(child, terminate_process_id) +} + +fn start_terminate_child_tree_with( + child: &mut Child, + terminate_tree: impl FnOnce(u32) -> io::Result<()>, +) -> io::Result<()> { let pid = child.id().unwrap_or(0); - let tree_result = terminate_process_id(pid); + let tree_result = terminate_tree(pid); let leader_result = child.start_kill().or_else(ignore_already_exited); combine_process_cleanup_results(tree_result, leader_result) } @@ -836,6 +880,40 @@ mod tests { ); } + #[cfg(unix)] + #[tokio::test] + async fn resource_shutdown_reaps_leader_before_reporting_tree_failure() { + let handle = spawn_shell_command("sleep 30", "r").expect("process should spawn"); + let pid = match &handle { + IoHandle::PopenRead { child, .. } => child.id().expect("child pid"), + other => panic!("expected popen read handle, got {other:?}"), + }; + let resource = IoResource::new_with_process_tree_terminator(handle, |_| { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "injected tree failure", + )) + }); + let mut resources = crate::vm::resource::ResourceTable::new().expect("resource table"); + resources.push(resource).expect("resource insert"); + + let error = + std::future::poll_fn(|cx| resources.poll_close_all(ResourceCloseReason::VmReset, cx)) + .await + .expect_err("tree failure must propagate after shutdown"); + + assert!( + resources.is_empty(), + "resource must be reclaimed after reap" + ); + assert!( + !std::path::Path::new(&format!("/proc/{pid}")).exists(), + "resource shutdown must reap the direct child before returning the tree error" + ); + assert_eq!(error.code(), ResourceErrorCode::ResourceCleanupFailed); + assert!(error.to_string().contains("injected tree failure")); + } + #[tokio::test] async fn process_tree_failure_still_attempts_direct_async_leader_cleanup() { let leader_attempted = Arc::new(StdMutex::new(false)); From 62b630096560d6094feedf45295d7eb9321fbcc7 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sat, 19 Sep 2026 14:25:23 +0800 Subject: [PATCH 07/23] refactor(sqlite): use macro-owned async hosts --- Cargo.lock | 208 ++- Cargo.toml | 5 +- docs/sqlite.md | 27 +- src/builtins/runtime/sqlite.rs | 1241 ++++++----------- .../builtins/sqlite_scope_lifecycle_tests.rs | 209 ++- tests/sqlite_async_host_arch_tests.rs | 63 + tests/sqlite_named_struct_tests.rs | 47 +- 7 files changed, 863 insertions(+), 937 deletions(-) create mode 100644 tests/sqlite_async_host_arch_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 21d8bf4e..491c2289 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,18 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", -] - [[package]] name = "aho-corasick" version = "1.1.4" @@ -277,6 +265,21 @@ version = "0.129.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d953932541249c91e3fa70a75ff1e52adc62979a2a8132145d4b9b3e6d1a9b6a" +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + [[package]] name = "displaydoc" version = "0.2.7" @@ -357,6 +360,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -429,35 +438,38 @@ checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" [[package]] name = "hashbrown" -version = "0.14.5" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "ahash", + "foldhash 0.1.5", ] [[package]] name = "hashbrown" -version = "0.15.5" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ - "foldhash", + "foldhash 0.2.0", ] [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash 0.2.0", +] [[package]] name = "hashlink" -version = "0.9.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.17.1", ] [[package]] @@ -690,6 +702,16 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -710,9 +732,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libsqlite3-sys" -version = "0.30.1" +version = "0.38.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" dependencies = [ "cc", "pkg-config", @@ -866,6 +888,7 @@ dependencies = [ "serde_json", "syn 2.0.117", "tokio", + "tokio-rusqlite", "tower-service", "url", "windows-sys 0.59.0", @@ -1012,6 +1035,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror", +] + [[package]] name = "rt-format" version = "0.3.1" @@ -1024,9 +1057,9 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.32.1" +version = "0.40.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" dependencies = [ "bitflags 2.11.0", "fallible-iterator", @@ -1034,6 +1067,7 @@ dependencies = [ "hashlink", "libsqlite3-sys", "smallvec", + "sqlite-wasm-rs", ] [[package]] @@ -1097,6 +1131,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + [[package]] name = "rustyline" version = "14.0.0" @@ -1209,6 +1249,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -1275,6 +1327,26 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + [[package]] name = "tinystr" version = "0.8.4" @@ -1312,6 +1384,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tokio-rusqlite" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bce5670317c76f8505e7e5a8d655330a435fc35d3deaf756de8406db64a94e8c" +dependencies = [ + "crossbeam-channel", + "rusqlite", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -1461,12 +1544,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - [[package]] name = "want" version = "0.3.1" @@ -1482,6 +1559,51 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.4", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] + [[package]] name = "wasmtime-internal-core" version = "42.0.1" @@ -1653,26 +1775,6 @@ dependencies = [ "synstructure", ] -[[package]] -name = "zerocopy" -version = "0.8.56" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.56" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "zerofrom" version = "0.1.8" diff --git a/Cargo.toml b/Cargo.toml index 0b3cb604..47b9f756 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ http-client = [ "dep:tower-service", "dep:url", ] -sqlite = ["runtime", "dep:rusqlite"] +sqlite = ["async", "dep:rusqlite", "dep:tokio-rusqlite"] edge-abi = [ "dep:edge_abi", "edge_abi/console", @@ -87,7 +87,8 @@ cranelift-jit = { version = "0.129.1", optional = true } cranelift-module = { version = "0.129.1", optional = true } cranelift-native = { version = "0.129.1", optional = true } pd-host-function = { path = "./pd-host-function", version = "0.1.0" } -rusqlite = { version = "0.32", default-features = false, features = ["bundled", "hooks", "limits"], optional = true } +rusqlite = { version = "0.40.1", default-features = false, features = ["bundled", "hooks", "limits"], optional = true } +tokio-rusqlite = { version = "0.8", optional = true } edge_abi = { package = "pd-edge-abi", version = "0.1.1", default-features = false, optional = true } futures-channel = "0.3" paste = "1" diff --git a/docs/sqlite.md b/docs/sqlite.md index 014ef013..4852966a 100644 --- a/docs/sqlite.md +++ b/docs/sqlite.md @@ -13,8 +13,9 @@ arguments are borrowed for operations and consumed by `sqlite::close`. - `sqlite::close` The embedding policy controls the allowed database root, unsafe-SQL capability, and host -ceilings. Configure that policy before opening a connection. Each operation is asynchronous; -the VM resumes after the host operation completes. +ceilings. Configure that policy before opening a connection. Each operation is an ordinary +macro-owned async host function. It captures owned call data and awaits `tokio-rusqlite`, which +serializes work on the connection and owns the blocking SQLite execution thread. ## Compiler and editor catalog boundary @@ -25,9 +26,10 @@ feature. Catalog-aware compiler callers and the LSP use these declarations for n field access and exact host signatures. The `sqlite` feature controls the executable SQLite module, generated SQLite namespace and -callables, the `rusqlite` dependency, and SQLite registration exports. A runtime build without -that feature can inspect the editor/compiler contract but has no SQLite implementation to bind; -execution requires a build with `sqlite` enabled and the SQLite module registered. +callables, the `rusqlite` and `tokio-rusqlite` dependencies, and SQLite registration exports. A +runtime build without that feature can inspect the editor/compiler contract but has no SQLite +implementation to bind; execution requires a build with `sqlite` enabled, an async host bridge, +and the SQLite module registered. ## Open options (`SqliteOpenOptions`) @@ -133,7 +135,7 @@ let rowid = inserted.last_insert_rowid; | `last_insert_rowid` | int | The parameter count and decoded text/blob byte length are checked against the connection -limits before the operation is scheduled. +limits before the adapter call is awaited. ## Query (`SqliteQueryResult` and `SqliteRow`) @@ -206,11 +208,14 @@ SqliteTransactionResult { An execute statement produces `{ kind: "execute", execute: ... }`; a query statement produces `{ kind: "query", query: ... }`. The unselected envelope field is `null`. Discriminate with `kind` before using `execute` or `query`. The transaction remains atomic: statement order, -rollback on failure, cancellation, deadlines, and result limits are preserved. +rollback on failure, transaction deadlines, and result limits are preserved. A SQLite progress +handler interrupts a transaction after its configured deadline so the transaction rolls back. ## Resource lifecycle -`sqlite::close(db)` consumes the connection, cancels pending operations on it, and waits for -worker cleanup through the VM execution scope. VM reset closes remaining connections and retires -pending SQLite operations. Handles are VM-local and generation-checked, so a closed or foreign -handle cannot be reused. +`sqlite::close(db)` consumes the connection, awaits adapter close, and removes the VM resource. +VM reset interrupts an active SQLite statement through the connection's interrupt handle, drops +the adapter handle, and retires submitted futures through the generic async bridge. Cancelling an +individual submitted future only drops that waiter; `tokio-rusqlite` may finish work already +queued or running. The host layer adds no worker or stronger cancellation mechanism. Handles are +VM-local and generation-checked, so a closed or foreign handle cannot be reused. diff --git a/src/builtins/runtime/sqlite.rs b/src/builtins/runtime/sqlite.rs index 8f32462f..6ea18844 100644 --- a/src/builtins/runtime/sqlite.rs +++ b/src/builtins/runtime/sqlite.rs @@ -1,33 +1,20 @@ //! Scoped SQLite host functions (optional `sqlite` feature). //! -//! SQLite connections are typed [`HostResource`]s owned by the VM's -//! [`ExecutionScope`](crate::vm::execution_scope::ExecutionScope), exactly -//! like IO handles. Pending `sqlite::execute` / `sqlite::query` / -//! `sqlite::transaction` work is driven by concrete [`HostOperation`] -//! drivers registered in the same scope and polled/cancelled directly by the -//! operation registry. There is no poller table, no operation-owner enum, and -//! no callback-payload resource: the driver holds the shared connection slot -//! and the scope drives its lifecycle. +//! Each public SQLite function is an ordinary macro-owned async host function. +//! Calls capture owned SQL, parameters, and connection context before awaiting +//! `tokio-rusqlite`, which owns the blocking SQLite execution thread. The host +//! layer owns no worker, operation driver, mailbox, or manual wakeup state. //! -//! Connection cleanup is adapter-owned: closing the resource (via -//! `sqlite::close`, VM reset, or scope drop) interrupts the connection -//! through the slot the resource owns and marks it closed. Pending drivers on -//! that connection observe the closed state and are retired through the -//! generic scope close, so no `close_resources_by_type` / -//! `cancel_operations_by_owner` helper is needed. -//! -//! Bounds preserved from the PR16 source: statement byte length, parameter -//! count and byte length, result rows/columns/bytes, connection count, -//! transaction statement count, and transaction deadline, plus SQL-safety -//! rejection and read-only enforcement. +//! Connections remain typed [`HostResource`] values in the VM execution scope. +//! A resource stores the adapter handle, immutable policy/limits, and only the +//! open/closed and in-flight accounting needed for configured limits. Explicit +//! close and scope teardown use SQLite's interrupt handle, while cancellation +//! of an individual submitted future has the semantics provided by the adapter. use std::fs; use std::path::{Component, Path, PathBuf}; +use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex}; -use std::task::{Context, Poll, Waker}; -use std::thread; -use std::thread::JoinHandle; use std::time::{Duration, Instant}; use pd_host_function::pd_host_function; @@ -36,23 +23,20 @@ use rusqlite::limits::Limit; use rusqlite::types::{Value as SqlValue, ValueRef}; use rusqlite::{Connection, OpenFlags, TransactionBehavior, params_from_iter}; -use super::typed::{VmArrayRef, VmMapRef}; -use super::{HostCallResult, VmMap}; +use super::VmMap; +use super::typed::{VmArrayHandle, VmArrayRef}; use crate::host_api::{HostApiCatalog, ResourceTypeKey}; -use crate::vm::operation::driver::HostOperation; -use crate::vm::operation::error::{OperationError, OperationErrorCode, OperationResult}; -use crate::vm::operation::reason::OperationCancelReason; -use crate::vm::operation::{OperationId, OperationOutcome, OperationSpec}; use crate::vm::resource::close::{CloseProgress, HostResource}; use crate::vm::resource::error::ResourceResult; use crate::vm::resource::{ResourceCloseReason, ResourceHandle}; -use crate::vm::{CallReturn, HostFunctionRegistry, HostOpId, Value, Vm, VmError, VmResult}; +use crate::vm::{ + CaptureAsyncHostContext, HostFunctionRegistry, HostFutureOutput, Value, Vm, VmError, VmResult, +}; -/// SQLite `progress_handler` step cadence used to surface cancellation while a -/// statement runs. +/// SQLite `progress_handler` step cadence used to enforce transaction deadlines. const SQLITE_PROGRESS_STEPS: i32 = 1_000; -/// Bounded SQLite connection/query limits, mirroring the PR16 source surface. +/// Bounded SQLite connection/query limits, mirroring the published surface. #[derive(Clone, Copy, Debug)] pub struct SqliteLimits { pub max_connections: usize, @@ -110,411 +94,155 @@ struct OpenOptions { allow_unsafe_sql: bool, } -/// Shared, adapter-owned per-connection state. -/// -/// The connection itself is a [`Mutex`] (SQLite connections are -/// not thread-safe), serialized by the `execution` mutex so at most one -/// worker uses the connection at a time. The slot records the currently -/// executing operation and every in-flight operation on this connection so -/// close can retire them without a type-dispatched helper. -struct ConnectionSlot { - connection: Mutex, - execution: Mutex<()>, - /// The operation currently executing on this connection, if any. - active_operation: Mutex>, - /// Every in-flight operation scheduled against this connection. - pending: Mutex>, - /// Workers that have been scheduled but whose completion guard has not - /// retired yet. This closes the publish/unregister tail window. - live_workers: AtomicUsize, - /// Waker for a resource close waiting for `pending` to become empty. - close_waker: Mutex>, - interrupt: Arc, - limits: SqliteLimits, - allow_unsafe_sql: bool, - closed: AtomicBool, +struct ConnectionCountPermit { + open_connections: Arc, } -impl ConnectionSlot { - fn register(&self, id: OperationId) { - self.pending.lock().expect("sqlite pending lock").push(id); - self.live_workers.fetch_add(1, Ordering::Release); - } - - fn unregister(&self, id: OperationId) { - let removed = { - let mut pending = self.pending.lock().expect("sqlite pending lock"); - let before = pending.len(); - pending.retain(|candidate| *candidate != id); - pending.len() != before - }; - if !removed { - return; - } - let workers = self.live_workers.fetch_sub(1, Ordering::AcqRel) - 1; - if self.pending_count() == 0 - && workers == 0 - && let Some(waker) = self - .close_waker - .lock() - .expect("sqlite close waker lock") - .take() - { - waker.wake(); - } - } - - fn register_close_waker(&self, waker: &Waker) { - if self.pending_count() == 0 && self.live_workers.load(Ordering::Acquire) == 0 { - return; - } - { - let mut close_waker = self.close_waker.lock().expect("sqlite close waker lock"); - *close_waker = Some(waker.clone()); - } - if self.pending_count() == 0 - && self.live_workers.load(Ordering::Acquire) == 0 - && let Some(waker) = self - .close_waker - .lock() - .expect("sqlite close waker lock") - .take() - { - waker.wake(); - } - } - - fn drained(&self) -> bool { - self.pending_count() == 0 && self.live_workers.load(Ordering::Acquire) == 0 - } - - fn pending_count(&self) -> usize { - self.pending.lock().expect("sqlite pending lock").len() +impl Drop for ConnectionCountPermit { + fn drop(&mut self) { + self.open_connections.fetch_sub(1, Ordering::AcqRel); } } -/// The typed connection resource stored in the execution scope. -/// -/// The slot is `Arc`-shared with worker threads so a closing resource does not -/// free the connection out from under an in-flight worker; the last Arc drops -/// the `Connection`. `begin_close` is exact-once: it marks the slot closed and -/// interrupts any currently executing statement so cancellation is prompt. -struct SqliteResource { - slot: Arc, - /// Adapter-owned live-connection counter (decremented on close). +fn reserve_connection( open_connections: Arc, - counter_released: bool, + limit: usize, +) -> VmResult { + open_connections + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { + (count < limit).then_some(count + 1) + }) + .map_err(|_| VmError::HostError(format!("SQLite connection limit {limit} reached")))?; + Ok(ConnectionCountPermit { open_connections }) } -impl SqliteResource { - fn new(slot: Arc, open_connections: Arc) -> Self { - Self { - slot, - open_connections, - counter_released: false, - } - } +struct SqliteOperationLease { + in_flight: Arc, +} - fn release_connection(&mut self) { - if !self.counter_released { - self.open_connections.fetch_sub(1, Ordering::SeqCst); - self.counter_released = true; - } +impl Drop for SqliteOperationLease { + fn drop(&mut self) { + self.in_flight.fetch_sub(1, Ordering::AcqRel); } } +/// The one script-visible SQLite connection resource. +struct SqliteResource { + connection: tokio_rusqlite::Connection, + interrupt: Arc, + limits: SqliteLimits, + allow_unsafe_sql: bool, + closed: Arc, + in_flight: Arc, + _connection_permit: ConnectionCountPermit, +} + impl HostResource for SqliteResource { fn resource_type_key() -> Option { ResourceTypeKey::new(super::sqlite_schema::SQLITE_CONNECTION_KEY).ok() } fn begin_close(&mut self, _reason: ResourceCloseReason) -> ResourceResult { - if !self.slot.closed.swap(true, Ordering::AcqRel) { - self.slot.interrupt.interrupt(); - } - if self.slot.drained() { - self.release_connection(); - Ok(CloseProgress::Ready) - } else { - Ok(CloseProgress::Pending) - } - } - - fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll> { - if self.slot.drained() { - self.release_connection(); - Poll::Ready(Ok(())) - } else { - self.slot.register_close_waker(cx.waker()); - if self.slot.drained() { - self.release_connection(); - Poll::Ready(Ok(())) - } else { - Poll::Pending - } + if !self.closed.swap(true, Ordering::AcqRel) { + self.interrupt.interrupt(); } + Ok(CloseProgress::Ready) } } -impl Drop for SqliteResource { - fn drop(&mut self) { - self.release_connection(); - } +#[derive(Default)] +pub(crate) struct SqliteState { + pub(crate) open_connections: Arc, } -/// Shared state between one SQLite worker, its [`SqliteOpDriver`] operation, -/// and the adapter-owned completion hook on the VM thread. -/// -/// The worker writes the terminal signal and guest-visible value; the driver -/// reflects the signal into the operation registry and the VM wrapper reads -/// the value after the registry drive returns terminal. -struct SqliteOpShared { - cancelled: AtomicBool, - worker_done: AtomicBool, - signal: Mutex>>, - value: Mutex>>, - waker: Mutex>, - quiescence_waker: Mutex>, - worker: Mutex>>, -} - -impl SqliteOpShared { - fn new() -> Self { - Self { - cancelled: AtomicBool::new(false), - worker_done: AtomicBool::new(false), - signal: Mutex::new(None), - value: Mutex::new(None), - waker: Mutex::new(None), - quiescence_waker: Mutex::new(None), - worker: Mutex::new(None), - } - } - - fn is_quiescent(&self) -> bool { - self.worker_done.load(Ordering::Acquire) - } - - fn mark_worker_done(&self) { - self.worker_done.store(true, Ordering::Release); - if let Some(waker) = self - .quiescence_waker - .lock() - .expect("sqlite quiescence waker lock") - .take() - { - waker.wake(); - } - } - - fn register_quiescence_waker(&self, waker: &Waker) { - let mut guard = self - .quiescence_waker - .lock() - .expect("sqlite quiescence waker lock"); - if self.is_quiescent() { - return; - } - *guard = Some(waker.clone()); - if self.is_quiescent() - && let Some(waker) = guard.take() - { - waker.wake(); - } - } - - fn set_worker(&self, worker: JoinHandle<()>) { - *self.worker.lock().expect("sqlite worker lock") = Some(worker); - } - - fn join_worker(&self) -> bool { - self.worker - .lock() - .expect("sqlite worker lock") - .take() - .is_some_and(|worker| worker.join().is_err()) - } - - fn is_cancelled(&self) -> bool { - self.cancelled.load(Ordering::SeqCst) - } - - fn publish(&self, signal: Result<(), String>) { - *self.signal.lock().expect("sqlite signal lock") = Some(signal); - if let Some(waker) = self.waker.lock().expect("sqlite waker lock").take() { - waker.wake(); - } - } - - fn take_signal(&self) -> Option> { - self.signal.lock().expect("sqlite signal lock").take() - } - - fn register_waker(&self, waker: &Waker) { - *self.waker.lock().expect("sqlite waker lock") = Some(waker.clone()); - } - - fn fail(&self, error: VmError) { - let message = error.to_string(); - *self.value.lock().expect("sqlite value lock") = Some(Err(error)); - self.publish(Err(message)); - } - - fn succeed(&self, value: CallReturn) { - *self.value.lock().expect("sqlite value lock") = Some(Ok(value)); - self.publish(Ok(())); - } +fn sqlite_state(vm: &mut Vm) -> VmResult<&mut SqliteState> { + vm.execution_scope() + .scope_state_or_insert_with(SqliteState::default) + .map_err(|error| VmError::HostError(format!("sqlite scope state unavailable: {error}"))) } -/// A concrete [`HostOperation`] driver for one pending SQLite operation. -/// -/// The operation id is filled in by [`schedule_operation`] after -/// [`ExecutionScope::start_operation`](crate::vm::execution_scope::ExecutionScope::start_operation) -/// assigns it, because the registry allocates packed ids internally. The -/// shared cell is written exactly once, before the driver can be polled or -/// cancelled (the operation is registered with the driver already boxed, but -/// the registry only drives it once the scheduler returns). -struct SqliteOpDriver { - shared: Arc, - slot: Arc, - id: Arc>>, - name: String, -} - -impl SqliteOpDriver { - fn new( - shared: Arc, - slot: Arc, - name: impl Into, - ) -> Self { - Self { - shared, - slot, - id: Arc::new(Mutex::new(None)), - name: name.into(), - } - } +#[derive(Clone)] +pub(super) struct SqliteOpenContext { + policy: SqlitePolicy, + open_connections: Arc, +} - fn worker_failed(&self, message: String) -> Poll> { - Poll::Ready(Err(OperationError::new( - OperationErrorCode::OperationDriverFailed, - "sqlite::operation", - message, - ))) +impl CaptureAsyncHostContext for SqliteOpenContext { + fn capture(vm: &mut Vm) -> VmResult { + let policy = current_policy(vm).clone(); + let open_connections = Arc::clone(&sqlite_state(vm)?.open_connections); + Ok(Self { + policy, + open_connections, + }) } } -impl HostOperation for SqliteOpDriver { - fn poll(&mut self, cx: &mut Context<'_>) -> Poll> { - if !self.shared.is_quiescent() { - self.shared.register_waker(cx.waker()); - self.shared.register_quiescence_waker(cx.waker()); - if !self.shared.is_quiescent() { - return Poll::Pending; - } - } - if self.shared.is_cancelled() { - return self.worker_failed(format!("{} was cancelled", self.name)); - } - match self.shared.take_signal() { - Some(Ok(())) => Poll::Ready(Ok(())), - Some(Err(message)) => self.worker_failed(message), - None => self.worker_failed(format!( - "{} worker terminated without a completion signal", - self.name - )), - } - } +#[derive(Clone)] +pub(super) struct SqliteConnectionContext { + handle: ResourceHandle, + connection: tokio_rusqlite::Connection, + interrupt: Arc, + limits: SqliteLimits, + allow_unsafe_sql: bool, + closed: Arc, + in_flight: Arc, +} - fn cancel(&mut self, _reason: OperationCancelReason) -> OperationResult<()> { - self.shared.cancelled.store(true, Ordering::Release); - // If this operation is the one currently executing on the connection, - // interrupt the statement so the worker aborts promptly. Interrupting a - // connection with no active statement is a harmless no-op. - let is_active = self - .id - .lock() - .expect("sqlite driver id lock") - .is_some_and(|id| { - *self - .slot - .active_operation - .lock() - .expect("sqlite active lock") - == Some(id) - }); - if self.slot.closed.load(Ordering::Acquire) || is_active { - self.slot.interrupt.interrupt(); +impl SqliteConnectionContext { + fn ensure_open(&self) -> VmResult<()> { + if self.closed.load(Ordering::Acquire) { + Err(VmError::HostError( + "SQLite database is already closed".to_string(), + )) + } else { + Ok(()) } - Ok(()) } - fn is_quiescent(&self) -> bool { - self.shared.is_quiescent() - } - - fn register_quiescence_waker(&mut self, cx: &Context<'_>) { - self.shared.register_quiescence_waker(cx.waker()); - } - - fn cancel_and_wait(&mut self, reason: OperationCancelReason) -> OperationResult<()> { - self.cancel(reason)?; - if self.shared.join_worker() { - return Err(OperationError::new( - OperationErrorCode::OperationDriverFailed, - "sqlite::operation", - format!("{} worker panicked while cancelling", self.name), + fn begin_operation(&self) -> VmResult { + self.ensure_open()?; + let limit = self.limits.max_pending_operations; + self.in_flight + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { + (count < limit).then_some(count + 1) + }) + .map_err(|_| { + VmError::HostError(format!("SQLite pending operation limit {limit} reached")) + })?; + if self.closed.load(Ordering::Acquire) { + self.in_flight.fetch_sub(1, Ordering::AcqRel); + return Err(VmError::HostError( + "SQLite database is already closed".to_string(), )); } - Ok(()) + Ok(SqliteOperationLease { + in_flight: Arc::clone(&self.in_flight), + }) } } -impl Drop for SqliteOpDriver { - fn drop(&mut self) { - if !self.shared.is_quiescent() { - let _ = self.cancel(OperationCancelReason::VmDrop); - } - let _ = self.shared.join_worker(); +impl CaptureAsyncHostContext for SqliteConnectionContext { + fn capture(_vm: &mut Vm) -> VmResult { + Err(VmError::HostError( + "SQLite connection context requires call arguments".to_string(), + )) } -} -/// The per-VM SQLite adapter runtime state, mirroring the IO subsystem. -/// -/// Lives in the execution scope's typed arena (accessed lazily through -/// `ExecutionScope::scope_state_or_insert_with`), so it follows the scope -/// lifecycle: it is destroyed on reset/drop and recreated fresh on next use. -/// It owns the completion mailboxes for pending operations and an -/// adapter-owned counter of live connections used to enforce -/// `max_connections`. The embedding policy (`SqlitePolicy`) is *persistent* -/// module state stored in the generic `ModuleStateStore`, so it survives -/// `reset_for_reuse` while this runtime state does not. -pub(crate) struct SqliteState { - /// Adapter-owned live connection count, shared with each - /// [`SqliteResource`] so `begin_close` can decrement it. Avoids a generic - /// by-type close helper. - pub(crate) open_connections: Arc, -} - -impl Default for SqliteState { - fn default() -> Self { - Self { - open_connections: Arc::new(AtomicUsize::new(0)), - } + fn capture_with_args(vm: &mut Vm, args: &[Value]) -> VmResult { + let db_id = match args.first() { + Some(Value::Int(value)) => *value, + Some(_) => return Err(VmError::TypeMismatch("int")), + None => { + return Err(VmError::HostError( + "missing SQLite database argument".to_string(), + )); + } + }; + lookup_connection(vm, db_id) } } -/// Returns the adapter-declared SQLite scope state (the per-op completion -/// mailbox and the live-connection counter), creating the empty default on -/// first access while the scope is Active. The state is owned by the -/// execution-scope arena, so it is destroyed with the scope on reset and -/// recreated lazily on next use. -fn sqlite_state(vm: &mut Vm) -> VmResult<&mut SqliteState> { - vm.execution_scope() - .scope_state_or_insert_with(SqliteState::default) - .map_err(|error| VmError::HostError(format!("sqlite scope state unavailable: {error}"))) -} - /// The default SQLite embedding policy used when no policy has been /// configured through [`SqliteHostExt::configure_sqlite`]. This is the /// value `SqlitePolicy::default()` produces, constructed explicitly so it can @@ -558,35 +286,26 @@ fn sqlite_error(error: rusqlite::Error) -> VmError { VmError::HostError(format!("SQLite error {name} ({code}): {error}")) } -fn cancellation_message(shared: &SqliteOpShared) -> String { - if shared.is_cancelled() { - "SQLite operation cancelled".to_string() - } else { - "SQLite connection was closed".to_string() +fn adapter_call_error(error: tokio_rusqlite::Error) -> VmError { + match error { + tokio_rusqlite::Error::ConnectionClosed => { + VmError::HostError("SQLite connection was closed".to_string()) + } + tokio_rusqlite::Error::Close((_, error)) => sqlite_error(error), + tokio_rusqlite::Error::Error(error) => error, + _ => VmError::HostError(format!("SQLite adapter error: {error}")), } } -/// Completes one operation after the generic scope registry reports a -/// terminal outcome. The adapter owns the mailbox; the VM only invokes this -/// opaque completion hook. -fn finish_sqlite_operation( - _vm: &mut Vm, - op_id: OperationId, - outcome: OperationOutcome, - shared: Arc, -) -> VmResult { - // A cancelled/closed operation reports a guest-visible error even if the - // worker happened to complete concurrently. - if matches!(outcome, OperationOutcome::Cancelled(_)) || shared.is_cancelled() { - return Err(VmError::HostError(cancellation_message(&shared))); - } - let value = shared.value.lock().expect("sqlite value lock").take(); - match value { - Some(value) => value, - None => Err(VmError::HostError(format!( - "scoped operation {} completed without a result", - op_id.raw() - ))), +fn adapter_close_error(error: tokio_rusqlite::Error) -> VmError { + match error { + tokio_rusqlite::Error::ConnectionClosed => { + VmError::HostError("SQLite connection was closed".to_string()) + } + tokio_rusqlite::Error::Close((_, error)) | tokio_rusqlite::Error::Error(error) => { + sqlite_error(error) + } + _ => VmError::HostError(format!("SQLite adapter error: {error}")), } } @@ -605,12 +324,8 @@ fn sqlite_handle(handle_id: i64) -> VmResult { }) } -/// Lifts a guest-visible integer handle into a typed, live scope token. -/// -/// This validates arena, slot, generation, open state, and `TypeId` through -/// the generic typed table — a foreign, stale, closed, or wrong-typed handle -/// is rejected here before any SQLite state is touched. -fn lookup_connection(vm: &mut Vm, handle_id: i64) -> VmResult> { +/// Lifts a guest-visible integer handle into owned adapter call context. +fn lookup_connection(vm: &mut Vm, handle_id: i64) -> VmResult { let handle = sqlite_handle(handle_id)?; let token = vm .execution_scope() @@ -622,12 +337,20 @@ fn lookup_connection(vm: &mut Vm, handle_id: i64) -> VmResult(&token) .map_err(|error| VmError::HostError(format!("SQLite database borrow failed: {error}")))?; - if resource.slot.closed.load(Ordering::SeqCst) { + if resource.closed.load(Ordering::Acquire) { return Err(VmError::HostError( "SQLite database is already closed".to_string(), )); } - Ok(Arc::clone(&resource.slot)) + Ok(SqliteConnectionContext { + handle, + connection: resource.connection.clone(), + interrupt: Arc::clone(&resource.interrupt), + limits: resource.limits, + allow_unsafe_sql: resource.allow_unsafe_sql, + closed: Arc::clone(&resource.closed), + in_flight: Arc::clone(&resource.in_flight), + }) } fn map_value<'a>(map: &'a VmMap, key: &str) -> Option<&'a Value> { @@ -835,48 +558,60 @@ fn sqlite_limit(value: usize, label: &str) -> VmResult { fn install_connection_limits(connection: &Connection, limits: SqliteLimits) -> VmResult<()> { let max_value_bytes = limits.max_result_bytes.max(limits.max_parameter_bytes); - connection.set_limit( - Limit::SQLITE_LIMIT_LENGTH, - sqlite_limit(max_value_bytes, "value byte limit")?, - ); - connection.set_limit( - Limit::SQLITE_LIMIT_SQL_LENGTH, - sqlite_limit(limits.max_statement_bytes, "statement byte limit")?, - ); - connection.set_limit( - Limit::SQLITE_LIMIT_COLUMN, - sqlite_limit(limits.max_columns, "column limit")?, - ); - connection.set_limit( - Limit::SQLITE_LIMIT_VARIABLE_NUMBER, - sqlite_limit(limits.max_parameters, "parameter count limit")?, - ); + connection + .set_limit( + Limit::SQLITE_LIMIT_LENGTH, + sqlite_limit(max_value_bytes, "value byte limit")?, + ) + .map_err(sqlite_error)?; + connection + .set_limit( + Limit::SQLITE_LIMIT_SQL_LENGTH, + sqlite_limit(limits.max_statement_bytes, "statement byte limit")?, + ) + .map_err(sqlite_error)?; + connection + .set_limit( + Limit::SQLITE_LIMIT_COLUMN, + sqlite_limit(limits.max_columns, "column limit")?, + ) + .map_err(sqlite_error)?; + connection + .set_limit( + Limit::SQLITE_LIMIT_VARIABLE_NUMBER, + sqlite_limit(limits.max_parameters, "parameter count limit")?, + ) + .map_err(sqlite_error)?; Ok(()) } -fn install_authorizer(connection: &Connection, allow_unsafe_sql: bool) { - connection.authorizer(Some(move |context: AuthContext<'_>| { - if allow_unsafe_sql { - return Authorization::Allow; - } - match context.action { - AuthAction::Attach { .. } - | AuthAction::Detach { .. } - | AuthAction::Pragma { .. } - | AuthAction::CreateVtable { .. } - | AuthAction::DropVtable { .. } - | AuthAction::Unknown { .. } => Authorization::Deny, - AuthAction::Function { function_name } - if function_name.eq_ignore_ascii_case("load_extension") => - { - Authorization::Deny +fn install_authorizer(connection: &Connection, allow_unsafe_sql: bool) -> VmResult<()> { + connection + .authorizer(Some(move |context: AuthContext<'_>| { + if allow_unsafe_sql { + return Authorization::Allow; } - _ => Authorization::Allow, - } - })); + match context.action { + AuthAction::Attach { .. } + | AuthAction::Detach { .. } + | AuthAction::Pragma { .. } + | AuthAction::CreateVtable { .. } + | AuthAction::DropVtable { .. } + | AuthAction::Unknown { .. } => Authorization::Deny, + AuthAction::Function { function_name } + if function_name.eq_ignore_ascii_case("load_extension") => + { + Authorization::Deny + } + _ => Authorization::Allow, + } + })) + .map_err(sqlite_error) } -fn open_connection(options: &OpenOptions) -> VmResult { +async fn open_connection( + options: &OpenOptions, +) -> VmResult<(tokio_rusqlite::Connection, Arc)> { let path = resolve_database_path(options)?; let flags = match options.mode { OpenMode::Memory => OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE, @@ -887,16 +622,24 @@ fn open_connection(options: &OpenOptions) -> VmResult { } } | OpenFlags::SQLITE_OPEN_NO_MUTEX; let connection = match path { - Some(path) => Connection::open_with_flags(path, flags), - None => Connection::open_in_memory_with_flags(flags), + Some(path) => tokio_rusqlite::Connection::open_with_flags(path, flags).await, + None => tokio_rusqlite::Connection::open_in_memory_with_flags(flags).await, } .map_err(sqlite_error)?; - connection - .busy_timeout(Duration::from_millis(options.limits.busy_timeout_ms)) - .map_err(sqlite_error)?; - install_connection_limits(&connection, options.limits)?; - install_authorizer(&connection, options.allow_unsafe_sql); - Ok(connection) + let limits = options.limits; + let allow_unsafe_sql = options.allow_unsafe_sql; + let interrupt = connection + .call(move |connection| { + connection + .busy_timeout(Duration::from_millis(limits.busy_timeout_ms)) + .map_err(sqlite_error)?; + install_connection_limits(connection, limits)?; + install_authorizer(connection, allow_unsafe_sql)?; + Ok(connection.get_interrupt_handle()) + }) + .await + .map_err(adapter_call_error)?; + Ok((connection, Arc::new(interrupt))) } fn normalized_sql(sql: &str) -> VmResult { @@ -1139,36 +882,6 @@ fn sqlite_params(values: VmArrayRef<'_>, limits: SqliteLimits) -> VmResult( - slot: &ConnectionSlot, - shared: &Arc, - operation: impl FnOnce(&mut Connection) -> Result, -) -> VmResult { - if slot.closed.load(Ordering::Acquire) || shared.is_cancelled() { - return Err(VmError::HostError(cancellation_message(shared))); - } - let mut connection = slot - .connection - .lock() - .map_err(|_| VmError::HostError("SQLite connection lock is poisoned".to_string()))?; - if slot.closed.load(Ordering::Acquire) || shared.is_cancelled() { - return Err(VmError::HostError(cancellation_message(shared))); - } - let handler_shared = Arc::clone(shared); - connection.progress_handler( - SQLITE_PROGRESS_STEPS, - Some(move || handler_shared.is_cancelled()), - ); - let result = operation(&mut connection); - connection.progress_handler(0, None:: bool>); - if slot.closed.load(Ordering::Acquire) || shared.is_cancelled() { - return Err(VmError::HostError(cancellation_message(shared))); - } - result.map_err(sqlite_error) -} - fn estimate_value_bytes(value: &Value) -> usize { match value { Value::Null => 1, @@ -1360,136 +1073,7 @@ fn transaction_result_value(kind: &str, execute: Option, query: Option, - shared: Arc, - id: OperationId, -} - -impl Drop for SqliteWorkerCompletion { - fn drop(&mut self) { - if let Ok(mut active) = self.slot.active_operation.lock() - && *active == Some(self.id) - { - *active = None; - } - self.shared.mark_worker_done(); - self.slot.unregister(self.id); - } -} - -/// Schedules a worker thread to run one SQLite operation on a connection and -/// registers its [`SqliteOpDriver`] in the VM's execution scope. -/// -/// The driver is constructed with a shared id cell that -/// [`ExecutionScope::start_operation`](crate::vm::execution_scope::ExecutionScope::start_operation) -/// fills in after allocating the packed operation id, so the driver's `cancel` -/// can compare against the connection's active operation without a registry -/// fixup. The worker holds the connection's execution mutex for the whole -/// operation (serializing access, since SQLite connections are not -/// thread-safe), records itself as the active operation, and publishes the -/// terminal signal plus the guest-visible value through the shared mailbox. -fn schedule_operation( - vm: &mut Vm, - slot: Arc, - operation: impl FnOnce(Arc, Arc) -> VmResult - + Send - + 'static, -) -> VmResult { - if slot.closed.load(Ordering::SeqCst) { - return Err(VmError::HostError( - "SQLite database is already closed".to_string(), - )); - } - if slot.pending_count() >= slot.limits.max_pending_operations { - return Err(VmError::HostError(format!( - "SQLite pending operation limit {} reached", - slot.limits.max_pending_operations - ))); - } - - let shared = Arc::new(SqliteOpShared::new()); - let worker_shared = Arc::clone(&shared); - let worker_slot = Arc::clone(&slot); - let worker_name = "sqlite::operation".to_string(); - let driver = SqliteOpDriver::new(Arc::clone(&shared), Arc::clone(&slot), worker_name.clone()); - let driver_id = Arc::clone(&driver.id); - - let deadline = - Instant::now().checked_add(Duration::from_millis(slot.limits.max_transaction_ms)); - let spec = OperationSpec::new(driver) - .with_deadline(deadline.unwrap_or_else(|| Instant::now() + Duration::from_secs(3600))); - - let op_id = vm - .execution_scope() - .start_operation(spec) - .map_err(|error| { - VmError::HostError(format!("failed to start sqlite operation: {error}")) - })?; - *driver_id - .lock() - .expect("sqlite driver id lock should not be poisoned") = Some(op_id); - slot.register(op_id); - if let Err(error) = vm.register_scoped_operation_completion(op_id, { - let completion_shared = Arc::clone(&shared); - move |vm, outcome| finish_sqlite_operation(vm, op_id, outcome, completion_shared) - }) { - let _ = vm - .execution_scope() - .abort_operation(op_id, OperationCancelReason::Requested); - slot.unregister(op_id); - return Err(error); - } - let raw = op_id.raw(); - - let worker = thread::Builder::new() - .name(format!("rustscript-sqlite-{raw}")) - .spawn(move || { - let _completion = SqliteWorkerCompletion { - slot: Arc::clone(&worker_slot), - shared: Arc::clone(&worker_shared), - id: op_id, - }; - let _execution = worker_slot - .execution - .lock() - .expect("SQLite execution lock should not be poisoned"); - if worker_slot.closed.load(Ordering::Acquire) || worker_shared.is_cancelled() { - worker_shared.fail(VmError::HostError(cancellation_message(&worker_shared))); - return; - } - *worker_slot - .active_operation - .lock() - .expect("SQLite active operation lock should not be poisoned") = Some(op_id); - if worker_slot.closed.load(Ordering::Acquire) || worker_shared.is_cancelled() { - worker_shared.fail(VmError::HostError(cancellation_message(&worker_shared))); - return; - } - let result = operation(Arc::clone(&worker_slot), Arc::clone(&worker_shared)); - match result { - Ok(value) => worker_shared.succeed(value), - Err(error) => worker_shared.fail(error), - } - }) - .map_err(|error| { - shared.mark_worker_done(); - vm.discard_scoped_operation_completion(op_id); - let _ = vm - .execution_scope() - .abort_operation(op_id, OperationCancelReason::Requested); - slot.unregister(op_id); - VmError::HostError(format!("failed to spawn sqlite worker: {error}")) - })?; - shared.set_worker(worker); - - Ok(raw) -} - -/// Parses the `sqlite::open` options map against the adapter-owned embedding -/// policy. -fn parse_open_options(vm: &Vm, options: &VmMap) -> VmResult { - let policy = current_policy(vm); +fn parse_open_options(policy: &SqlitePolicy, options: &VmMap) -> VmResult { let path = required_string(options, "path")?; let mode = match optional_string(options, "mode")?.as_deref() { Some("memory") => OpenMode::Memory, @@ -1527,88 +1111,93 @@ fn parse_open_options(vm: &Vm, options: &VmMap) -> VmResult { } /// Opens a SQLite database under the embedding-owned path and limit policy. -/// -/// The connection is stored as a typed [`SqliteResource`] in the execution -/// scope; the guest-visible handle is the raw scope handle, validated for -/// arena, slot, generation, open state, and type on every later use. The -/// live-connection count is adapter-owned (shared with each resource) so -/// `max_connections` is enforced without a generic by-type helper. #[pd_host_function(name = "sqlite::open", contract = super::sqlite_schema::sqlite_open_contract)] -pub(super) fn builtin_sqlite_open_impl(vm: &mut Vm, options: VmMapRef<'_>) -> VmResult { - let options = parse_open_options(vm, options)?; - // The adapter-declared scope state owns the live-connection counter; - // clone the `Arc` so the scope borrow ends before `push_resource` below. - let open_connections: Arc = Arc::clone(&sqlite_state(vm)?.open_connections); - if open_connections.load(Ordering::SeqCst) >= options.limits.max_connections { - return Err(VmError::HostError(format!( - "SQLite connection limit {} reached", - options.limits.max_connections - ))); - } - let connection = open_connection(&options)?; - let interrupt = connection.get_interrupt_handle(); - let slot = Arc::new(ConnectionSlot { - connection: Mutex::new(connection), - execution: Mutex::new(()), - active_operation: Mutex::new(None), - pending: Mutex::new(Vec::new()), - live_workers: AtomicUsize::new(0), - close_waker: Mutex::new(None), - interrupt: Arc::new(interrupt), +pub(super) async fn builtin_sqlite_open_impl( + #[pd_host_context] context: SqliteOpenContext, + options: VmMap, +) -> VmResult> { + let options = parse_open_options(&context.policy, &options)?; + let connection_permit = reserve_connection( + Arc::clone(&context.open_connections), + options.limits.max_connections, + )?; + let (connection, interrupt) = open_connection(&options).await?; + let resource = SqliteResource { + connection, + interrupt, limits: options.limits, allow_unsafe_sql: options.allow_unsafe_sql, - closed: AtomicBool::new(false), - }); - let resource = vm - .execution_scope() - .push_resource(SqliteResource::new(slot, Arc::clone(&open_connections))) - .map_err(|error| VmError::HostError(format!("failed to open SQLite database: {error}")))?; - open_connections.fetch_add(1, Ordering::SeqCst); - Ok(handle_value(resource.handle())) + closed: Arc::new(AtomicBool::new(false)), + in_flight: Arc::new(AtomicUsize::new(0)), + _connection_permit: connection_permit, + }; + Ok(HostFutureOutput::complete(move |vm| { + let token = vm + .execution_scope() + .push_resource(resource) + .map_err(|error| { + VmError::HostError(format!("failed to open SQLite database: {error}")) + })?; + Ok(handle_value(token.handle())) + })) } /// Executes one parameterized SQLite statement asynchronously. -#[pd_host_function(name = "sqlite::execute", contract = super::sqlite_schema::sqlite_execute_contract, runtime_owned_pending)] -pub(super) fn builtin_sqlite_execute_impl( - vm: &mut Vm, - db_id: i64, - sql: &str, - params: VmArrayRef<'_>, -) -> VmResult> { - let slot = lookup_connection(vm, db_id)?; - validate_sql(sql, slot.limits, slot.allow_unsafe_sql)?; - let sql = sql.to_string(); - let params = sqlite_params(params, slot.limits)?; - let op_id = schedule_operation(vm, slot, move |slot, shared| { - with_connection(&slot, &shared, |connection| { - execute_with_connection(connection, &sql, ¶ms) +#[pd_host_function(name = "sqlite::execute", contract = super::sqlite_schema::sqlite_execute_contract)] +pub(super) async fn builtin_sqlite_execute_impl( + #[pd_host_context] context: SqliteConnectionContext, + _db_id: i64, + sql: String, + params: VmArrayHandle, +) -> VmResult { + let _lease = context.begin_operation()?; + validate_sql(&sql, context.limits, context.allow_unsafe_sql)?; + let params = sqlite_params(params.as_ref(), context.limits)?; + let closed = Arc::clone(&context.closed); + let value = context + .connection + .call(move |connection| { + if closed.load(Ordering::Acquire) { + return Err(VmError::HostError( + "SQLite database is already closed".to_string(), + )); + } + execute_with_connection(connection, &sql, ¶ms).map_err(sqlite_error) }) - .map(|value| CallReturn::one(Value::Map(Arc::new(value)))) - })?; - Ok(HostCallResult::Pending(op_id)) + .await + .map_err(adapter_call_error)?; + context.ensure_open()?; + Ok(value) } /// Runs one parameterized SQLite query with row and result-byte bounds. -#[pd_host_function(name = "sqlite::query", contract = super::sqlite_schema::sqlite_query_contract, runtime_owned_pending)] -pub(super) fn builtin_sqlite_query_impl( - vm: &mut Vm, - db_id: i64, - sql: &str, - params: VmArrayRef<'_>, - limits: VmMapRef<'_>, -) -> VmResult> { - let slot = lookup_connection(vm, db_id)?; - let query_limits = parse_query_limits(limits, slot.limits)?; - validate_sql(sql, query_limits, slot.allow_unsafe_sql)?; - let sql = sql.to_string(); - let params = sqlite_params(params, slot.limits)?; - let op_id = schedule_operation(vm, slot, move |slot, shared| { - with_connection(&slot, &shared, |connection| { - query_with_connection(connection, &sql, ¶ms, query_limits) +#[pd_host_function(name = "sqlite::query", contract = super::sqlite_schema::sqlite_query_contract)] +pub(super) async fn builtin_sqlite_query_impl( + #[pd_host_context] context: SqliteConnectionContext, + _db_id: i64, + sql: String, + params: VmArrayHandle, + limits: VmMap, +) -> VmResult { + let _lease = context.begin_operation()?; + let query_limits = parse_query_limits(&limits, context.limits)?; + validate_sql(&sql, query_limits, context.allow_unsafe_sql)?; + let params = sqlite_params(params.as_ref(), context.limits)?; + let closed = Arc::clone(&context.closed); + let value = context + .connection + .call(move |connection| { + if closed.load(Ordering::Acquire) { + return Err(VmError::HostError( + "SQLite database is already closed".to_string(), + )); + } + query_with_connection(connection, &sql, ¶ms, query_limits).map_err(sqlite_error) }) - .map(|value| CallReturn::one(Value::Map(Arc::new(value)))) - })?; - Ok(HostCallResult::Pending(op_id)) + .await + .map_err(adapter_call_error)?; + context.ensure_open()?; + Ok(value) } struct TransactionStatement { @@ -1667,54 +1256,131 @@ fn parse_transaction_statements( .collect() } +fn transaction_with_connection( + connection: &mut Connection, + statements: Vec, + deadline: Instant, + max_transaction_ms: u64, +) -> VmResult> { + connection + .progress_handler( + SQLITE_PROGRESS_STEPS, + Some(move || Instant::now() >= deadline), + ) + .map_err(sqlite_error)?; + let result = (|| { + let transaction = connection + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(sqlite_error)?; + let mut results = Vec::with_capacity(statements.len()); + for statement in statements { + if Instant::now() >= deadline { + return Err(VmError::HostError(format!( + "SQLite transaction exceeded the configured {max_transaction_ms} ms deadline" + ))); + } + let result = if statement.query { + let value = query_with_connection( + &transaction, + &statement.sql, + &statement.params, + statement.limits, + ) + .map_err(sqlite_error)?; + transaction_result_value("query", None, Some(value)) + } else { + let value = + execute_with_connection(&transaction, &statement.sql, &statement.params) + .map_err(sqlite_error)?; + transaction_result_value("execute", Some(value), None) + }; + results.push(result); + } + if Instant::now() >= deadline { + return Err(VmError::HostError(format!( + "SQLite transaction exceeded the configured {max_transaction_ms} ms deadline" + ))); + } + transaction.commit().map_err(sqlite_error)?; + Ok(results) + })(); + connection + .progress_handler(0, None:: bool>) + .map_err(sqlite_error)?; + if Instant::now() >= deadline && result.is_err() { + return Err(VmError::HostError(format!( + "SQLite transaction exceeded the configured {max_transaction_ms} ms deadline" + ))); + } + result +} + /// Runs ordered statements atomically and returns ordered result envelopes. -#[pd_host_function(name = "sqlite::transaction", contract = super::sqlite_schema::sqlite_transaction_contract, runtime_owned_pending)] -pub(super) fn builtin_sqlite_transaction_impl( - vm: &mut Vm, - db_id: i64, - statements: VmArrayRef<'_>, -) -> VmResult>> { - let slot = lookup_connection(vm, db_id)?; - let statements = parse_transaction_statements(statements, slot.limits, slot.allow_unsafe_sql)?; - let op_id = schedule_operation(vm, slot, move |slot, shared| { - with_connection(&slot, &shared, |connection| { - let transaction = - connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - let mut results = Vec::with_capacity(statements.len()); - for statement in statements { - let result = if statement.query { - let value = query_with_connection( - &transaction, - &statement.sql, - &statement.params, - statement.limits, - )?; - transaction_result_value("query", None, Some(value)) - } else { - let value = - execute_with_connection(&transaction, &statement.sql, &statement.params)?; - transaction_result_value("execute", Some(value), None) - }; - results.push(result); +#[pd_host_function(name = "sqlite::transaction", contract = super::sqlite_schema::sqlite_transaction_contract)] +pub(super) async fn builtin_sqlite_transaction_impl( + #[pd_host_context] context: SqliteConnectionContext, + _db_id: i64, + statements: VmArrayHandle, +) -> VmResult> { + let _lease = context.begin_operation()?; + let statements = parse_transaction_statements( + statements.as_ref(), + context.limits, + context.allow_unsafe_sql, + )?; + let max_transaction_ms = context.limits.max_transaction_ms; + let deadline = Instant::now() + .checked_add(Duration::from_millis(max_transaction_ms)) + .ok_or_else(|| { + VmError::HostError("SQLite transaction deadline is out of range".to_string()) + })?; + let closed = Arc::clone(&context.closed); + let value = context + .connection + .call(move |connection| { + if closed.load(Ordering::Acquire) { + return Err(VmError::HostError( + "SQLite database is already closed".to_string(), + )); } - transaction.commit()?; - Ok(results) + transaction_with_connection(connection, statements, deadline, max_transaction_ms) }) - .map(|values| CallReturn::one(Value::array(values))) - })?; - Ok(HostCallResult::Pending(op_id)) + .await + .map_err(adapter_call_error)?; + context.ensure_open()?; + Ok(value) } -/// Closes a SQLite resource through the generic scope close. Pending drivers -/// on the connection observe the closed slot and are retired through the -/// scope's operation registry; no type-dispatched helper is needed. +/// Closes the adapter connection, then removes its VM resource. #[pd_host_function(name = "sqlite::close", contract = super::sqlite_schema::sqlite_close_contract)] -pub(super) fn builtin_sqlite_close_impl(vm: &mut Vm, db_id: i64) -> VmResult<()> { - let handle = sqlite_handle(db_id)?; - vm.execution_scope() - .close_resource::(handle, ResourceCloseReason::Requested) - .map_err(|error| VmError::HostError(format!("unknown SQLite database: {error}")))?; - Ok(()) +pub(super) async fn builtin_sqlite_close_impl( + #[pd_host_context] context: SqliteConnectionContext, + _db_id: i64, +) -> VmResult> { + let _lease = context.begin_operation()?; + if context.closed.swap(true, Ordering::AcqRel) { + return Err(VmError::HostError( + "SQLite database is already closed".to_string(), + )); + } + context.interrupt.interrupt(); + if let Err(error) = context.connection.close().await { + context.closed.store(false, Ordering::Release); + return Err(adapter_close_error(error)); + } + let handle = context.handle; + Ok(HostFutureOutput::complete(move |vm| { + let progress = vm + .execution_scope() + .close_resource::(handle, ResourceCloseReason::Requested) + .map_err(|error| VmError::HostError(format!("unknown SQLite database: {error}")))?; + if progress != CloseProgress::Ready { + return Err(VmError::HostError( + "SQLite resource removal remained pending".to_string(), + )); + } + Ok(()) + })) } /// Every SQLite catalog function the feature-enabled build owns. @@ -1804,88 +1470,3 @@ impl SqliteHostExt for Vm { current_policy(self) } } - -#[cfg(test)] -mod tests { - use super::*; - - fn test_operation_id(slot: u64) -> OperationId { - OperationId::from_raw((1 << 43) | (slot << 22) | 1).expect("valid test operation id") - } - - #[test] - fn close_waits_for_active_and_queued_workers() { - let connection = Connection::open_in_memory().expect("in-memory SQLite connection"); - let interrupt = connection.get_interrupt_handle(); - let slot = Arc::new(ConnectionSlot { - connection: Mutex::new(connection), - execution: Mutex::new(()), - active_operation: Mutex::new(None), - pending: Mutex::new(Vec::new()), - live_workers: AtomicUsize::new(0), - close_waker: Mutex::new(None), - interrupt: Arc::new(interrupt), - limits: SqliteLimits::default(), - allow_unsafe_sql: false, - closed: AtomicBool::new(false), - }); - let open_connections = Arc::new(AtomicUsize::new(1)); - let mut resource = SqliteResource::new(Arc::clone(&slot), Arc::clone(&open_connections)); - let active_id = test_operation_id(1); - let queued_id = test_operation_id(2); - slot.register(active_id); - slot.register(queued_id); - - let release_active = Arc::new(AtomicBool::new(false)); - let active_started = Arc::new(AtomicBool::new(false)); - let active_slot = Arc::clone(&slot); - let active_release = Arc::clone(&release_active); - let active_started_flag = Arc::clone(&active_started); - let active = thread::spawn(move || { - let _execution = active_slot.execution.lock().expect("execution lock"); - active_started_flag.store(true, Ordering::Release); - while !active_release.load(Ordering::Acquire) { - thread::yield_now(); - } - active_slot.unregister(active_id); - }); - while !active_started.load(Ordering::Acquire) { - thread::yield_now(); - } - - let queued_started = Arc::new(AtomicBool::new(false)); - let queued_executed = Arc::new(AtomicBool::new(false)); - let queued_slot = Arc::clone(&slot); - let queued_started_flag = Arc::clone(&queued_started); - let queued_executed_flag = Arc::clone(&queued_executed); - let queued = thread::spawn(move || { - queued_started_flag.store(true, Ordering::Release); - let _execution = queued_slot.execution.lock().expect("execution lock"); - if !queued_slot.closed.load(Ordering::Acquire) { - queued_executed_flag.store(true, Ordering::Release); - } - queued_slot.unregister(queued_id); - }); - while !queued_started.load(Ordering::Acquire) { - thread::yield_now(); - } - - assert_eq!( - resource - .begin_close(ResourceCloseReason::Requested) - .expect("close should begin"), - CloseProgress::Pending - ); - let mut cx = Context::from_waker(Waker::noop()); - assert!(matches!(resource.poll_close(&mut cx), Poll::Pending)); - assert!(!queued_executed.load(Ordering::Acquire)); - - release_active.store(true, Ordering::Release); - active.join().expect("active worker should finish"); - queued.join().expect("queued worker should finish"); - assert!(!queued_executed.load(Ordering::Acquire)); - assert!(slot.drained()); - assert!(matches!(resource.poll_close(&mut cx), Poll::Ready(Ok(())))); - assert_eq!(open_connections.load(Ordering::Acquire), 0); - } -} diff --git a/tests/builtins/sqlite_scope_lifecycle_tests.rs b/tests/builtins/sqlite_scope_lifecycle_tests.rs index f8c7ced7..6aec3f51 100644 --- a/tests/builtins/sqlite_scope_lifecycle_tests.rs +++ b/tests/builtins/sqlite_scope_lifecycle_tests.rs @@ -1,26 +1,133 @@ //! Focused tests for the scoped SQLite host functions (PR16 commit 4). //! -//! Connections are typed [`HostResource`]s owned by the VM's execution -//! scope; `sqlite::execute` / `sqlite::query` / `sqlite::transaction` are -//! driven by concrete [`HostOperation`] drivers in the same scope and polled -//! through the shared operation registry. These tests exercise the +//! Connections are typed [`HostResource`]s owned by the VM's execution scope; +//! `sqlite::open` / `execute` / `query` / `transaction` / `close` are ordinary +//! macro-owned async functions backed by `tokio-rusqlite`. These tests exercise the //! scope-backed behaviour through the public VM + SQLite API: typed-value //! round trips and ordered transactions, read-only and SQL-safety policy, //! row/result-byte truncation bounds, stale/foreign/typed handle rejection, //! and adapter-owned `configure`/`clear`/`close` cleanup. +use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; +use std::task::{Context, Poll, Waker}; use std::time::{SystemTime, UNIX_EPOCH}; +use vm::operation::OperationCancelReason; use vm::{ - CompileSourceFileOptions, HostFunctionRegistry, SqliteHostExt, Vm, VmError, VmStatus, - compile_source, compile_source_with_flavor_and_options, - register_sqlite_builtin_module_from_catalog, sqlite_host_catalog, + CallReturn, CompileSourceFileOptions, HostAsyncBridge, HostFunctionRegistry, HostFuture, + HostFutureOutput, HostOpId, SqliteHostExt, Vm, VmError, VmResult, VmStatus, compile_source, + compile_source_with_flavor_and_options, register_sqlite_builtin_module_from_catalog, + sqlite_host_catalog, }; use super::vm_reset::reset_for_reuse_to_ready; +#[derive(Default)] +struct TokioHostDriver { + submitted: HashMap, +} + +impl HostAsyncBridge for TokioHostDriver { + fn submit_op(&mut self, op_id: HostOpId, future: HostFuture) -> VmResult<()> { + self.submitted.insert(op_id, future); + Ok(()) + } + + fn poll_op(&mut self, op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Err(VmError::HostError(format!( + "unknown external host operation {op_id}" + )))) + } + + fn poll_submitted_op( + &mut self, + op_id: HostOpId, + cx: &mut Context<'_>, + ) -> Poll> { + let poll = self.submitted.get_mut(&op_id).map_or_else( + || { + Poll::Ready(Err(VmError::HostError(format!( + "unknown submitted host operation {op_id}" + )))) + }, + |future| future.as_mut().poll(cx), + ); + if poll.is_ready() { + self.submitted.remove(&op_id); + } + poll + } + + fn cancel_op(&mut self, op_id: HostOpId) { + self.submitted.remove(&op_id); + } + + fn request_cancel_op( + &mut self, + op_id: HostOpId, + _reason: OperationCancelReason, + ) -> VmResult<()> { + self.submitted.remove(&op_id); + Ok(()) + } + + fn poll_cancel_op(&mut self, _op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } +} + +fn install_host_driver(vm: &mut Vm) { + vm.set_async_bridge(Box::::default()) + .expect("test async bridge should install"); +} + +fn drive_vm_to_host_error(vm: &mut Vm) -> String { + let mut status = match vm.run() { + Ok(status) => status, + Err(VmError::HostError(message)) => return message, + Err(other) => return format!("{other:?}"), + }; + loop { + status = match status { + VmStatus::Halted => panic!("expected host error, got success"), + VmStatus::Yielded => match vm.resume() { + Ok(status) => status, + Err(VmError::HostError(message)) => return message, + Err(other) => return format!("{other:?}"), + }, + VmStatus::Waiting(_) => { + if let Err(error) = vm.wait_for_host_op_blocking() { + return match error { + VmError::HostError(message) => message, + other => format!("{other:?}"), + }; + } + match vm.resume() { + Ok(status) => status, + Err(VmError::HostError(message)) => return message, + Err(other) => return format!("{other:?}"), + } + } + }; + } +} + +fn start_long_sqlite_query(vm: &mut Vm) { + let open_status = vm.run().expect("SQLite open should start"); + assert!(matches!(open_status, VmStatus::Waiting(_))); + vm.wait_for_host_op_blocking() + .expect("SQLite open should complete"); + let query_status = vm.resume().expect("SQLite query should start"); + assert!(matches!(query_status, VmStatus::Waiting(_))); + let mut cx = Context::from_waker(Waker::noop()); + assert!( + matches!(vm.poll_waiting_host_op(&mut cx), Poll::Pending), + "long SQLite query should remain pending after its first poll" + ); +} + /// Helper: run a SQLite source to completion. Scripts use `assert(...)` for /// value checks; a failed assert surfaces as a host error. fn run_sqlite_source(policy: vm::SqlitePolicy, source: &str) -> Result<(), VmError> { @@ -33,6 +140,7 @@ fn run_sqlite_source(policy: vm::SqlitePolicy, source: &str) -> Result<(), VmErr ) .expect("source should compile"); let mut vm = Vm::try_new(compiled.program)?; + install_host_driver(&mut vm); let mut registry = HostFunctionRegistry::empty(); register_sqlite_builtin_module_from_catalog(&mut registry, catalog.as_ref())?; registry.bind_vm_cached(&mut vm)?; @@ -60,13 +168,9 @@ fn run_sqlite_builtin_host_error(policy: vm::SqlitePolicy, source: &str) -> Stri let wrapped = format!("use sqlite;\n{source}"); let compiled = compile_source(&wrapped).expect("source should compile"); let mut vm = Vm::new(compiled.program); + install_host_driver(&mut vm); vm.configure_sqlite(policy); - match vm.run() { - Ok(VmStatus::Halted) => panic!("expected host error, got success"), - Ok(other) => panic!("expected host error, got status: {other:?}"), - Err(VmError::HostError(message)) => message, - Err(other) => format!("{other:?}"), - } + drive_vm_to_host_error(&mut vm) } /// Helper: run a SQLite source expecting a host error, returning its message. @@ -98,6 +202,24 @@ fn policy_for(root: &Path) -> vm::SqlitePolicy { } } +#[test] +fn sqlite_async_hosts_require_an_async_bridge() { + let compiled = compile_source( + "use sqlite;\nlet db = sqlite::open({ path: \":memory:\", mode: \"memory\", limits: {} });", + ) + .expect("source should compile"); + let mut vm = Vm::new(compiled.program); + let error = vm + .run() + .expect_err("async SQLite open must require a host bridge"); + assert!( + error + .to_string() + .contains("async host function requires a host async bridge"), + "unexpected missing-bridge error: {error}" + ); +} + #[test] fn sqlite_round_trip_supports_typed_values_and_ordered_transactions() { let root = temporary_root("round-trip"); @@ -431,14 +553,10 @@ fn sqlite_configure_and_clear_own_the_policy() { let compiled = compile_source("use sqlite;\nlet db = sqlite::open({ path: \"state.db\", mode: \"read_write_create\", limits: {} });") .expect("source should compile"); let mut vm = Vm::new(compiled.program); + install_host_driver(&mut vm); vm.configure_sqlite(policy); vm.clear_sqlite(); - let err = match vm.run() { - Ok(VmStatus::Halted) => panic!("open without a root must fail"), - Ok(_) => panic!("open without a root must fail"), - Err(VmError::HostError(message)) => message, - Err(other) => panic!("expected host error, got: {other:?}"), - }; + let err = drive_vm_to_host_error(&mut vm); assert!( err.contains("root"), "cleared policy must reject file opens, got: {err}" @@ -448,14 +566,12 @@ fn sqlite_configure_and_clear_own_the_policy() { } #[test] -fn sqlite_close_cancels_siblings_and_reset_retires_all() { +fn sqlite_close_and_reset_retire_resources() { let root = temporary_root("cancel-reset"); let policy = policy_for(&root); - // Schedule a long-running query, then close the connection while it is - // still pending. The pending driver observes the closed slot and is - // retired through the generic scope close; a fresh connection on the same - // root then works normally. + // Explicit close retires the adapter connection; a fresh connection on the + // same root then works normally. run_sqlite_source( policy.clone(), r#" @@ -475,7 +591,7 @@ fn sqlite_close_cancels_siblings_and_reset_retires_all() { sqlite::close(db2); "#, ) - .expect("close should cancel pending siblings and leave a reusable connection"); + .expect("close should retire the connection and allow a fresh connection"); // VM reset retires all pending sqlite operations and closes every open // connection through the generic scope lifecycle. @@ -484,14 +600,11 @@ fn sqlite_close_cancels_siblings_and_reset_retires_all() { ) .expect("reset source should compile"); let mut vm = Vm::new(compiled.program); + install_host_driver(&mut vm); vm.configure_sqlite(policy); - // Run until the long query is pending (the VM is waiting on it), then - // reset: the scope close must cancel the driver without hanging. - let status = vm.run().expect("run should start"); - assert!( - matches!(status, VmStatus::Waiting(_)), - "long query should leave the VM waiting, got: {status:?}" - ); + // Poll the long query once so it reaches the adapter, then reset. Scope + // close interrupts the active SQLite statement and retires the resource. + start_long_sqlite_query(&mut vm); reset_for_reuse_to_ready(&mut vm).expect("reset should reach quiescence"); assert!( vm.execution_scope().operations().is_empty(), @@ -538,7 +651,28 @@ fn sqlite_pending_operation_slots_are_reclaimed_after_completion() { } #[test] -fn sqlite_pending_reset_repeatedly_drains_workers_and_keeps_vm_reusable() { +fn sqlite_transaction_deadline_interrupts_and_rolls_back() { + let root = temporary_root("transaction-deadline"); + let error = run_sqlite_host_error( + policy_for(&root), + r#" + let db = sqlite::open({ path: "state.db", mode: "read_write_create", limits: { max_transaction_ms: 1 } }); + sqlite::transaction(&db, [{ + sql: "WITH RECURSIVE numbers(value) AS (SELECT 1 UNION ALL SELECT value + 1 FROM numbers LIMIT 10000000) SELECT sum(value) FROM numbers", + query: true, + limits: { max_rows: 1 } + }]); + "#, + ); + assert!( + error.contains("transaction exceeded") && error.contains("1 ms deadline"), + "transaction deadline must surface explicitly, got: {error}" + ); + fs::remove_dir_all(root).expect("temporary SQLite root should be removed"); +} + +#[test] +fn sqlite_pending_reset_repeatedly_keeps_vm_reusable() { let root = temporary_root("reset-stress"); let policy = policy_for(&root); let compiled = compile_source( @@ -546,16 +680,11 @@ fn sqlite_pending_reset_repeatedly_drains_workers_and_keeps_vm_reusable() { ) .expect("stress source should compile"); let mut vm = Vm::new(compiled.program); + install_host_driver(&mut vm); vm.configure_sqlite(policy); for iteration in 0..32 { - assert!( - matches!( - vm.run().expect("stress run should start"), - VmStatus::Waiting(_) - ), - "iteration {iteration} should leave the SQLite query pending" - ); + start_long_sqlite_query(&mut vm); reset_for_reuse_to_ready(&mut vm).expect("stress reset should reach quiescence"); assert!( vm.execution_scope().operations().is_empty(), diff --git a/tests/sqlite_async_host_arch_tests.rs b/tests/sqlite_async_host_arch_tests.rs new file mode 100644 index 00000000..3f60c39b --- /dev/null +++ b/tests/sqlite_async_host_arch_tests.rs @@ -0,0 +1,63 @@ +#![cfg(all(feature = "sqlite", not(target_arch = "wasm32")))] +//! Architecture guard for the macro-owned async SQLite host adapter. + +const SQLITE_SOURCE: &str = include_str!("../src/builtins/runtime/sqlite.rs"); + +#[test] +fn sqlite_host_functions_are_macro_owned_async_functions() { + for function in ["open", "execute", "query", "transaction", "close"] { + let signature = format!("async fn builtin_sqlite_{function}_impl"); + assert!( + SQLITE_SOURCE.contains(&signature), + "sqlite::{function} must be an ordinary async #[pd_host_function]" + ); + } + + assert!( + SQLITE_SOURCE.contains("CaptureAsyncHostContext"), + "SQLite calls must capture owned VM context before async submission" + ); + assert!( + SQLITE_SOURCE.contains("tokio_rusqlite::Connection"), + "the SQLite resource must use the maintained Tokio-facing adapter" + ); + assert!( + !SQLITE_SOURCE.contains("runtime_owned_pending"), + "SQLite async functions must use macro-owned future submission" + ); + assert_eq!( + SQLITE_SOURCE.matches("HostFutureOutput::complete").count(), + 2, + "only open insertion and close removal may require terminal VM completion" + ); +} + +#[test] +fn sqlite_host_owns_no_threads_or_custom_operation_driver() { + for forbidden in [ + "std::thread", + "thread::", + "JoinHandle", + "std::sync::mpsc", + "crossbeam_channel", + "AtomicWaker", + "RawWaker", + "HostOperation", + "OperationSpec", + "OperationOutcome", + "OperationCancelReason", + "SqliteWorker", + "SqliteOpDriver", + "SqliteOpShared", + "schedule_operation", + "register_scoped_operation_completion", + "completion mailbox", + "close_waker", + "quiescence_waker", + ] { + assert!( + !SQLITE_SOURCE.contains(forbidden), + "SQLite host source must not contain custom scheduling token `{forbidden}`" + ); + } +} diff --git a/tests/sqlite_named_struct_tests.rs b/tests/sqlite_named_struct_tests.rs index 94a5c570..7a5678b8 100644 --- a/tests/sqlite_named_struct_tests.rs +++ b/tests/sqlite_named_struct_tests.rs @@ -3,6 +3,7 @@ //! boundary. Runtime values remain maps; positional params, row cells, and //! transaction results use named wrappers. +use std::collections::HashMap; use std::sync::Arc; use std::task::{Context, Poll, Wake, Waker}; @@ -11,10 +12,52 @@ use vm::compiler::{ }; use vm::host_api::{HostStructField, HostTypeSchema}; use vm::{ - CompiledProgram, HostFunctionRegistry, SourcePathError, SqliteHostExt, SqlitePolicy, + CallReturn, CompiledProgram, HostAsyncBridge, HostFunctionRegistry, HostFuture, + HostFutureOutput, HostOpId, SourcePathError, SqliteHostExt, SqlitePolicy, VmError, VmResult, register_sqlite_builtin_module, sqlite_host_catalog, standard_host_catalog, }; +#[derive(Default)] +struct TokioHostDriver { + submitted: HashMap, +} + +impl HostAsyncBridge for TokioHostDriver { + fn submit_op(&mut self, op_id: HostOpId, future: HostFuture) -> VmResult<()> { + self.submitted.insert(op_id, future); + Ok(()) + } + + fn poll_op(&mut self, op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Err(VmError::HostError(format!( + "unknown external host operation {op_id}" + )))) + } + + fn poll_submitted_op( + &mut self, + op_id: HostOpId, + cx: &mut Context<'_>, + ) -> Poll> { + let poll = self.submitted.get_mut(&op_id).map_or_else( + || { + Poll::Ready(Err(VmError::HostError(format!( + "unknown submitted host operation {op_id}" + )))) + }, + |future| future.as_mut().poll(cx), + ); + if poll.is_ready() { + self.submitted.remove(&op_id); + } + poll + } + + fn cancel_op(&mut self, op_id: HostOpId) { + self.submitted.remove(&op_id); + } +} + fn opt(inner: HostTypeSchema) -> HostTypeSchema { HostTypeSchema::Optional(Box::new(inner)) } @@ -465,6 +508,8 @@ fn drive_to_halt(vm: &mut vm::vm::Vm) { fn run_compiled_sqlite(compiled: CompiledProgram) { let mut vm = vm::vm::Vm::try_new(compiled.program).expect("vm"); + vm.set_async_bridge(Box::::default()) + .expect("test async bridge should install"); let mut registry = HostFunctionRegistry::empty(); register_sqlite_builtin_module(&mut registry) .expect("sqlite exact registration should succeed"); From 4184752cf93c48c337e6faadbf01a589a9c829ee Mon Sep 17 00:00:00 2001 From: fffonion Date: Sat, 19 Sep 2026 15:45:04 +0800 Subject: [PATCH 08/23] fix(sqlite): await adapter close on reset --- docs/sqlite.md | 12 +- src/builtins/runtime/sqlite.rs | 313 ++++++++++++++++-- .../builtins/sqlite_scope_lifecycle_tests.rs | 79 ++++- 3 files changed, 369 insertions(+), 35 deletions(-) diff --git a/docs/sqlite.md b/docs/sqlite.md index 4852966a..988ee103 100644 --- a/docs/sqlite.md +++ b/docs/sqlite.md @@ -214,8 +214,10 @@ handler interrupts a transaction after its configured deadline so the transactio ## Resource lifecycle `sqlite::close(db)` consumes the connection, awaits adapter close, and removes the VM resource. -VM reset interrupts an active SQLite statement through the connection's interrupt handle, drops -the adapter handle, and retires submitted futures through the generic async bridge. Cancelling an -individual submitted future only drops that waiter; `tokio-rusqlite` may finish work already -queued or running. The host layer adds no worker or stronger cancellation mechanism. Handles are -VM-local and generation-checked, so a closed or foreign handle cannot be reused. +VM reset interrupts active SQLite work and remains pending while the resource polls +`tokio-rusqlite`'s `Connection::close`; the connection permit is released only after the adapter +confirms that queued and running work has drained and the connection has closed. Cancelling an +individual submitted future only drops that waiter. Its pending-operation lease remains owned by +the queued adapter closure until the closure runs or is discarded, so canceled work still counts +toward `max_pending_operations`. The host layer adds no worker or stronger cancellation mechanism. +Handles are VM-local and generation-checked, so a closed or foreign handle cannot be reused. diff --git a/src/builtins/runtime/sqlite.rs b/src/builtins/runtime/sqlite.rs index 6ea18844..8fb94e1b 100644 --- a/src/builtins/runtime/sqlite.rs +++ b/src/builtins/runtime/sqlite.rs @@ -6,15 +6,21 @@ //! layer owns no worker, operation driver, mailbox, or manual wakeup state. //! //! Connections remain typed [`HostResource`] values in the VM execution scope. -//! A resource stores the adapter handle, immutable policy/limits, and only the -//! open/closed and in-flight accounting needed for configured limits. Explicit -//! close and scope teardown use SQLite's interrupt handle, while cancellation -//! of an individual submitted future has the semantics provided by the adapter. +//! A resource stores the adapter handle, immutable policy/limits, close lifecycle, +//! and open/in-flight accounting needed for configured limits. Explicit close and +//! reusable scope teardown interrupt active work and await the adapter's own +//! `Connection::close` confirmation before releasing the resource permit. Dropping +//! the VM remains nonblocking, while cancellation of an individual submitted future +//! retains its operation lease in the adapter closure until that work finishes or +//! is discarded. use std::fs; +use std::future::Future; use std::path::{Component, Path, PathBuf}; -use std::sync::Arc; +use std::pin::Pin; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; use std::time::{Duration, Instant}; use pd_host_function::pd_host_function; @@ -27,7 +33,7 @@ use super::VmMap; use super::typed::{VmArrayHandle, VmArrayRef}; use crate::host_api::{HostApiCatalog, ResourceTypeKey}; use crate::vm::resource::close::{CloseProgress, HostResource}; -use crate::vm::resource::error::ResourceResult; +use crate::vm::resource::error::{ResourceError, ResourceErrorCode, ResourceResult}; use crate::vm::resource::{ResourceCloseReason, ResourceHandle}; use crate::vm::{ CaptureAsyncHostContext, HostFunctionRegistry, HostFutureOutput, Value, Vm, VmError, VmResult, @@ -126,6 +132,82 @@ impl Drop for SqliteOperationLease { } } +type SqliteCloseFuture = Pin> + Send + 'static>>; + +enum SqliteCloseState { + Open, + Closing(SqliteCloseFuture), + Finished(Result<(), String>), +} + +struct SqliteCloseLifecycle { + state: Mutex, +} + +impl SqliteCloseLifecycle { + fn new() -> Self { + Self { + state: Mutex::new(SqliteCloseState::Open), + } + } + + fn begin( + &self, + connection: tokio_rusqlite::Connection, + interrupt: &rusqlite::InterruptHandle, + closed: &AtomicBool, + ) -> Result { + let mut state = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + match &*state { + SqliteCloseState::Open => { + closed.store(true, Ordering::Release); + interrupt.interrupt(); + *state = SqliteCloseState::Closing(Box::pin(async move { + connection + .close() + .await + .map_err(adapter_close_error_message) + })); + Ok(false) + } + SqliteCloseState::Closing(_) => Ok(false), + SqliteCloseState::Finished(result) => result.clone().map(|()| true), + } + } + + fn poll(&self, cx: &mut Context<'_>) -> Poll> { + let mut state = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + match &mut *state { + SqliteCloseState::Open => Poll::Pending, + SqliteCloseState::Closing(future) => match future.as_mut().poll(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(result) => { + *state = SqliteCloseState::Finished(result.clone()); + Poll::Ready(result) + } + }, + SqliteCloseState::Finished(result) => Poll::Ready(result.clone()), + } + } + + fn reopen_after_failure(&self, closed: &AtomicBool) { + let mut state = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if matches!(&*state, SqliteCloseState::Finished(Err(_))) { + *state = SqliteCloseState::Open; + closed.store(false, Ordering::Release); + } + } +} + /// The one script-visible SQLite connection resource. struct SqliteResource { connection: tokio_rusqlite::Connection, @@ -134,6 +216,7 @@ struct SqliteResource { allow_unsafe_sql: bool, closed: Arc, in_flight: Arc, + close_lifecycle: Arc, _connection_permit: ConnectionCountPermit, } @@ -142,11 +225,28 @@ impl HostResource for SqliteResource { ResourceTypeKey::new(super::sqlite_schema::SQLITE_CONNECTION_KEY).ok() } - fn begin_close(&mut self, _reason: ResourceCloseReason) -> ResourceResult { - if !self.closed.swap(true, Ordering::AcqRel) { + fn begin_close(&mut self, reason: ResourceCloseReason) -> ResourceResult { + if reason == ResourceCloseReason::VmDrop { + self.closed.store(true, Ordering::Release); self.interrupt.interrupt(); + return Ok(CloseProgress::Ready); + } + match self.close_lifecycle.begin( + self.connection.clone(), + self.interrupt.as_ref(), + self.closed.as_ref(), + ) { + Ok(true) => Ok(CloseProgress::Ready), + Ok(false) => Ok(CloseProgress::Pending), + Err(message) => Err(sqlite_close_resource_error(message)), + } + } + + fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll> { + match self.close_lifecycle.poll(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(result) => Poll::Ready(result.map_err(sqlite_close_resource_error)), } - Ok(CloseProgress::Ready) } } @@ -187,6 +287,7 @@ pub(super) struct SqliteConnectionContext { allow_unsafe_sql: bool, closed: Arc, in_flight: Arc, + close_lifecycle: Arc, } impl SqliteConnectionContext { @@ -274,7 +375,7 @@ fn current_policy(vm: &Vm) -> &SqlitePolicy { .unwrap_or(&DEFAULT_POLICY) } -fn sqlite_error(error: rusqlite::Error) -> VmError { +fn sqlite_error_message(error: rusqlite::Error) -> String { let code = error .sqlite_error() .map(|value| value.extended_code.to_string()) @@ -283,7 +384,11 @@ fn sqlite_error(error: rusqlite::Error) -> VmError { .sqlite_error_code() .map(|value| format!("{value:?}")) .unwrap_or_else(|| "RusqliteError".to_string()); - VmError::HostError(format!("SQLite error {name} ({code}): {error}")) + format!("SQLite error {name} ({code}): {error}") +} + +fn sqlite_error(error: rusqlite::Error) -> VmError { + VmError::HostError(sqlite_error_message(error)) } fn adapter_call_error(error: tokio_rusqlite::Error) -> VmError { @@ -297,18 +402,24 @@ fn adapter_call_error(error: tokio_rusqlite::Error) -> VmError { } } -fn adapter_close_error(error: tokio_rusqlite::Error) -> VmError { +fn adapter_close_error_message(error: tokio_rusqlite::Error) -> String { match error { - tokio_rusqlite::Error::ConnectionClosed => { - VmError::HostError("SQLite connection was closed".to_string()) - } + tokio_rusqlite::Error::ConnectionClosed => "SQLite connection was closed".to_string(), tokio_rusqlite::Error::Close((_, error)) | tokio_rusqlite::Error::Error(error) => { - sqlite_error(error) + sqlite_error_message(error) } - _ => VmError::HostError(format!("SQLite adapter error: {error}")), + _ => format!("SQLite adapter error: {error}"), } } +fn sqlite_close_resource_error(message: String) -> ResourceError { + ResourceError::new( + ResourceErrorCode::ResourceCleanupFailed, + "sqlite::close", + message, + ) +} + fn handle_value(handle: ResourceHandle) -> i64 { handle.raw() as i64 } @@ -350,6 +461,7 @@ fn lookup_connection(vm: &mut Vm, handle_id: i64) -> VmResult VmResult { - let _lease = context.begin_operation()?; + let lease = context.begin_operation()?; validate_sql(&sql, context.limits, context.allow_unsafe_sql)?; let params = sqlite_params(params.as_ref(), context.limits)?; let closed = Arc::clone(&context.closed); let value = context .connection .call(move |connection| { + let _lease = lease; if closed.load(Ordering::Acquire) { return Err(VmError::HostError( "SQLite database is already closed".to_string(), @@ -1179,7 +1293,7 @@ pub(super) async fn builtin_sqlite_query_impl( params: VmArrayHandle, limits: VmMap, ) -> VmResult { - let _lease = context.begin_operation()?; + let lease = context.begin_operation()?; let query_limits = parse_query_limits(&limits, context.limits)?; validate_sql(&sql, query_limits, context.allow_unsafe_sql)?; let params = sqlite_params(params.as_ref(), context.limits)?; @@ -1187,6 +1301,7 @@ pub(super) async fn builtin_sqlite_query_impl( let value = context .connection .call(move |connection| { + let _lease = lease; if closed.load(Ordering::Acquire) { return Err(VmError::HostError( "SQLite database is already closed".to_string(), @@ -1322,7 +1437,7 @@ pub(super) async fn builtin_sqlite_transaction_impl( _db_id: i64, statements: VmArrayHandle, ) -> VmResult> { - let _lease = context.begin_operation()?; + let lease = context.begin_operation()?; let statements = parse_transaction_statements( statements.as_ref(), context.limits, @@ -1338,6 +1453,7 @@ pub(super) async fn builtin_sqlite_transaction_impl( let value = context .connection .call(move |connection| { + let _lease = lease; if closed.load(Ordering::Acquire) { return Err(VmError::HostError( "SQLite database is already closed".to_string(), @@ -1357,16 +1473,35 @@ pub(super) async fn builtin_sqlite_close_impl( #[pd_host_context] context: SqliteConnectionContext, _db_id: i64, ) -> VmResult> { - let _lease = context.begin_operation()?; + let lease = context.begin_operation()?; if context.closed.swap(true, Ordering::AcqRel) { return Err(VmError::HostError( "SQLite database is already closed".to_string(), )); } context.interrupt.interrupt(); - if let Err(error) = context.connection.close().await { - context.closed.store(false, Ordering::Release); - return Err(adapter_close_error(error)); + let _ = context + .connection + .call(move |_connection| { + drop(lease); + Ok::<(), VmError>(()) + }) + .await; + if let Err(message) = context.close_lifecycle.begin( + context.connection.clone(), + context.interrupt.as_ref(), + context.closed.as_ref(), + ) { + context + .close_lifecycle + .reopen_after_failure(context.closed.as_ref()); + return Err(VmError::HostError(message)); + } + if let Err(message) = std::future::poll_fn(|cx| context.close_lifecycle.poll(cx)).await { + context + .close_lifecycle + .reopen_after_failure(context.closed.as_ref()); + return Err(VmError::HostError(message)); } let handle = context.handle; Ok(HostFutureOutput::complete(move |vm| { @@ -1470,3 +1605,133 @@ impl SqliteHostExt for Vm { current_policy(self) } } + +#[cfg(test)] +mod tests { + use std::future::Future; + use std::task::{Context, Poll, Waker}; + use std::time::{SystemTime, UNIX_EPOCH}; + + use super::*; + + #[tokio::test] + async fn canceled_host_future_holds_operation_slot_until_adapter_closure_finishes() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock should be after the Unix epoch") + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "rustscript-sqlite-operation-lease-{}-{nonce}", + std::process::id() + )); + fs::create_dir_all(&root).expect("temporary SQLite root should be created"); + let database_path = root.join("state.db"); + let blocker = Connection::open(&database_path).expect("blocking connection should open"); + blocker + .execute_batch("CREATE TABLE items (value INTEGER); BEGIN IMMEDIATE") + .expect("blocking transaction should hold the writer lock"); + + let limits = SqliteLimits::default(); + let options = OpenOptions { + path: "state.db".to_string(), + mode: OpenMode::ReadWriteCreate, + root: Some(root.clone()), + limits, + allow_unsafe_sql: false, + }; + let (connection, interrupt) = open_connection(&options) + .await + .expect("adapter connection should open"); + let in_flight = Arc::new(AtomicUsize::new(0)); + let context = SqliteConnectionContext { + handle: ResourceHandle::encode(1, 0, 1).expect("test handle should encode"), + connection: connection.clone(), + interrupt, + limits, + allow_unsafe_sql: false, + closed: Arc::new(AtomicBool::new(false)), + in_flight: Arc::clone(&in_flight), + close_lifecycle: Arc::new(SqliteCloseLifecycle::new()), + }; + let mut operation = Box::pin(builtin_sqlite_execute_impl( + context, + 1, + "INSERT INTO items (value) VALUES (1)".to_string(), + Arc::new(Vec::new()), + )); + let mut cx = Context::from_waker(Waker::noop()); + assert!(matches!(operation.as_mut().poll(&mut cx), Poll::Pending)); + drop(operation); + + assert_eq!( + in_flight.load(Ordering::Acquire), + 1, + "canceling the host waiter must not release a queued adapter operation slot" + ); + + blocker + .execute_batch("ROLLBACK") + .expect("blocking transaction should release the writer lock"); + tokio::time::timeout(Duration::from_secs(5), async { + while in_flight.load(Ordering::Acquire) != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("adapter closure should eventually release its operation slot"); + connection + .close() + .await + .expect("adapter connection should close"); + drop(blocker); + fs::remove_dir_all(root).expect("temporary SQLite root should be removed"); + } + + #[tokio::test] + async fn operation_slots_release_on_validation_and_adapter_send_errors() { + let limits = SqliteLimits::default(); + let options = OpenOptions { + path: ":memory:".to_string(), + mode: OpenMode::Memory, + root: None, + limits, + allow_unsafe_sql: false, + }; + let (connection, interrupt) = open_connection(&options) + .await + .expect("adapter connection should open"); + let in_flight = Arc::new(AtomicUsize::new(0)); + let context = SqliteConnectionContext { + handle: ResourceHandle::encode(1, 0, 1).expect("test handle should encode"), + connection: connection.clone(), + interrupt, + limits, + allow_unsafe_sql: false, + closed: Arc::new(AtomicBool::new(false)), + in_flight: Arc::clone(&in_flight), + close_lifecycle: Arc::new(SqliteCloseLifecycle::new()), + }; + + builtin_sqlite_execute_impl(context.clone(), 1, String::new(), Arc::new(Vec::new())) + .await + .expect_err("empty SQL should fail validation"); + assert_eq!( + in_flight.load(Ordering::Acquire), + 0, + "validation failure must release its reserved operation slot" + ); + + connection + .close() + .await + .expect("adapter connection should close"); + builtin_sqlite_execute_impl(context, 1, "SELECT 1".to_string(), Arc::new(Vec::new())) + .await + .expect_err("sending a closure to a closed adapter should fail"); + assert_eq!( + in_flight.load(Ordering::Acquire), + 0, + "adapter send failure must drop the closure-owned operation slot" + ); + } +} diff --git a/tests/builtins/sqlite_scope_lifecycle_tests.rs b/tests/builtins/sqlite_scope_lifecycle_tests.rs index 6aec3f51..a2de7774 100644 --- a/tests/builtins/sqlite_scope_lifecycle_tests.rs +++ b/tests/builtins/sqlite_scope_lifecycle_tests.rs @@ -12,7 +12,7 @@ use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; use std::task::{Context, Poll, Waker}; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use vm::operation::OperationCancelReason; use vm::{ @@ -618,6 +618,59 @@ fn sqlite_close_and_reset_retire_resources() { fs::remove_dir_all(root).expect("temporary SQLite root should be removed"); } +#[test] +fn sqlite_reset_waits_for_adapter_close_before_reuse_and_prevents_late_writes() { + let root = temporary_root("reset-adapter-close"); + let database_path = root.join("state.db"); + let blocker = + rusqlite::Connection::open(&database_path).expect("blocking SQLite connection should open"); + blocker + .execute_batch("CREATE TABLE items (value INTEGER); BEGIN IMMEDIATE") + .expect("blocking transaction should hold the writer lock"); + + let compiled = compile_source( + "use sqlite;\nlet db = sqlite::open({ path: \"state.db\", mode: \"read_write_create\", limits: { busy_timeout_ms: 5000 } });\nsqlite::execute(&db, \"INSERT INTO items (value) VALUES (1)\", []);", + ) + .expect("reset source should compile"); + let mut vm = Vm::new(compiled.program); + install_host_driver(&mut vm); + vm.configure_sqlite(policy_for(&root)); + + let open_status = vm.run().expect("SQLite open should start"); + assert!(matches!(open_status, VmStatus::Waiting(_))); + vm.wait_for_host_op_blocking() + .expect("SQLite open should complete"); + let write_status = vm.resume().expect("blocked SQLite write should start"); + assert!(matches!(write_status, VmStatus::Waiting(_))); + let mut cx = Context::from_waker(Waker::noop()); + assert!(matches!(vm.poll_waiting_host_op(&mut cx), Poll::Pending)); + std::thread::sleep(Duration::from_millis(50)); + assert!(matches!(vm.poll_waiting_host_op(&mut cx), Poll::Pending)); + + vm.reset_for_reuse().expect("reset should start"); + assert!( + vm.scope_reset_pending(), + "reset must retain the scope until tokio-rusqlite confirms close" + ); + assert!(matches!(vm.poll_reset_for_reuse(&mut cx), Poll::Pending)); + + blocker + .execute_batch("ROLLBACK") + .expect("blocking transaction should release the writer lock"); + reset_for_reuse_to_ready(&mut vm).expect("reset should finish after adapter close"); + + let verifier = rusqlite::Connection::open(&database_path) + .expect("verification SQLite connection should open"); + let count: i64 = verifier + .query_row("SELECT count(*) FROM items", [], |row| row.get(0)) + .expect("verification query should succeed"); + assert_eq!(count, 0, "canceled work must not mutate after reset"); + + drop(verifier); + drop(blocker); + fs::remove_dir_all(root).expect("temporary SQLite root should be removed"); +} + #[test] fn sqlite_pending_operation_slots_are_reclaimed_after_completion() { let root = temporary_root("pending-reclaim"); @@ -657,17 +710,31 @@ fn sqlite_transaction_deadline_interrupts_and_rolls_back() { policy_for(&root), r#" let db = sqlite::open({ path: "state.db", mode: "read_write_create", limits: { max_transaction_ms: 1 } }); - sqlite::transaction(&db, [{ - sql: "WITH RECURSIVE numbers(value) AS (SELECT 1 UNION ALL SELECT value + 1 FROM numbers LIMIT 10000000) SELECT sum(value) FROM numbers", - query: true, - limits: { max_rows: 1 } - }]); + sqlite::execute(&db, "CREATE TABLE items (value INTEGER)", []); + sqlite::transaction(&db, { + { sql: "INSERT INTO items (value) VALUES (1)" }, + { + sql: "WITH RECURSIVE numbers(value) AS (SELECT 1 UNION ALL SELECT value + 1 FROM numbers LIMIT 10000000) SELECT sum(value) FROM numbers", + query: true, + limits: { max_rows: 1 } + } + }); "#, ); assert!( error.contains("transaction exceeded") && error.contains("1 ms deadline"), "transaction deadline must surface explicitly, got: {error}" ); + run_sqlite_source( + policy_for(&root), + r#" + let db = sqlite::open({ path: "state.db", mode: "read_write_create", limits: {} }); + let result = sqlite::query(&db, "SELECT count(*) FROM items", [], {}); + assert(result.rows[0].cells[0].int_value == 0); + sqlite::close(db); + "#, + ) + .expect("the timed-out transaction write should be rolled back"); fs::remove_dir_all(root).expect("temporary SQLite root should be removed"); } From daf6a67f63cecf09fbcb035325537b45a8b4a42b Mon Sep 17 00:00:00 2001 From: fffonion Date: Sat, 19 Sep 2026 16:31:06 +0800 Subject: [PATCH 09/23] fix(sqlite): harden deadlines and close retries --- src/builtins/runtime/sqlite.rs | 295 +++++++++++++++++- .../builtins/sqlite_scope_lifecycle_tests.rs | 4 +- 2 files changed, 281 insertions(+), 18 deletions(-) diff --git a/src/builtins/runtime/sqlite.rs b/src/builtins/runtime/sqlite.rs index 8fb94e1b..b370b81a 100644 --- a/src/builtins/runtime/sqlite.rs +++ b/src/builtins/runtime/sqlite.rs @@ -132,25 +132,95 @@ impl Drop for SqliteOperationLease { } } -type SqliteCloseFuture = Pin> + Send + 'static>>; +enum SqliteCloseAttempt { + Closed, + Retryable { + connection: tokio_rusqlite::Connection, + message: String, + }, + Failed(String), +} + +type SqliteCloseFuture = Pin + Send + 'static>>; enum SqliteCloseState { Open, Closing(SqliteCloseFuture), + // `tokio-rusqlite` returns a live handle with `Error::Close`; retain it so + // explicit close can fail without losing the resource and scope teardown + // can retry without releasing the connection permit. + Retryable { + connection: tokio_rusqlite::Connection, + message: String, + }, Finished(Result<(), String>), } struct SqliteCloseLifecycle { state: Mutex, + #[cfg(test)] + injected_failures: AtomicUsize, + #[cfg(test)] + failures_seen: Arc, } impl SqliteCloseLifecycle { fn new() -> Self { Self { state: Mutex::new(SqliteCloseState::Open), + #[cfg(test)] + injected_failures: AtomicUsize::new(0), + #[cfg(test)] + failures_seen: Arc::new(AtomicUsize::new(0)), } } + #[cfg(test)] + fn new_with_failures(failures: usize) -> Self { + Self { + state: Mutex::new(SqliteCloseState::Open), + injected_failures: AtomicUsize::new(failures), + failures_seen: Arc::new(AtomicUsize::new(0)), + } + } + + #[cfg(test)] + fn failures_seen(&self) -> usize { + self.failures_seen.load(Ordering::Acquire) + } + + fn close_future(&self, connection: tokio_rusqlite::Connection) -> SqliteCloseFuture { + #[cfg(test)] + let inject_failure = self + .injected_failures + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| { + remaining.checked_sub(1) + }) + .is_ok(); + #[cfg(test)] + let failures_seen = Arc::clone(&self.failures_seen); + Box::pin(async move { + #[cfg(test)] + if inject_failure { + failures_seen.fetch_add(1, Ordering::AcqRel); + return SqliteCloseAttempt::Retryable { + connection, + message: "injected retryable SQLite close failure".to_string(), + }; + } + match connection.close().await { + Ok(()) | Err(tokio_rusqlite::Error::ConnectionClosed) => SqliteCloseAttempt::Closed, + Err(tokio_rusqlite::Error::Close((connection, error))) => { + SqliteCloseAttempt::Retryable { + connection, + message: sqlite_error_message(error), + } + } + Err(error) => SqliteCloseAttempt::Failed(adapter_close_error_message(error)), + } + }) + } + fn begin( &self, connection: tokio_rusqlite::Connection, @@ -165,15 +235,11 @@ impl SqliteCloseLifecycle { SqliteCloseState::Open => { closed.store(true, Ordering::Release); interrupt.interrupt(); - *state = SqliteCloseState::Closing(Box::pin(async move { - connection - .close() - .await - .map_err(adapter_close_error_message) - })); + *state = SqliteCloseState::Closing(self.close_future(connection)); Ok(false) } SqliteCloseState::Closing(_) => Ok(false), + SqliteCloseState::Retryable { message, .. } => Err(message.clone()), SqliteCloseState::Finished(result) => result.clone().map(|()| true), } } @@ -187,21 +253,59 @@ impl SqliteCloseLifecycle { SqliteCloseState::Open => Poll::Pending, SqliteCloseState::Closing(future) => match future.as_mut().poll(cx) { Poll::Pending => Poll::Pending, - Poll::Ready(result) => { - *state = SqliteCloseState::Finished(result.clone()); + Poll::Ready(SqliteCloseAttempt::Closed) => { + *state = SqliteCloseState::Finished(Ok(())); + Poll::Ready(Ok(())) + } + Poll::Ready(SqliteCloseAttempt::Retryable { + connection, + message, + }) => { + let result = Err(message.clone()); + *state = SqliteCloseState::Retryable { + connection, + message, + }; + Poll::Ready(result) + } + Poll::Ready(SqliteCloseAttempt::Failed(message)) => { + let result = Err(message.clone()); + *state = SqliteCloseState::Finished(Err(message)); Poll::Ready(result) } }, + SqliteCloseState::Retryable { message, .. } => Poll::Ready(Err(message.clone())), SqliteCloseState::Finished(result) => Poll::Ready(result.clone()), } } + fn retry_after_failure(&self) -> bool { + let mut state = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let previous = std::mem::replace(&mut *state, SqliteCloseState::Open); + match previous { + SqliteCloseState::Retryable { connection, .. } => { + *state = SqliteCloseState::Closing(self.close_future(connection)); + true + } + previous => { + *state = previous; + false + } + } + } + fn reopen_after_failure(&self, closed: &AtomicBool) { let mut state = self .state .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); - if matches!(&*state, SqliteCloseState::Finished(Err(_))) { + if matches!( + &*state, + SqliteCloseState::Retryable { .. } | SqliteCloseState::Finished(Err(_)) + ) { *state = SqliteCloseState::Open; closed.store(false, Ordering::Release); } @@ -245,6 +349,12 @@ impl HostResource for SqliteResource { fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll> { match self.close_lifecycle.poll(cx) { Poll::Pending => Poll::Pending, + // A retryable adapter failure cannot become `Ready`: the resource + // table reclaims every ready resource, including cleanup errors. + Poll::Ready(Err(_)) if self.close_lifecycle.retry_after_failure() => { + cx.waker().wake_by_ref(); + Poll::Pending + } Poll::Ready(result) => Poll::Ready(result.map_err(sqlite_close_resource_error)), } } @@ -1320,6 +1430,8 @@ struct TransactionStatement { params: Vec, query: bool, limits: SqliteLimits, + #[cfg(test)] + after_execute: Option>, } fn parse_transaction_statements( @@ -1366,12 +1478,27 @@ fn parse_transaction_statements( params, query, limits: statement_limits, + #[cfg(test)] + after_execute: None, }) }) .collect() } fn transaction_with_connection( + connection: &mut Connection, + statements: Vec, + max_transaction_ms: u64, +) -> VmResult> { + let deadline = Instant::now() + .checked_add(Duration::from_millis(max_transaction_ms)) + .ok_or_else(|| { + VmError::HostError("SQLite transaction deadline is out of range".to_string()) + })?; + transaction_with_connection_until(connection, statements, deadline, max_transaction_ms) +} + +fn transaction_with_connection_until( connection: &mut Connection, statements: Vec, deadline: Instant, @@ -1410,6 +1537,10 @@ fn transaction_with_connection( transaction_result_value("execute", Some(value), None) }; results.push(result); + #[cfg(test)] + if let Some(after_execute) = statement.after_execute.as_ref() { + after_execute(); + } } if Instant::now() >= deadline { return Err(VmError::HostError(format!( @@ -1444,11 +1575,6 @@ pub(super) async fn builtin_sqlite_transaction_impl( context.allow_unsafe_sql, )?; let max_transaction_ms = context.limits.max_transaction_ms; - let deadline = Instant::now() - .checked_add(Duration::from_millis(max_transaction_ms)) - .ok_or_else(|| { - VmError::HostError("SQLite transaction deadline is out of range".to_string()) - })?; let closed = Arc::clone(&context.closed); let value = context .connection @@ -1459,7 +1585,7 @@ pub(super) async fn builtin_sqlite_transaction_impl( "SQLite database is already closed".to_string(), )); } - transaction_with_connection(connection, statements, deadline, max_transaction_ms) + transaction_with_connection(connection, statements, max_transaction_ms) }) .await .map_err(adapter_call_error)?; @@ -1609,10 +1735,147 @@ impl SqliteHostExt for Vm { #[cfg(test)] mod tests { use std::future::Future; + use std::sync::mpsc; use std::task::{Context, Poll, Waker}; use std::time::{SystemTime, UNIX_EPOCH}; use super::*; + use crate::vm::resource::ResourceTable; + + #[test] + fn transaction_deadline_rolls_back_an_observed_write() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock should be after the Unix epoch") + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "rustscript-sqlite-observed-rollback-{}-{nonce}", + std::process::id() + )); + fs::create_dir_all(&root).expect("temporary SQLite root should be created"); + let database_path = root.join("state.db"); + Connection::open(&database_path) + .expect("setup connection should open") + .execute_batch("CREATE TABLE items (value INTEGER)") + .expect("setup table should be created"); + + let (write_observed_tx, write_observed_rx) = mpsc::sync_channel(0); + let worker_path = database_path.clone(); + let transaction = std::thread::spawn(move || { + let mut connection = + Connection::open(worker_path).expect("transaction connection should open"); + let limits = SqliteLimits::default(); + let statements = vec![ + TransactionStatement { + sql: "INSERT INTO items (value) VALUES (1)".to_string(), + params: Vec::new(), + query: false, + limits, + after_execute: Some(Box::new(move || { + write_observed_tx + .send(()) + .expect("write observation receiver should remain open"); + })), + }, + TransactionStatement { + sql: "WITH RECURSIVE numbers(value) AS (SELECT 1 UNION ALL SELECT value + 1 FROM numbers LIMIT 10000000) SELECT sum(value) FROM numbers".to_string(), + params: Vec::new(), + query: true, + limits, + after_execute: None, + }, + ]; + transaction_with_connection_until( + &mut connection, + statements, + Instant::now() + Duration::from_millis(500), + 500, + ) + }); + + write_observed_rx + .recv_timeout(Duration::from_secs(2)) + .expect("the INSERT must execute before the expensive statement starts"); + let error = transaction + .join() + .expect("transaction worker should not panic") + .expect_err("the expensive statement should cross the deadline"); + assert!( + error.to_string().contains("500 ms deadline"), + "transaction deadline must surface explicitly, got: {error}" + ); + + let verifier = Connection::open(&database_path) + .expect("verification connection should reopen the database"); + let count: i64 = verifier + .query_row("SELECT count(*) FROM items", [], |row| row.get(0)) + .expect("verification query should succeed"); + assert_eq!(count, 0, "the observed write must be rolled back"); + drop(verifier); + fs::remove_dir_all(root).expect("temporary SQLite root should be removed"); + } + + #[tokio::test] + async fn retryable_close_failure_retains_resource_and_connection_permit() { + let limits = SqliteLimits::default(); + let options = OpenOptions { + path: ":memory:".to_string(), + mode: OpenMode::Memory, + root: None, + limits, + allow_unsafe_sql: false, + }; + let (connection, interrupt) = open_connection(&options) + .await + .expect("adapter connection should open"); + let open_connections = Arc::new(AtomicUsize::new(1)); + let close_lifecycle = Arc::new(SqliteCloseLifecycle::new_with_failures(1)); + let resource = SqliteResource { + connection, + interrupt, + limits, + allow_unsafe_sql: false, + closed: Arc::new(AtomicBool::new(false)), + in_flight: Arc::new(AtomicUsize::new(0)), + close_lifecycle: Arc::clone(&close_lifecycle), + _connection_permit: ConnectionCountPermit { + open_connections: Arc::clone(&open_connections), + }, + }; + let mut table = ResourceTable::new().expect("resource table should initialize"); + let token = table.push(resource).expect("SQLite resource should insert"); + assert_eq!( + table + .begin_close(token, ResourceCloseReason::VmReset) + .expect("close should begin"), + CloseProgress::Pending + ); + + let mut cx = Context::from_waker(Waker::noop()); + assert!(matches!(table.poll_close(token, &mut cx), Poll::Pending)); + assert_eq!(close_lifecycle.failures_seen(), 1); + assert_eq!(table.len(), 1, "retryable close must retain the resource"); + assert_eq!( + open_connections.load(Ordering::Acquire), + 1, + "retryable close must retain the connection permit" + ); + + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let mut cx = Context::from_waker(Waker::noop()); + match table.poll_close(token, &mut cx) { + Poll::Ready(result) => break result, + Poll::Pending => tokio::task::yield_now().await, + } + } + }) + .await + .expect("retry should eventually confirm adapter closure") + .expect("retry should close the resource"); + assert!(table.is_empty()); + assert_eq!(open_connections.load(Ordering::Acquire), 0); + } #[tokio::test] async fn canceled_host_future_holds_operation_slot_until_adapter_closure_finishes() { diff --git a/tests/builtins/sqlite_scope_lifecycle_tests.rs b/tests/builtins/sqlite_scope_lifecycle_tests.rs index a2de7774..5146d931 100644 --- a/tests/builtins/sqlite_scope_lifecycle_tests.rs +++ b/tests/builtins/sqlite_scope_lifecycle_tests.rs @@ -709,7 +709,7 @@ fn sqlite_transaction_deadline_interrupts_and_rolls_back() { let error = run_sqlite_host_error( policy_for(&root), r#" - let db = sqlite::open({ path: "state.db", mode: "read_write_create", limits: { max_transaction_ms: 1 } }); + let db = sqlite::open({ path: "state.db", mode: "read_write_create", limits: { max_transaction_ms: 500 } }); sqlite::execute(&db, "CREATE TABLE items (value INTEGER)", []); sqlite::transaction(&db, { { sql: "INSERT INTO items (value) VALUES (1)" }, @@ -722,7 +722,7 @@ fn sqlite_transaction_deadline_interrupts_and_rolls_back() { "#, ); assert!( - error.contains("transaction exceeded") && error.contains("1 ms deadline"), + error.contains("transaction exceeded") && error.contains("500 ms deadline"), "transaction deadline must surface explicitly, got: {error}" ); run_sqlite_source( From 29c474ad4d12d5894d977f3c5ad72cbd53cbbe45 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sat, 19 Sep 2026 17:24:15 +0800 Subject: [PATCH 10/23] fix(sqlite): bound persistent close retries --- src/builtins/runtime/sqlite.rs | 268 ++++++++++++++++++-------- tests/sqlite_async_host_arch_tests.rs | 5 +- 2 files changed, 190 insertions(+), 83 deletions(-) diff --git a/src/builtins/runtime/sqlite.rs b/src/builtins/runtime/sqlite.rs index b370b81a..02606032 100644 --- a/src/builtins/runtime/sqlite.rs +++ b/src/builtins/runtime/sqlite.rs @@ -42,6 +42,9 @@ use crate::vm::{ /// SQLite `progress_handler` step cadence used to enforce transaction deadlines. const SQLITE_PROGRESS_STEPS: i32 = 1_000; +/// Maximum adapter close attempts, including the initial request. +const SQLITE_CLOSE_MAX_ATTEMPTS: usize = 3; + /// Bounded SQLite connection/query limits, mirroring the published surface. #[derive(Clone, Copy, Debug)] pub struct SqliteLimits { @@ -145,15 +148,19 @@ type SqliteCloseFuture = Pin + Send enum SqliteCloseState { Open, - Closing(SqliteCloseFuture), - // `tokio-rusqlite` returns a live handle with `Error::Close`; retain it so - // explicit close can fail without losing the resource and scope teardown - // can retry without releasing the connection permit. - Retryable { - connection: tokio_rusqlite::Connection, + Closing { + future: SqliteCloseFuture, + attempts: usize, + }, + // A terminal error cannot be reported as resource-close completion: + // `ResourceTable` reclaims every Ready resource, including cleanup errors. + // Retain a returned adapter handle when available and park the resource so + // its connection permit and the VM reuse guard remain held. + Terminal { + _connection: Option, message: String, }, - Finished(Result<(), String>), + Closed, } struct SqliteCloseLifecycle { @@ -235,12 +242,15 @@ impl SqliteCloseLifecycle { SqliteCloseState::Open => { closed.store(true, Ordering::Release); interrupt.interrupt(); - *state = SqliteCloseState::Closing(self.close_future(connection)); + *state = SqliteCloseState::Closing { + future: self.close_future(connection), + attempts: 1, + }; Ok(false) } - SqliteCloseState::Closing(_) => Ok(false), - SqliteCloseState::Retryable { message, .. } => Err(message.clone()), - SqliteCloseState::Finished(result) => result.clone().map(|()| true), + SqliteCloseState::Closing { .. } => Ok(false), + SqliteCloseState::Terminal { message, .. } => Err(message.clone()), + SqliteCloseState::Closed => Ok(true), } } @@ -249,66 +259,64 @@ impl SqliteCloseLifecycle { .state .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); - match &mut *state { - SqliteCloseState::Open => Poll::Pending, - SqliteCloseState::Closing(future) => match future.as_mut().poll(cx) { - Poll::Pending => Poll::Pending, - Poll::Ready(SqliteCloseAttempt::Closed) => { - *state = SqliteCloseState::Finished(Ok(())); - Poll::Ready(Ok(())) - } - Poll::Ready(SqliteCloseAttempt::Retryable { - connection, - message, - }) => { - let result = Err(message.clone()); - *state = SqliteCloseState::Retryable { - connection, - message, - }; - Poll::Ready(result) + loop { + match &mut *state { + SqliteCloseState::Open => return Poll::Pending, + SqliteCloseState::Closing { future, attempts } => { + match future.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(SqliteCloseAttempt::Closed) => { + *state = SqliteCloseState::Closed; + return Poll::Ready(Ok(())); + } + Poll::Ready(SqliteCloseAttempt::Retryable { + connection, + message: _, + }) if *attempts < SQLITE_CLOSE_MAX_ATTEMPTS => { + let attempts = *attempts + 1; + *state = SqliteCloseState::Closing { + future: self.close_future(connection), + attempts, + }; + // Poll the replacement now so it either registers + // the caller's waker or consumes another bounded, + // immediately-ready retry. Never self-wake here. + } + Poll::Ready(SqliteCloseAttempt::Retryable { + connection, + message, + }) => { + let result = Err(message.clone()); + *state = SqliteCloseState::Terminal { + _connection: Some(connection), + message, + }; + return Poll::Ready(result); + } + Poll::Ready(SqliteCloseAttempt::Failed(message)) => { + let result = Err(message.clone()); + *state = SqliteCloseState::Terminal { + _connection: None, + message, + }; + return Poll::Ready(result); + } + } } - Poll::Ready(SqliteCloseAttempt::Failed(message)) => { - let result = Err(message.clone()); - *state = SqliteCloseState::Finished(Err(message)); - Poll::Ready(result) + SqliteCloseState::Terminal { message, .. } => { + return Poll::Ready(Err(message.clone())); } - }, - SqliteCloseState::Retryable { message, .. } => Poll::Ready(Err(message.clone())), - SqliteCloseState::Finished(result) => Poll::Ready(result.clone()), - } - } - - fn retry_after_failure(&self) -> bool { - let mut state = self - .state - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - let previous = std::mem::replace(&mut *state, SqliteCloseState::Open); - match previous { - SqliteCloseState::Retryable { connection, .. } => { - *state = SqliteCloseState::Closing(self.close_future(connection)); - true - } - previous => { - *state = previous; - false + SqliteCloseState::Closed => return Poll::Ready(Ok(())), } } } - fn reopen_after_failure(&self, closed: &AtomicBool) { - let mut state = self + fn has_terminal_failure(&self) -> bool { + let state = self .state .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); - if matches!( - &*state, - SqliteCloseState::Retryable { .. } | SqliteCloseState::Finished(Err(_)) - ) { - *state = SqliteCloseState::Open; - closed.store(false, Ordering::Release); - } + matches!(&*state, SqliteCloseState::Terminal { .. }) } } @@ -342,6 +350,7 @@ impl HostResource for SqliteResource { ) { Ok(true) => Ok(CloseProgress::Ready), Ok(false) => Ok(CloseProgress::Pending), + Err(_) if self.close_lifecycle.has_terminal_failure() => Ok(CloseProgress::Pending), Err(message) => Err(sqlite_close_resource_error(message)), } } @@ -349,12 +358,7 @@ impl HostResource for SqliteResource { fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll> { match self.close_lifecycle.poll(cx) { Poll::Pending => Poll::Pending, - // A retryable adapter failure cannot become `Ready`: the resource - // table reclaims every ready resource, including cleanup errors. - Poll::Ready(Err(_)) if self.close_lifecycle.retry_after_failure() => { - cx.waker().wake_by_ref(); - Poll::Pending - } + Poll::Ready(Err(_)) if self.close_lifecycle.has_terminal_failure() => Poll::Pending, Poll::Ready(result) => Poll::Ready(result.map_err(sqlite_close_resource_error)), } } @@ -1618,15 +1622,9 @@ pub(super) async fn builtin_sqlite_close_impl( context.interrupt.as_ref(), context.closed.as_ref(), ) { - context - .close_lifecycle - .reopen_after_failure(context.closed.as_ref()); return Err(VmError::HostError(message)); } if let Err(message) = std::future::poll_fn(|cx| context.close_lifecycle.poll(cx)).await { - context - .close_lifecycle - .reopen_after_failure(context.closed.as_ref()); return Err(VmError::HostError(message)); } let handle = context.handle; @@ -1736,12 +1734,24 @@ impl SqliteHostExt for Vm { mod tests { use std::future::Future; use std::sync::mpsc; - use std::task::{Context, Poll, Waker}; + use std::task::{Context, Poll, Wake, Waker}; use std::time::{SystemTime, UNIX_EPOCH}; use super::*; use crate::vm::resource::ResourceTable; + struct CountingWake(Arc); + + impl Wake for CountingWake { + fn wake(self: Arc) { + self.0.fetch_add(1, Ordering::SeqCst); + } + + fn wake_by_ref(self: &Arc) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + #[test] fn transaction_deadline_rolls_back_an_observed_write() { let nonce = SystemTime::now() @@ -1852,14 +1862,24 @@ mod tests { ); let mut cx = Context::from_waker(Waker::noop()); - assert!(matches!(table.poll_close(token, &mut cx), Poll::Pending)); + let first_poll = table.poll_close(token, &mut cx); assert_eq!(close_lifecycle.failures_seen(), 1); - assert_eq!(table.len(), 1, "retryable close must retain the resource"); - assert_eq!( - open_connections.load(Ordering::Acquire), - 1, - "retryable close must retain the connection permit" - ); + match first_poll { + Poll::Pending => { + assert_eq!(table.len(), 1, "pending close must retain the resource"); + assert_eq!( + open_connections.load(Ordering::Acquire), + 1, + "pending close must retain the connection permit" + ); + } + Poll::Ready(Ok(())) => { + assert!(table.is_empty()); + assert_eq!(open_connections.load(Ordering::Acquire), 0); + return; + } + Poll::Ready(Err(error)) => panic!("transient retry must close cleanly: {error}"), + }; tokio::time::timeout(Duration::from_secs(5), async { loop { @@ -1877,6 +1897,90 @@ mod tests { assert_eq!(open_connections.load(Ordering::Acquire), 0); } + #[tokio::test] + async fn persistent_close_failure_parks_without_self_waking_or_releasing_the_permit() { + let limits = SqliteLimits::default(); + let options = OpenOptions { + path: ":memory:".to_string(), + mode: OpenMode::Memory, + root: None, + limits, + allow_unsafe_sql: false, + }; + let (connection, interrupt) = open_connection(&options) + .await + .expect("adapter connection should open"); + let open_connections = Arc::new(AtomicUsize::new(1)); + let close_lifecycle = Arc::new(SqliteCloseLifecycle::new_with_failures(usize::MAX)); + let resource = SqliteResource { + connection, + interrupt, + limits, + allow_unsafe_sql: false, + closed: Arc::new(AtomicBool::new(false)), + in_flight: Arc::new(AtomicUsize::new(0)), + close_lifecycle: Arc::clone(&close_lifecycle), + _connection_permit: ConnectionCountPermit { + open_connections: Arc::clone(&open_connections), + }, + }; + let program = crate::compile_source("null;") + .expect("test program should compile") + .program; + let mut vm = Vm::new(program); + vm.execution_scope() + .push_resource(resource) + .expect("SQLite resource should insert"); + vm.reset_for_reuse().expect("reset should start"); + assert!( + vm.scope_reset_pending(), + "failed cleanup must keep reset pending" + ); + assert!(!vm.is_reusable(), "pending cleanup must block VM reuse"); + + let wake_count = Arc::new(AtomicUsize::new(0)); + let waker = Waker::from(Arc::new(CountingWake(Arc::clone(&wake_count)))); + let mut cx = Context::from_waker(&waker); + assert!(matches!(vm.poll_reset_for_reuse(&mut cx), Poll::Pending)); + assert_eq!( + close_lifecycle.failures_seen(), + 3, + "persistent failure must stop after the finite close-attempt budget" + ); + assert_eq!( + wake_count.load(Ordering::SeqCst), + 0, + "terminal cleanup failure must not self-wake" + ); + + for _ in 0..16 { + assert!(matches!(vm.poll_reset_for_reuse(&mut cx), Poll::Pending)); + } + assert_eq!( + close_lifecycle.failures_seen(), + 3, + "polling a parked failure must not start another close" + ); + assert_eq!(wake_count.load(Ordering::SeqCst), 0); + assert_eq!( + vm.host_context().resource_count(), + 1, + "failed close must retain the resource" + ); + assert_eq!( + open_connections.load(Ordering::Acquire), + 1, + "failed close must retain the connection permit" + ); + assert!(vm.scope_reset_pending()); + assert!(!vm.is_reusable()); + assert_eq!( + close_lifecycle.failures_seen(), + 3, + "repeated reset polls must not double-close a parked connection" + ); + } + #[tokio::test] async fn canceled_host_future_holds_operation_slot_until_adapter_closure_finishes() { let nonce = SystemTime::now() diff --git a/tests/sqlite_async_host_arch_tests.rs b/tests/sqlite_async_host_arch_tests.rs index 3f60c39b..555f5752 100644 --- a/tests/sqlite_async_host_arch_tests.rs +++ b/tests/sqlite_async_host_arch_tests.rs @@ -34,6 +34,9 @@ fn sqlite_host_functions_are_macro_owned_async_functions() { #[test] fn sqlite_host_owns_no_threads_or_custom_operation_driver() { + let implementation = SQLITE_SOURCE + .split_once("\n#[cfg(test)]\nmod tests") + .map_or(SQLITE_SOURCE, |(implementation, _tests)| implementation); for forbidden in [ "std::thread", "thread::", @@ -56,7 +59,7 @@ fn sqlite_host_owns_no_threads_or_custom_operation_driver() { "quiescence_waker", ] { assert!( - !SQLITE_SOURCE.contains(forbidden), + !implementation.contains(forbidden), "SQLite host source must not contain custom scheduling token `{forbidden}`" ); } From d913b7c75caf59d06b6f7e4ea335d89cb7960302 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sat, 19 Sep 2026 18:06:17 +0800 Subject: [PATCH 11/23] fix(integration): align async host feature gates and fingerprints --- src/builtins/runtime/mod.rs | 9 +++++++-- src/host_api.rs | 5 ++++- tests/standard_host_descriptor_arch_tests.rs | 14 +++++++------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/builtins/runtime/mod.rs b/src/builtins/runtime/mod.rs index f26d60c5..b5d8b671 100644 --- a/src/builtins/runtime/mod.rs +++ b/src/builtins/runtime/mod.rs @@ -5,10 +5,15 @@ use std::sync::{Arc, OnceLock}; use crate::builtins::BuiltinFunction; use crate::host_api::{HostApiCatalog, HostApiFingerprint}; +#[cfg(all(feature = "async", not(target_family = "wasm")))] +use crate::vm::CaptureAsyncHostContext; +#[cfg(all( + any(feature = "http-client", feature = "sqlite"), + not(target_family = "wasm") +))] +use crate::vm::HostFutureOutput; #[allow(unused_imports)] use crate::vm::{CallOutcome, CallReturn, HostOpId, Value, Vm, VmError, VmResult}; -#[cfg(all(feature = "async", not(target_family = "wasm")))] -use crate::vm::{CaptureAsyncHostContext, HostFutureOutput}; mod aot; mod bytes; diff --git a/src/host_api.rs b/src/host_api.rs index a013a8e8..7e7c027a 100644 --- a/src/host_api.rs +++ b/src/host_api.rs @@ -5858,7 +5858,10 @@ mod tests { &bytes[..FINGERPRINT_DOMAIN_MAGIC.len()], FINGERPRINT_DOMAIN_MAGIC ); - assert_eq!(bytes[FINGERPRINT_DOMAIN_MAGIC.len()], 2); + assert_eq!( + bytes[FINGERPRINT_DOMAIN_MAGIC.len()], + FINGERPRINT_FORMAT_VERSION + ); assert_eq!(catalog.canonical_bytes(), bytes); } diff --git a/tests/standard_host_descriptor_arch_tests.rs b/tests/standard_host_descriptor_arch_tests.rs index 979ced13..3c23d575 100644 --- a/tests/standard_host_descriptor_arch_tests.rs +++ b/tests/standard_host_descriptor_arch_tests.rs @@ -30,16 +30,16 @@ const HTTP_SURFACE_ENABLED: bool = cfg!(all(feature = "http-client", not(target_ /// omits transport-only resources; the default build (no `http-client`) /// composes one module fewer and must reproduce /// [`STANDARD_CATALOG_FINGERPRINT_NO_HTTP`]. -const STANDARD_CATALOG_FINGERPRINT: &str = "4e3b7572a59b3dae"; +const STANDARD_CATALOG_FINGERPRINT: &str = "8aad996e0b2f010b"; /// Fingerprint of the published standard host catalog **without** the HTTP /// surface: the `--workspace` default build and every wasm build. -const STANDARD_CATALOG_FINGERPRINT_NO_HTTP: &str = "a6b4b2dcadc5df14"; -const IO_CATALOG_FINGERPRINT: &str = "234a7fdc3aaa3f95"; -const SQLITE_CATALOG_FINGERPRINT: &str = "b6d4c278145edacf"; -const JIT_CATALOG_FINGERPRINT: &str = "d0a3efbca2d0923c"; -const TIMER_CATALOG_FINGERPRINT: &str = "4af2dfa2aee1f42e"; +const STANDARD_CATALOG_FINGERPRINT_NO_HTTP: &str = "8afd8a69c58f02bd"; +const IO_CATALOG_FINGERPRINT: &str = "a16730bd11bf5e10"; +const SQLITE_CATALOG_FINGERPRINT: &str = "dce61460a46c421a"; +const JIT_CATALOG_FINGERPRINT: &str = "ae81318e8a018681"; +const TIMER_CATALOG_FINGERPRINT: &str = "7ecc0517cea3570b"; #[cfg(all(feature = "http-client", not(target_family = "wasm")))] -const HTTP_CATALOG_FINGERPRINT: &str = "66db92730480d50a"; +const HTTP_CATALOG_FINGERPRINT: &str = "329e09ffbdd82a6f"; /// The standard catalog fingerprint this build must reproduce exactly. fn standard_catalog_fingerprint() -> &'static str { From 3671a8ace9bb562f9a9a0e19936af8092e694f4a Mon Sep 17 00:00:00 2001 From: fffonion Date: Sat, 19 Sep 2026 19:34:55 +0800 Subject: [PATCH 12/23] Remove legacy core compatibility paths --- docs/callable-runtime.md | 8 +- docs/host-sdk-descriptors.md | 7 +- docs/http-client.md | 8 +- pd-host-function/src/lib.rs | 81 +-------- pd-vm-nostd/README.md | 2 +- pd-vm-nostd/src/vmbc.rs | 35 ++-- pd-vm-nostd/tests/call_script_tests.rs | 14 -- pd-vm-nostd/tests/embedded_vmbc.rs | 79 +++------ pd-vm-wasm/src/runtime.rs | 15 +- plans/tail_call_optimization_plan.md | 4 +- src/builtins/runtime/timer.rs | 55 +++++- src/bytecode.rs | 7 +- src/vm/host.rs | 59 +------ src/vm/host_extension.rs | 21 +-- src/vm/tests.rs | 164 +++++++++--------- src/vmbc.rs | 61 ++----- .../builtins/sqlite_scope_lifecycle_tests.rs | 9 +- tests/host_descriptor_effect_tests.rs | 51 ------ tests/http_async_arch_tests.rs | 1 - tests/http_named_struct_contract_tests.rs | 16 +- tests/io_descriptor_install_tests.rs | 24 ++- tests/sqlite_async_host_arch_tests.rs | 5 +- tests/sqlite_named_struct_tests.rs | 16 +- tests/support/async_test_bridge.rs | 9 +- tests/timer_host_tests.rs | 37 +++- tests/vm/http_host_tests.rs | 9 +- tests/vm/http_sse_tests.rs | 11 +- tests/vm/io_http_coexistence_tests.rs | 16 +- tests/vm/vm_async_runtime_tests.rs | 49 +++++- tests/wire/wire_tests.rs | 137 ++------------- 30 files changed, 402 insertions(+), 608 deletions(-) diff --git a/docs/callable-runtime.md b/docs/callable-runtime.md index 3e3d2d8a..fa0d88fa 100644 --- a/docs/callable-runtime.md +++ b/docs/callable-runtime.md @@ -1,6 +1,6 @@ # Script call frames and callable values -RustScript bytecode format version 14 (VMBC v14) carries runtime script call frames, first-class callable values, the static builtin ID catalog, the direct script-call opcode, and an explicit guest named-struct declaration section. Version 11 introduced frames, callable values, and the static catalog; version 12 added `callscript` for statically resolved named calls; version 13 framed guest struct declarations so a 4-byte zero trailer could not be mistaken for an empty table. Version 14 marks catalog fingerprint format v3 after transport-only HTTP resource declarations were removed. +RustScript bytecode format version 14 (VMBC v14) carries runtime script call frames, first-class callable values, the static builtin ID catalog, the direct script-call opcode, full host schemas, callable metadata, and an explicit guest named-struct declaration section. Catalog identities use fingerprint format v3. ## Bytecode contract @@ -18,7 +18,7 @@ The three call opcodes differ in who owns the callee and what the frame must pro - `callvalue` — the callee is a `Value::Callable` owned by the caller operand stack at the call site, and remains the caller's responsibility after the call. This path carries environments, closures, and any callable whose identity or capture state is runtime-valued. - `callscript` — the callee is owned by program callable metadata (the prototype table). The frame contributes only `argc` arguments and no callable value, but unlike `call` the callee is a script function rather than a builtin, so the call enters a new script frame with its own local base. -VMBC v14 is the current format. It decodes the legacy v11 stream without host-schema metadata and the v12 stream without a named-struct section. VMBC v13 is rejected because its catalog fingerprints use the prior catalog surface; source recompilation is required. VMBC v14 carries full host schemas, callable metadata, an explicit guest named-struct table, and catalog fingerprint format v3 identities. Unknown versions and malformed resource schemas are rejected deterministically. PDRC v6 recordings and AOT artifacts (format 8, ABI 8) embed VMBC and therefore reject a nested v13 program; native cache identity includes bytecode ABI 14. +VMBC v14 is the current format. Other versions and malformed resource schemas are rejected through the ordinary wire-format validation path. PDRC v6 recordings and AOT artifacts (format 8, ABI 8) embed VMBC; native cache identity includes bytecode ABI 14. ## Static builtin IDs @@ -27,7 +27,7 @@ Every VM-visible builtin (ordinary, internal, and special-call) has one explicit - **Immutable explicit IDs.** IDs never change once assigned. Adding or reordering catalog entries never renumbers existing entries; new builtins take the next free ID in their documented block (extension `0x0000..=0xFF8F` for future builtins and host imports, special-call `0xFF90..=0xFFA1`, ordinary `0xFFA2..=0xFFFF`). The reserved sentinel gap `0xFF90..=0xFF92` stays unassigned. - **Build-time validation.** The build fails on duplicate IDs, duplicate source names, duplicate Rust variants, out-of-block IDs, class/gate inconsistencies, a discovered runtime callable without an explicit ID, or a catalog entry without a runtime callable. - **Shared std/no-std IDs.** `pd-vm-nostd` dispatches on the same static indices through the checked-in generated mirror `pd-vm-nostd/src/generated_builtin_ids.rs`; the workspace test `static_builtin_ids_are_frozen` fails when the mirror drifts from the catalog. -- **Format breaks are permanent.** The static ID migration bumped VMBC to v11 (and the internal bytecode ABI to 11); the `callscript` opcode break bumped both to v12; the guest named-struct section bumped both to v13; catalog fingerprint format v3 bumped both to v14. Versions are never rewritten in place: v11/v12 remain readable only in their original framing, while v13 requires recompilation. +- **Current format identity.** VMBC and the internal bytecode ABI use version 14, with catalog fingerprint format v3. ## Runtime model @@ -100,4 +100,4 @@ Whole-program AOT and Trace JIT use the same builtin call path (static catalog I ## Embedded runtime -`pd-vm-nostd` decodes the same VMBC v14 callable metadata, rejects v13 catalog artifacts, and executes callable binding, `callvalue`, `callscript`, recursive frames, captures, and direct host targets using `core` plus `alloc`, dispatching on the identical static builtin IDs via its checked-in generated mirror. +`pd-vm-nostd` decodes the same VMBC v14 callable metadata and executes callable binding, `callvalue`, `callscript`, recursive frames, captures, and direct host targets using `core` plus `alloc`, dispatching on the identical static builtin IDs via its checked-in generated mirror. diff --git a/docs/host-sdk-descriptors.md b/docs/host-sdk-descriptors.md index 3add8411..8fa2ce72 100644 --- a/docs/host-sdk-descriptors.md +++ b/docs/host-sdk-descriptors.md @@ -22,7 +22,7 @@ One host function declaration produces, in a single macro expansion: | Part | Meaning | |---|---| | `schema` | The guest ABI: parameter names, types, passing modes, return type. This is the only fingerprint input. | -| `binding` | The dispatch class (`Static`, `StaticStack`, `StaticStackRuntimeOwned`, `StaticArgs`, `StaticNonYieldingArgs`, `Owned`). | +| `binding` | The dispatch class (`Static`, `StaticStack`, `StaticArgs`, `StaticNonYieldingArgs`, `Owned`). | | `adapter` | The concrete adapter or owned-dispatch factory installed into a registry. | | `effects` | Guest resource effects (borrow/borrow-mut/take-owned/create) and hidden host-state read/write effects. Runtime-only metadata; excluded from the fingerprint. | | `resource_types` | The concrete resource-type declarations this function contributes. | @@ -220,11 +220,6 @@ What the contract does and does not change: source. Deriving the contract's key from that declaration (as above) keeps the two from drifting. -`runtime_owned_pending` remains available for stack-shaped synchronous adapters -whose pending operation is already owned by a generic VM operation registry. -It requires a declared contract. Prefer an ordinary async declaration whenever -the implementation can await the library future directly; do not wrap such a -function in a domain-specific operation, resource, runtime, or thread. ## 4. Installing a module diff --git a/docs/http-client.md b/docs/http-client.md index 68ed314d..ca0db620 100644 --- a/docs/http-client.md +++ b/docs/http-client.md @@ -41,12 +41,8 @@ Both builtins are ordinary `#[pd_host_function] async fn` declarations. Their macro-generated wrappers own async-host submission; the HTTP module does not publish transient request, response, or stream resources. -Removing those transport-only resource declarations is an intentional catalog -artifact break. Catalog fingerprints use format v3, and bytecode/VMBC use ABI -and wire version 14. VMBC v13 artifacts are rejected and must be recompiled; -the runtime does not recreate unused resource declarations solely to retain an -obsolete digest. Function schemas continue to receive exact validation at bind -time. +Catalog fingerprints use format v3, and bytecode/VMBC use ABI and wire version +14. Function schemas receive exact validation at bind time. The HTTP and SSE behavior, cancellation, and native async bridge contracts are uniform across supported native targets. See diff --git a/pd-host-function/src/lib.rs b/pd-host-function/src/lib.rs index f5836e29..9469242e 100644 --- a/pd-host-function/src/lib.rs +++ b/pd-host-function/src/lib.rs @@ -23,7 +23,7 @@ fn expand_pd_host_function( attr: Punctuated, mut item: ItemFn, ) -> Result { - let (guest_name, contract, runtime_owned_pending) = parse_function_args(&attr)?; + let (guest_name, contract) = parse_function_args(&attr)?; let is_async = item.sig.asyncness.is_some(); let docs = doc_string(&item.attrs); let mut resource_params = Vec::<(String, ResourceSpec)>::new(); @@ -127,7 +127,6 @@ fn expand_pd_host_function( &resource_params, &state_params, contract.as_ref(), - runtime_owned_pending, )?; for input in &mut item.sig.inputs { if let FnArg::Typed(pat_type) = input { @@ -258,8 +257,7 @@ fn is_async_owned_type(ty: &Type) -> bool { } /// Parses `#[pd_host_function(name = "...")]` plus the optional -/// `contract = ` guest-schema override and the optional -/// `runtime_owned_pending` dispatch flag. +/// `contract = ` guest-schema override. /// /// `contract` names a zero-argument callable returning a /// [`HostFunctionSchema`](pd_host_schema) for functions whose guest contract @@ -267,21 +265,13 @@ fn is_async_owned_type(ty: &Type) -> bool { /// fixed-shape map returns). The contract is declared next to the function it /// describes, so the adapter, binding class, and effects still come from one /// macro expansion and there is no parallel catalog entry. -/// -/// `runtime_owned_pending` selects the stack dispatch class whose pending -/// operation is owned by the generic runtime operation/stream registries -/// instead of a registered operation driver. fn parse_function_args( args: &Punctuated, -) -> Result<(LitStr, Option, bool), Error> { +) -> Result<(LitStr, Option), Error> { let mut name: Option = None; let mut contract: Option = None; - let mut runtime_owned_pending = false; for meta in args { match meta { - Meta::Path(path) if path.is_ident("runtime_owned_pending") => { - runtime_owned_pending = true; - } Meta::NameValue(name_value) if name_value.path.is_ident("name") => { let syn::Expr::Lit(expr_lit) = &name_value.value else { return Err(Error::new_spanned( @@ -322,7 +312,7 @@ fn parse_function_args( return Err(Error::new_spanned( other, "#[pd_host_function] only supports name = \"...\", an optional \ - contract = , and the runtime_owned_pending dispatch flag", + contract = ", )); } } @@ -333,7 +323,7 @@ fn parse_function_args( "expected #[pd_host_function(name = \"...\")]", )); }; - Ok((name, contract, runtime_owned_pending)) + Ok((name, contract)) } fn doc_string(attrs: &[syn::Attribute]) -> String { @@ -1006,7 +996,6 @@ fn generate_host_function_descriptor( resource_params: &[(String, ResourceSpec)], state_params: &[(String, StateSpec)], contract: Option<&syn::Path>, - runtime_owned_pending: bool, ) -> Result { let descriptor_name = syn::Ident::new(&format!("{wrapper_name}_descriptor"), wrapper_name.span()); @@ -1026,7 +1015,6 @@ fn generate_host_function_descriptor( state_params, binding, item, - runtime_owned_pending, ); } @@ -1252,15 +1240,7 @@ fn generate_contract_host_function_descriptor( state_params: &[(String, StateSpec)], binding: GeneratedBinding, item: &ItemFn, - runtime_owned_pending: bool, ) -> Result { - if runtime_owned_pending && !matches!(binding, GeneratedBinding::Stack) { - return Err(Error::new_spanned( - &item.sig.ident, - "runtime_owned_pending requires a stack-dispatch signature (`&mut Vm` or a resource \ - parameter)", - )); - } let mut state_effect_tokens = Vec::new(); for input in &item.sig.inputs { let FnArg::Typed(pat_type) = input else { @@ -1286,15 +1266,8 @@ fn generate_contract_host_function_descriptor( }); } - let (mut binding_kind, adapter, adapter_fn) = + let (binding_kind, adapter, adapter_fn) = generated_adapter_tokens(sdk, adapter_name, wrapper_name, binding); - let mut adapter = adapter; - if runtime_owned_pending { - binding_kind = quote!(#sdk::host_extension::HostBindingKind::StaticStackRuntimeOwned); - adapter = quote!(#sdk::host_extension::HostAdapterDescriptor::StaticStackRuntimeOwned( - #adapter_name - )); - } Ok(quote! { #adapter_fn @@ -1800,48 +1773,6 @@ mod tests { assert!(expanded.contains("ResourceHandle :: from_raw")); } - #[test] - fn runtime_owned_pending_flag_selects_the_pending_owning_stack_adapter() { - let attr: Punctuated = parse_quote!( - name = "http::client::request", - contract = http_request_contract, - runtime_owned_pending - ); - let item: ItemFn = parse_quote! { - /// Suspends until the generic runtime operation registry completes it. - fn request(vm: &mut Vm, request: VmMapHandle) -> VmResult> { - todo!() - } - }; - let expanded = expand_pd_host_function(attr, item).unwrap().to_string(); - assert!( - expanded.contains("StaticStackRuntimeOwned"), - "the dispatch flag must select the pending-owning stack adapter: {expanded}" - ); - assert!( - expanded.contains("declared_host_contract"), - "the guest contract is still declared on the same function: {expanded}" - ); - } - - #[test] - fn runtime_owned_pending_flag_requires_a_stack_dispatch_signature() { - let attr: Punctuated = parse_quote!( - name = "demo::count", - contract = demo_count_contract, - runtime_owned_pending - ); - let item: ItemFn = parse_quote! { - /// A non-yielding signature cannot own a pending operation. - fn count() -> i64 { - todo!() - } - }; - let error = expand_pd_host_function(attr, item) - .expect_err("the pending flag requires a stack-dispatch signature"); - assert!(error.to_string().contains("runtime_owned_pending requires")); - } - #[test] fn contract_argument_declares_schema_effects_and_adapter_in_one_expansion() { let attr: Punctuated = diff --git a/pd-vm-nostd/README.md b/pd-vm-nostd/README.md index 57d7741c..a9590611 100644 --- a/pd-vm-nostd/README.md +++ b/pd-vm-nostd/README.md @@ -6,7 +6,7 @@ compiler, parser, CLI, debugger, JIT/AOT backends, filesystem support, and opera ## Runtime surface -- VMBC v14 decoding with environment-free `CallScript` direct script calls alongside dynamic callable calls, an explicit guest named-struct declaration section, and deterministic rejection of v13 catalog artifacts that require recompilation +- VMBC v14 decoding with environment-free `CallScript` direct script calls alongside dynamic callable calls and an explicit guest named-struct declaration section - stack, local, and recursive script-frame execution for direct bytecode opcodes - instruction fuel with pause/resume support - synchronous named host bindings and dynamic host dispatch diff --git a/pd-vm-nostd/src/vmbc.rs b/pd-vm-nostd/src/vmbc.rs index 4b31ef53..8c133479 100644 --- a/pd-vm-nostd/src/vmbc.rs +++ b/pd-vm-nostd/src/vmbc.rs @@ -8,10 +8,7 @@ use super::{ }; const MAGIC: [u8; 4] = *b"VMBC"; -const VERSION_V11: u16 = 11; -const VERSION_V12: u16 = 12; -const VERSION_V13: u16 = 13; -const VERSION_V14: u16 = 14; +const VERSION: u16 = 14; const FLAGS: u16 = 0; const MAX_WIRE_PAYLOAD_BYTES: usize = 64 * 1024 * 1024; const MAX_WIRE_BLOB_BYTES: usize = 16 * 1024 * 1024; @@ -74,12 +71,9 @@ pub fn decode_program(bytes: &[u8]) -> Result { } let version = cursor.read_u16()?; - let has_host_import_schemas = match version { - VERSION_V11 => false, - VERSION_V12 | VERSION_V14 => true, - VERSION_V13 => return Err(WireError::UnsupportedVersion(VERSION_V13)), - _ => return Err(WireError::UnsupportedVersion(version)), - }; + if version != VERSION { + return Err(WireError::UnsupportedVersion(version)); + } let flags = cursor.read_u16()?; if flags != FLAGS { return Err(WireError::UnsupportedFlags(flags)); @@ -96,10 +90,7 @@ pub fn decode_program(bytes: &[u8]) -> Result { let mut code = Vec::new(); reserve(&mut code, "code", code_bytes.len())?; code.extend_from_slice(code_bytes); - if version == VERSION_V11 && code.contains(&(OpCode::CallScript as u8)) { - return Err(WireError::UnsupportedVersion(VERSION_V11)); - } - let import_count = cursor.read_count("imports", if has_host_import_schemas { 7 } else { 6 })?; + let import_count = cursor.read_count("imports", 7)?; let mut imports = Vec::new(); reserve(&mut imports, "imports", import_count)?; for _ in 0..import_count { @@ -108,12 +99,10 @@ pub fn decode_program(bytes: &[u8]) -> Result { arity: cursor.read_u8()?, return_type: read_value_type(cursor.read_u8()?)?, }); - if has_host_import_schemas { - match cursor.read_u8()? { - 0 => {} - 1 => skip_host_import_schema(&mut cursor)?, - value => return Err(WireError::InvalidBool(value)), - } + match cursor.read_u8()? { + 0 => {} + 1 => skip_host_import_schema(&mut cursor)?, + value => return Err(WireError::InvalidBool(value)), } } @@ -126,9 +115,7 @@ pub fn decode_program(bytes: &[u8]) -> Result { root_callable_bindings, exported_callables, ) = read_callable_metadata(&mut cursor)?; - if version >= VERSION_V13 { - skip_named_struct_decls(&mut cursor)?; - } + skip_named_struct_decls(&mut cursor)?; if !cursor.is_empty() { return Err(WireError::TrailingBytes); } @@ -490,7 +477,7 @@ fn read_callable_metadata(cursor: &mut Cursor<'_>) -> Result int { value + 2 } add2(40);") - .expect("direct call source should compile"); - let mut bytes = encode_program(&compiled.program.with_local_count(compiled.locals)) - .expect("direct call program should encode"); - bytes[4..6].copy_from_slice(&11u16.to_le_bytes()); - let err = decode_program(&bytes).expect_err("VMBC v11 must be rejected"); - assert!( - matches!(err, WireError::UnsupportedVersion(11)), - "expected UnsupportedVersion(11), got {err:?}" - ); -} - #[test] fn call_script_fuel_interruption() { let compiled = compile_source( diff --git a/pd-vm-nostd/tests/embedded_vmbc.rs b/pd-vm-nostd/tests/embedded_vmbc.rs index b641ebde..523b9194 100644 --- a/pd-vm-nostd/tests/embedded_vmbc.rs +++ b/pd-vm-nostd/tests/embedded_vmbc.rs @@ -54,17 +54,6 @@ fn embedded_decoder_reads_host_generated_v14() { assert_eq!(program.imports()[0].arity, 1); } -#[test] -fn embedded_decoder_rejects_v13_catalog_artifacts() { - let mut bytes = encoded_scalar_program(); - bytes[4..6].copy_from_slice(&13u16.to_le_bytes()); - - assert!(matches!( - decode_program(&bytes), - Err(WireError::UnsupportedVersion(13)) - )); -} - #[test] fn embedded_decoder_skips_full_host_schema_metadata() { let resource = ResourceTypeKey::new("embedded.resource").expect("resource key"); @@ -156,25 +145,10 @@ fn embedded_decoder_skips_named_host_schema_from_std_encode() { assert_eq!(decoded.imports()[0].name, "embedded::named"); } -#[test] -fn embedded_decoder_reads_legacy_v11_without_schema_markers() { - let program = Program::new( - vec![Value::Int(7)], - vec![OpCode::Ldc as u8, 0, 0, 0, 0, OpCode::Ret as u8], - ); - let mut bytes = encode_program(&program).expect("legacy fixture should encode"); - assert_eq!(&bytes[bytes.len() - 4..], &[0, 0, 0, 0]); - bytes.truncate(bytes.len() - 4); - bytes[4..6].copy_from_slice(&11u16.to_le_bytes()); - - let decoded = decode_program(&bytes).expect("embedded decoder should accept VMBC v11"); - assert_eq!(decoded.constants()[0], EmbeddedValue::Int(7)); -} - fn minimal_vmbc_prefix(constant_count: u32, code: &[u8], import_count: u32) -> Vec { let mut bytes = Vec::new(); bytes.extend_from_slice(b"VMBC"); - bytes.extend_from_slice(&12u16.to_le_bytes()); + bytes.extend_from_slice(&14u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&constant_count.to_le_bytes()); bytes.extend_from_slice(&(code.len() as u32).to_le_bytes()); @@ -200,7 +174,7 @@ fn embedded_decoder_rejects_oversized_zero_byte_counts_before_allocation() { )); } -fn v12_with_local_schema(schema: &[u8]) -> Vec { +fn current_vmbc_with_local_schema(schema: &[u8]) -> Vec { let mut bytes = minimal_vmbc_prefix(0, &[EmbeddedOpCode::Ret as u8], 0); bytes.extend_from_slice(&[1, 0]); bytes.extend_from_slice(&1u32.to_le_bytes()); @@ -218,10 +192,11 @@ fn v12_with_local_schema(schema: &[u8]) -> Vec { bytes.extend_from_slice(&0u32.to_le_bytes()); bytes.extend_from_slice(&0u32.to_le_bytes()); bytes.extend_from_slice(&0u32.to_le_bytes()); + bytes.extend_from_slice(&0u32.to_le_bytes()); bytes } -fn v12_with_callable_frame_counts(frame_counts: &[u32]) -> Vec { +fn current_vmbc_with_callable_frame_counts(frame_counts: &[u32]) -> Vec { let mut bytes = minimal_vmbc_prefix(0, &[EmbeddedOpCode::Ret as u8], 0); bytes.extend_from_slice(&[0, 0]); // no type map, no debug info bytes.extend_from_slice(&0u32.to_le_bytes()); // script functions @@ -240,10 +215,11 @@ fn v12_with_callable_frame_counts(frame_counts: &[u32]) -> Vec { bytes.extend_from_slice(&0u32.to_le_bytes()); // function regions bytes.extend_from_slice(&0u32.to_le_bytes()); // root callable bindings bytes.extend_from_slice(&0u32.to_le_bytes()); // exported callables + bytes.extend_from_slice(&0u32.to_le_bytes()); // named structs bytes } -fn v12_with_large_type_map(local_count: u32) -> Vec { +fn current_vmbc_with_large_type_map(local_count: u32) -> Vec { let mut bytes = minimal_vmbc_prefix(0, &[EmbeddedOpCode::Ret as u8], 0); bytes.extend_from_slice(&[1, 0]); // type map, strict=false bytes.extend_from_slice(&local_count.to_le_bytes()); @@ -263,6 +239,7 @@ fn v12_with_large_type_map(local_count: u32) -> Vec { bytes.extend_from_slice(&0u32.to_le_bytes()); // function regions bytes.extend_from_slice(&0u32.to_le_bytes()); // root callable bindings bytes.extend_from_slice(&0u32.to_le_bytes()); // exported callables + bytes.extend_from_slice(&0u32.to_le_bytes()); // named structs bytes } @@ -285,7 +262,7 @@ fn embedded_decoder_accepts_root_resource_schema_tag_17() { #[test] fn embedded_decoder_debits_repeated_callable_frame_counts_from_one_budget() { - let bytes = v12_with_callable_frame_counts(&[40_000; 30]); + let bytes = current_vmbc_with_callable_frame_counts(&[40_000; 30]); assert!( matches!( decode_program(&bytes), @@ -308,7 +285,7 @@ fn embedded_decoder_validates_the_complete_resource_schema_key() { schema.extend_from_slice(&(key.len() as u32).to_le_bytes()); schema.extend_from_slice(key); assert!(matches!( - decode_program(&v12_with_local_schema(&schema)), + decode_program(¤t_vmbc_with_local_schema(&schema)), Err(WireError::InvalidResourceKey) )); } @@ -318,14 +295,14 @@ fn embedded_decoder_validates_the_complete_resource_schema_key() { schema.extend_from_slice(&(key_with_trailing_byte.len() as u32).to_le_bytes()); schema.extend_from_slice(key_with_trailing_byte); assert!(matches!( - decode_program(&v12_with_local_schema(&schema)), + decode_program(¤t_vmbc_with_local_schema(&schema)), Err(WireError::InvalidResourceKey) )); } #[test] fn embedded_decoder_rejects_a_single_oversized_callable_frame() { - let bytes = v12_with_callable_frame_counts(&[65_537]); + let bytes = current_vmbc_with_callable_frame_counts(&[65_537]); assert!(matches!( decode_program(&bytes), Err(WireError::LengthTooLarge("callable frame locals", 65_537)) @@ -334,7 +311,7 @@ fn embedded_decoder_rejects_a_single_oversized_callable_frame() { #[test] fn embedded_decoder_rejects_oversized_program_frame_count_from_type_map() { - let bytes = v12_with_large_type_map(65_537); + let bytes = current_vmbc_with_large_type_map(65_537); assert!(matches!( decode_program(&bytes), Err(WireError::LengthTooLarge("type map locals", 65_537)) @@ -354,7 +331,7 @@ fn schema_with_oversized_count(tag: u8, count: u32) -> Vec { fn embedded_decoder_rejects_oversized_nested_schema_counts() { const TOO_MANY: u32 = 1_000_001; for tag in [9, 11, 12, 14, 15] { - let bytes = v12_with_local_schema(&schema_with_oversized_count(tag, TOO_MANY)); + let bytes = current_vmbc_with_local_schema(&schema_with_oversized_count(tag, TOO_MANY)); assert!(matches!( decode_program(&bytes), Err(WireError::LengthTooLarge(_, count)) if count == TOO_MANY as usize @@ -520,8 +497,7 @@ fn embedded_runtime_executes_compiler_generated_capturing_callable() { #[test] fn call_script_opcode_is_0x1a_in_both_crates() { - // The historical callable-creation opcode slot (0x1A) is now the static - // script-call opcode in both the std and embedded opcode tables. + // The static script-call opcode is identical in the std and embedded tables. assert_eq!(OpCode::try_from(0x1a), Ok(OpCode::CallScript)); assert_eq!( EmbeddedOpCode::try_from(0x1a), @@ -535,7 +511,7 @@ fn append_wire_string(out: &mut Vec, value: &str) { out.extend_from_slice(value.as_bytes()); } -fn v12_with_named_host_return_schema(schema: &[u8]) -> Vec { +fn current_vmbc_with_named_host_return_schema(schema: &[u8]) -> Vec { let mut bytes = minimal_vmbc_prefix(0, &[EmbeddedOpCode::Ret as u8], 1); bytes.extend_from_slice(&1u32.to_le_bytes()); bytes.push(b'h'); @@ -552,6 +528,7 @@ fn v12_with_named_host_return_schema(schema: &[u8]) -> Vec { bytes.extend_from_slice(&0u32.to_le_bytes()); bytes.extend_from_slice(&0u32.to_le_bytes()); bytes.extend_from_slice(&0u32.to_le_bytes()); + bytes.extend_from_slice(&0u32.to_le_bytes()); bytes } @@ -577,7 +554,7 @@ fn nested_named_host_schema(depth: usize) -> Vec { #[test] fn embedded_decoder_accepts_empty_named_host_schema() { - let bytes = v12_with_named_host_return_schema(&empty_named_host_schema("Point")); + let bytes = current_vmbc_with_named_host_return_schema(&empty_named_host_schema("Point")); decode_program(&bytes).expect("empty Named host schema should skip"); } @@ -587,7 +564,7 @@ fn embedded_decoder_rejects_truncated_named_host_schema() { append_wire_string(&mut schema, "Point"); schema.extend_from_slice(&1u32.to_le_bytes()); assert_eq!( - decode_program(&v12_with_named_host_return_schema(&schema)), + decode_program(¤t_vmbc_with_named_host_return_schema(&schema)), Err(WireError::UnexpectedEof) ); } @@ -599,7 +576,7 @@ fn embedded_decoder_rejects_oversized_named_host_field_count() { append_wire_string(&mut schema, "Point"); schema.extend_from_slice(&TOO_MANY.to_le_bytes()); assert!(matches!( - decode_program(&v12_with_named_host_return_schema(&schema)), + decode_program(¤t_vmbc_with_named_host_return_schema(&schema)), Err(WireError::LengthTooLarge("host named struct fields", count)) if count == TOO_MANY as usize )); @@ -613,7 +590,7 @@ fn embedded_decoder_rejects_malformed_nested_named_host_schema() { append_wire_string(&mut schema, "inner"); schema.push(99); assert_eq!( - decode_program(&v12_with_named_host_return_schema(&schema)), + decode_program(¤t_vmbc_with_named_host_return_schema(&schema)), Err(WireError::InvalidValueType(99)) ); } @@ -628,14 +605,14 @@ fn embedded_decoder_rejects_truncated_nested_named_host_schema() { append_wire_string(&mut schema, "Inner"); schema.extend_from_slice(&1u32.to_le_bytes()); assert_eq!( - decode_program(&v12_with_named_host_return_schema(&schema)), + decode_program(¤t_vmbc_with_named_host_return_schema(&schema)), Err(WireError::UnexpectedEof) ); } #[test] fn embedded_decoder_rejects_oversized_nested_named_host_depth() { - let bytes = v12_with_named_host_return_schema(&nested_named_host_schema(64)); + let bytes = current_vmbc_with_named_host_return_schema(&nested_named_host_schema(64)); assert_eq!(decode_program(&bytes), Err(WireError::SchemaTooDeep)); } @@ -688,15 +665,3 @@ fn embedded_decoder_rejects_duplicate_named_struct_names() { bytes.extend_from_slice(&0u32.to_le_bytes()); assert_eq!(decode_program(&bytes), Err(WireError::InvalidValueType(0))); } - -#[test] -fn embedded_decoder_rejects_v12_trailing_zero_named_struct_garbage() { - let mut bytes = encode_program(&Program::new(Vec::new(), vec![OpCode::Ret as u8])) - .expect("empty program should encode"); - assert_eq!(&bytes[bytes.len() - 4..], &[0, 0, 0, 0]); - bytes.truncate(bytes.len() - 4); - bytes[4..6].copy_from_slice(&12u16.to_le_bytes()); - decode_program(&bytes).expect("clean v12 should decode"); - bytes.extend_from_slice(&0u32.to_le_bytes()); - assert_eq!(decode_program(&bytes), Err(WireError::TrailingBytes)); -} diff --git a/pd-vm-wasm/src/runtime.rs b/pd-vm-wasm/src/runtime.rs index 92439d96..4f0afc5d 100644 --- a/pd-vm-wasm/src/runtime.rs +++ b/pd-vm-wasm/src/runtime.rs @@ -12,8 +12,9 @@ use std::time::Instant; use serde::Deserialize; use vm::{ CallOutcome, CallReturn, FunctionDecl, HostAsyncBridge, HostAsyncOpTerminal, HostFunction, - HostOpId, LocalInfo, SourceFlavor, SourcePathError, Value, Vm, VmError, VmResult, VmStatus, - VmYieldReason, compile_source_with_flavor_and_options, format_value, render_vm_error, + HostFutureOutput, HostOpId, LocalInfo, SourceFlavor, SourcePathError, Value, Vm, VmError, + VmResult, VmStatus, VmYieldReason, compile_source_with_flavor_and_options, format_value, + render_vm_error, }; use crate::analyzer::{LintDiagnostic, lint_source_with_flavor, lint_success_diagnostics}; @@ -295,6 +296,16 @@ impl HostAsyncBridge for BrowserAsyncBridge { } } + fn poll_submitted_op( + &mut self, + op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Err(VmError::HostError(format!( + "unknown submitted host operation {op_id}" + )))) + } + fn request_cancel_op( &mut self, op_id: HostOpId, diff --git a/plans/tail_call_optimization_plan.md b/plans/tail_call_optimization_plan.md index 7ff57e08..4dfa74aa 100644 --- a/plans/tail_call_optimization_plan.md +++ b/plans/tail_call_optimization_plan.md @@ -46,14 +46,14 @@ The configurable call-depth limit remains a safety guard for non-tail recursion **Tasks:** - Add `TailCallValue(argc)` with the same operand layout as `CallValue(argc)`. -- Bump `BYTECODE_ABI_VERSION` and VMBC to v11 as a hard internal-format break; update no-std decoding and reject all earlier versions. +- Update `BYTECODE_ABI_VERSION`, VMBC encoding, and no-std decoding for the new opcode. - Update opcode parsing, mnemonic rendering, assembler APIs, disassembly, validation, stack-effect analysis, function-region checks, and typed operand metadata. - Require `TailCallValue` to occur inside a script function region. Reject it in the root region and reject malformed arity/stack shapes. - Keep `Call` host-only and retain existing `CallValue` for non-tail script calls and Rust-host invocation boundaries. **Tests:** -- VMBC v11 round-trip and old-version rejection; +- current VMBC round-trip; - assembler/disassembler round-trip for `tailcallvalue`; - validator rejection in the root region and across malformed function regions; - no-std decoder parity. diff --git a/src/builtins/runtime/timer.rs b/src/builtins/runtime/timer.rs index bbefb036..d0e31577 100644 --- a/src/builtins/runtime/timer.rs +++ b/src/builtins/runtime/timer.rs @@ -1086,6 +1086,22 @@ mod tests { ) -> Poll> { Poll::Pending } + + fn poll_submitted_op( + &mut self, + _op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Pending + } + + fn cleanup_op( + &mut self, + _op_id: HostOpId, + _terminal: crate::HostAsyncOpTerminal, + ) -> VmResult<()> { + Ok(()) + } } /// Bridge stub driven by a test-owned state: it completes, fails, or @@ -1123,8 +1139,45 @@ mod tests { } } - fn cancel_op(&mut self, _op_id: HostOpId) { + fn poll_submitted_op( + &mut self, + _op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { + let state = self.state.lock().expect("bridge state"); + assert!(!state.panic_poll, "registered bridge poll panic"); + if state.poll_error { + Poll::Ready(Err(VmError::HostError("bridge poll failed".to_string()))) + } else if state.complete { + Poll::Ready(Ok(crate::HostFutureOutput::returning(CallReturn::none()))) + } else { + Poll::Pending + } + } + + fn request_cancel_op( + &mut self, + _op_id: HostOpId, + _reason: crate::operation::OperationCancelReason, + ) -> VmResult<()> { self.state.lock().expect("bridge state").cancellations += 1; + Ok(()) + } + + fn poll_cancel_op( + &mut self, + _op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + + fn cleanup_op( + &mut self, + _op_id: HostOpId, + _terminal: crate::HostAsyncOpTerminal, + ) -> VmResult<()> { + Ok(()) } } diff --git a/src/bytecode.rs b/src/bytecode.rs index faa6d9c0..3425b723 100644 --- a/src/bytecode.rs +++ b/src/bytecode.rs @@ -7,11 +7,8 @@ use crate::compiler::{StructDecl, TypeSchema}; use crate::host_api::HostImportSchema; /// Bytecode ABI version used for VM-internal cache identity (JIT trace cache, -/// program cache keys). The VMBC wire format version lives in `src/vmbc.rs` -/// (`VERSION_V14`); both were bumped together for the static builtin ID break -/// and again for the direct script-call (`CallScript`) opcode break. Version 13 -/// adds an explicit guest named-struct declaration section. Version 14 marks -/// the catalog fingerprint v3 break after transient HTTP resources were removed. +/// program cache keys). The VMBC wire format version lives in `src/vmbc.rs`; +/// both current formats use version 14 and catalog fingerprint v3. pub const BYTECODE_ABI_VERSION: u16 = 14; pub type SharedString = Arc; diff --git a/src/vm/host.rs b/src/vm/host.rs index c6757542..25646fb9 100644 --- a/src/vm/host.rs +++ b/src/vm/host.rs @@ -804,21 +804,6 @@ pub enum HostAsyncOpTerminal { Failed, } -impl HostAsyncOpTerminal { - /// Returns the reason used when default cleanup finalizes this terminal - /// operation. Cleanup is a terminal resource-release action rather than a - /// new cancellation request, so every terminal state uses the stable - /// `Requested` compatibility reason; an actual cancellation reason is - /// delivered earlier through `request_cancel_op`. - pub const fn cleanup_reason(self) -> OperationCancelReason { - match self { - Self::Completed => OperationCancelReason::Requested, - Self::Cancelled => OperationCancelReason::Requested, - Self::Failed => OperationCancelReason::Requested, - } - } -} - pub trait HostAsyncBridge: Send { fn submit_op(&mut self, _op_id: HostOpId, _future: HostFuture) -> VmResult<()> { Err(VmError::HostError( @@ -832,38 +817,18 @@ pub trait HostAsyncBridge: Send { &mut self, op_id: HostOpId, cx: &mut Context<'_>, - ) -> Poll> { - self.poll_op(op_id, cx) - .map(|result| result.map(HostFutureOutput::Return)) - } - - /// Legacy cancellation hook kept for bridge implementations that do not - /// need a lifecycle reason. It is used as a best-effort fallback by the - /// default [`request_cancel_op`](Self::request_cancel_op) implementation. - fn cancel_op(&mut self, _op_id: HostOpId) {} - - /// Legacy cancellation hook kept for bridge implementations that do not - /// need a lifecycle reason. New bridges should implement - /// [`request_cancel_op`](Self::request_cancel_op) and - /// [`poll_cancel_op`](Self::poll_cancel_op) instead. - fn cancel_op_with_reason(&mut self, op_id: HostOpId, _reason: OperationCancelReason) { - self.cancel_op(op_id); - } + ) -> Poll>; /// Requests cancellation of one bridge-owned operation. /// /// Returning `Ok(())` only records that the request was accepted. It does /// not mean that the operation has stopped; callers must poll /// [`poll_cancel_op`](Self::poll_cancel_op) until it returns `Ready(Ok(()))`. - /// The default invokes the legacy best-effort hook, then fails explicitly so - /// an adapter that has not opted into acknowledgement can never claim - /// quiescence. fn request_cancel_op( &mut self, op_id: HostOpId, - reason: OperationCancelReason, + _reason: OperationCancelReason, ) -> VmResult<()> { - self.cancel_op_with_reason(op_id, reason); Err(VmError::HostError(format!( "async host bridge does not provide cancellation acknowledgement for op {op_id}" ))) @@ -881,13 +846,7 @@ pub trait HostAsyncBridge: Send { /// Runs bridge-side cleanup after a terminal/quiescent outcome has been /// reported. The VM invokes this at most once for each tracked operation. - /// The default preserves compatibility with bridges whose legacy - /// `cancel_op` method also removes completed operation state while routing - /// through the reason-aware hook for newer bridges. - fn cleanup_op(&mut self, op_id: HostOpId, terminal: HostAsyncOpTerminal) -> VmResult<()> { - self.cancel_op_with_reason(op_id, terminal.cleanup_reason()); - Ok(()) - } + fn cleanup_op(&mut self, op_id: HostOpId, terminal: HostAsyncOpTerminal) -> VmResult<()>; } pub type StaticHostFunction = fn(&mut Vm, &[Value]) -> VmResult; @@ -1575,18 +1534,6 @@ impl HostFunctionRegistry { self.invalidate_plan_cache(); } - /// Marks an exact import as owning its pending operation. Pending - /// dispatch is resolved from the generic VM operation/stream registries; - /// the marker is intentionally a registration hook with no domain state. - pub fn mark_exact_runtime_owned_pending(&mut self, name: &str) -> VmResult<()> { - if !self.contains_name(name) { - return Err(VmError::HostError(format!( - "cannot mark unregistered host import '{name}' as runtime-owned" - ))); - } - Ok(()) - } - pub fn register_catalog_stack( &mut self, schema: HostImportSchema, diff --git a/src/vm/host_extension.rs b/src/vm/host_extension.rs index 6f824f67..42b86242 100644 --- a/src/vm/host_extension.rs +++ b/src/vm/host_extension.rs @@ -111,10 +111,6 @@ pub enum HostBindingKind { Static, /// Stack-mutating adapter with `&mut Vm`. StaticStack, - /// Stack-mutating adapter whose pending operation is owned by the generic - /// runtime operation/stream registries rather than by a registered - /// operation driver. - StaticStackRuntimeOwned, /// Args-slice adapter without `&mut Vm`. StaticArgs, /// Non-yielding args-slice adapter. @@ -154,8 +150,6 @@ pub enum HostAdapterDescriptor { Static(super::host::StaticHostFunction), /// [`super::host::StaticHostStackFunction`]. StaticStack(super::host::StaticHostStackFunction), - /// Pending-operation-owning [`super::host::StaticHostStackFunction`]. - StaticStackRuntimeOwned(super::host::StaticHostStackFunction), /// [`super::host::StaticHostArgsFunction`]. StaticArgs(super::host::StaticHostArgsFunction), /// Non-yielding [`super::host::StaticHostArgsFunction`]. @@ -534,7 +528,6 @@ fn install_descriptor_adapter( descriptor: &HostFunctionDescriptor, schema: HostImportSchema, ) -> VmResult<()> { - let mut runtime_owned_pending: Option = None; let result = match (&descriptor.binding.kind, &descriptor.adapter) { (HostBindingKind::Static, HostAdapterDescriptor::Static(function)) => { registry.register_catalog_static(schema, *function) @@ -542,13 +535,7 @@ fn install_descriptor_adapter( (HostBindingKind::StaticStack, HostAdapterDescriptor::StaticStack(function)) => { registry.register_catalog_static_stack(schema, *function) } - ( - HostBindingKind::StaticStackRuntimeOwned, - HostAdapterDescriptor::StaticStackRuntimeOwned(function), - ) => { - runtime_owned_pending = Some(schema.name.clone()); - registry.register_catalog_static_stack(schema, *function) - } + (HostBindingKind::StaticArgs, HostAdapterDescriptor::StaticArgs(function)) => { registry.register_catalog_static_args(schema, *function) } @@ -572,12 +559,6 @@ fn install_descriptor_adapter( descriptor.schema.name )) })?; - if let Some(name) = runtime_owned_pending { - // The pending operation is resolved from the generic VM - // operation/stream registries, which requires the registered import to - // be marked runtime-owned. - registry.mark_exact_runtime_owned_pending(&name)?; - } let _ = registered; Ok(()) } diff --git a/src/vm/tests.rs b/src/vm/tests.rs index f6cb010d..ac0f35eb 100644 --- a/src/vm/tests.rs +++ b/src/vm/tests.rs @@ -89,6 +89,18 @@ fn builtin_pending_completion_uses_declared_return_type() { ) -> Poll> { Poll::Pending } + + fn poll_submitted_op( + &mut self, + _op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Pending + } + + fn cleanup_op(&mut self, _op_id: HostOpId, _terminal: HostAsyncOpTerminal) -> VmResult<()> { + Ok(()) + } } let builtin = BuiltinFunction::from_namespaced_name("io::exists") @@ -410,6 +422,18 @@ impl HostAsyncBridge for ReserveBeforeSubmitBridge { fn poll_op(&mut self, _op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { Poll::Pending } + + fn poll_submitted_op( + &mut self, + _op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Pending + } + + fn cleanup_op(&mut self, _op_id: HostOpId, _terminal: HostAsyncOpTerminal) -> VmResult<()> { + Ok(()) + } } fn empty_host_future() -> HostFuture { @@ -493,66 +517,6 @@ fn submitted_bridge_operation_reserves_before_submit_and_rolls_back_on_failure() ); } -struct ReasonAwareCleanupOnlyBridge { - cleanups: Arc>>, -} - -impl HostAsyncBridge for ReasonAwareCleanupOnlyBridge { - fn submit_op(&mut self, _op_id: HostOpId, _future: HostFuture) -> VmResult<()> { - Ok(()) - } - - fn poll_op(&mut self, _op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { - Poll::Pending - } - - fn cancel_op_with_reason(&mut self, op_id: HostOpId, reason: OperationCancelReason) { - self.cleanups - .lock() - .expect("cleanup lock") - .push((op_id, reason)); - } -} - -#[test] -fn default_bridge_cleanup_routes_every_terminal_state_through_reason_aware_hook() { - let cleanups = Arc::new(Mutex::new(Vec::new())); - let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8])); - vm.set_async_bridge(Box::new(ReasonAwareCleanupOnlyBridge { - cleanups: Arc::clone(&cleanups), - })) - .expect("bridge should install"); - - let mut op_ids = Vec::new(); - for _ in 0..3 { - let CallOutcome::Pending(op_id) = vm - .submit_host_future(empty_host_future()) - .expect("submission should succeed") - else { - panic!("submission should suspend"); - }; - op_ids.push(op_id); - } - for (op_id, terminal) in op_ids.iter().copied().zip([ - HostAsyncOpTerminal::Completed, - HostAsyncOpTerminal::Failed, - HostAsyncOpTerminal::Cancelled, - ]) { - vm.host - .complete_bridge_operation(op_id, terminal) - .expect("terminal cleanup should succeed"); - } - - assert_eq!( - *cleanups.lock().expect("cleanup lock"), - op_ids - .into_iter() - .map(|op_id| (op_id, OperationCancelReason::Requested)) - .collect::>() - ); - assert!(vm.host.submitted_host_ops.is_empty()); -} - #[test] fn shared_capture_cell_rejects_callable_ownership_cycle() { let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8]).with_local_count(1)); @@ -2542,6 +2506,18 @@ fn async_host_future_is_submitted_to_the_host_bridge() { ) -> std::task::Poll> { std::task::Poll::Pending } + + fn poll_submitted_op( + &mut self, + _op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Pending + } + + fn cleanup_op(&mut self, _op_id: HostOpId, _terminal: HostAsyncOpTerminal) -> VmResult<()> { + Ok(()) + } } let submitted = Arc::new(Mutex::new(Vec::new())); @@ -2631,8 +2607,9 @@ fn async_host_future_completion_error_cleans_up_bridge_operation_once() { }))) } - fn cancel_op(&mut self, op_id: HostOpId) { + fn cleanup_op(&mut self, op_id: HostOpId, _terminal: HostAsyncOpTerminal) -> VmResult<()> { self.cleanup_calls.lock().expect("cleanup lock").push(op_id); + Ok(()) } } @@ -2694,11 +2671,12 @@ impl HostAsyncBridge for CleanupRecordingBridge { Poll::Pending } - fn cancel_op_with_reason(&mut self, op_id: HostOpId, reason: OperationCancelReason) { - self.cancellations - .lock() - .expect("cancellation lock") - .push((op_id, reason)); + fn poll_submitted_op( + &mut self, + _op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Pending } fn request_cancel_op( @@ -2706,7 +2684,10 @@ impl HostAsyncBridge for CleanupRecordingBridge { op_id: HostOpId, reason: OperationCancelReason, ) -> VmResult<()> { - self.cancel_op_with_reason(op_id, reason); + self.cancellations + .lock() + .expect("cancellation lock") + .push((op_id, reason)); Ok(()) } @@ -2714,10 +2695,7 @@ impl HostAsyncBridge for CleanupRecordingBridge { Poll::Ready(Ok(())) } - fn cleanup_op(&mut self, op_id: HostOpId, terminal: HostAsyncOpTerminal) -> VmResult<()> { - if terminal == HostAsyncOpTerminal::Completed { - self.cancel_op_with_reason(op_id, OperationCancelReason::Requested); - } + fn cleanup_op(&mut self, _op_id: HostOpId, _terminal: HostAsyncOpTerminal) -> VmResult<()> { Ok(()) } } @@ -2750,27 +2728,21 @@ fn manually_completing_submitted_bridge_op_retires_entry_once() { assert_eq!(vm.waiting_host_op_id(), None); assert!(vm.host.submitted_host_ops.is_empty()); assert_eq!(vm.stack(), &[Value::Int(7)]); - assert_eq!( - *cancellations.lock().expect("cancellation lock"), - vec![(op_id, OperationCancelReason::Requested)] - ); + assert!(cancellations.lock().expect("cancellation lock").is_empty()); assert_eq!(*submissions.lock().expect("submission lock"), vec![op_id]); let error = vm .complete_host_op(op_id, CallReturn::none()) .expect_err("a second completion has no waiting operation"); assert!(error.to_string().contains("not waiting on any op")); - assert_eq!( - cancellations.lock().expect("cancellation lock").as_slice(), - &[(op_id, OperationCancelReason::Requested)] - ); + assert!(cancellations.lock().expect("cancellation lock").is_empty()); } #[test] -fn legacy_manual_pending_without_bridge_is_untracked_and_resets_cleanly() { - struct LegacyManualPending; +fn manual_external_pending_without_bridge_is_untracked_and_resets_cleanly() { + struct ManualExternalPending; - impl HostFunction for LegacyManualPending { + impl HostFunction for ManualExternalPending { fn call(&mut self, _vm: &mut Vm, _args: &[Value]) -> VmResult { Ok(CallOutcome::Pending(404)) } @@ -2780,17 +2752,17 @@ fn legacy_manual_pending_without_bridge_is_untracked_and_resets_cleanly() { bytecode.call(0, 0); bytecode.ret(); let mut vm = Vm::new(Program::new(Vec::new(), bytecode.finish())); - vm.register_function(Box::new(LegacyManualPending)); + vm.register_function(Box::new(ManualExternalPending)); assert_eq!( - vm.run().expect("legacy host op should suspend"), + vm.run().expect("external host op should suspend"), VmStatus::Waiting(404) ); let waiting = vm .instance .waiting_host_op .as_ref() - .expect("legacy pending operation should be recorded"); + .expect("external pending operation should be recorded"); assert_eq!(waiting.source, WaitingHostOpSource::Manual); assert!( !vm.host.is_bridge_operation_tracked(404), @@ -2798,7 +2770,7 @@ fn legacy_manual_pending_without_bridge_is_untracked_and_resets_cleanly() { ); vm.reset_for_reuse() - .expect("reset must clear a legacy manual pending operation"); + .expect("reset must clear a manual external pending operation"); assert_eq!(vm.waiting_host_op_id(), None); assert!(!vm.host.is_bridge_operation_tracked(404)); assert!(vm.is_reusable()); @@ -3228,6 +3200,14 @@ impl HostAsyncBridge for DelayedCancellationBridge { Poll::Pending } + fn poll_submitted_op( + &mut self, + _op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Pending + } + fn request_cancel_op( &mut self, op_id: HostOpId, @@ -3267,6 +3247,18 @@ impl HostAsyncBridge for NoAcknowledgementBridge { fn poll_op(&mut self, _op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { Poll::Pending } + + fn poll_submitted_op( + &mut self, + _op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Pending + } + + fn cleanup_op(&mut self, _op_id: HostOpId, _terminal: HostAsyncOpTerminal) -> VmResult<()> { + Ok(()) + } } fn submitted_host_future(vm: &mut Vm) -> HostOpId { diff --git a/src/vmbc.rs b/src/vmbc.rs index af13240d..3fb40997 100644 --- a/src/vmbc.rs +++ b/src/vmbc.rs @@ -18,10 +18,7 @@ use crate::host_api::{ use crate::vm::{HostImport, OpCode, Program, Value}; const MAGIC: [u8; 4] = *b"VMBC"; -const VERSION_V11: u16 = 11; -const VERSION_V12: u16 = 12; -const VERSION_V13: u16 = 13; -const VERSION_V14: u16 = 14; +const VERSION: u16 = 14; const FLAGS: u16 = 0; const MAX_WIRE_PAYLOAD_BYTES: usize = 64 * 1024 * 1024; const MAX_WIRE_BLOB_BYTES: usize = 16 * 1024 * 1024; @@ -307,7 +304,7 @@ fn read_constant(cursor: &mut Cursor<'_>, depth: usize) -> Result Result, WireError> { let mut out = Vec::new(); out.extend_from_slice(&MAGIC); - out.extend_from_slice(&VERSION_V14.to_le_bytes()); + out.extend_from_slice(&VERSION.to_le_bytes()); out.extend_from_slice(&FLAGS.to_le_bytes()); write_u32_count("constants", program.constants.len(), &mut out)?; @@ -361,12 +358,9 @@ pub fn decode_program(bytes: &[u8]) -> Result { } let version = cursor.read_u16()?; - let has_host_import_schemas = match version { - VERSION_V11 => false, - VERSION_V12 | VERSION_V14 => true, - VERSION_V13 => return Err(WireError::UnsupportedVersion(VERSION_V13)), - _ => return Err(WireError::UnsupportedVersion(version)), - }; + if version != VERSION { + return Err(WireError::UnsupportedVersion(version)); + } let flags = cursor.read_u16()?; if flags != FLAGS { @@ -384,43 +378,30 @@ pub fn decode_program(bytes: &[u8]) -> Result { let mut code = Vec::new(); reserve_vec(&mut code, "code", code_bytes.len())?; code.extend_from_slice(code_bytes); - if version == VERSION_V11 && code.contains(&(OpCode::CallScript as u8)) { - // CallScript was added in V12. Keep the legacy version branch based - // on the version discriminant, independent of the import count. - return Err(WireError::UnsupportedVersion(VERSION_V11)); - } - let import_count = cursor.read_count("imports", if has_host_import_schemas { 7 } else { 6 })?; + let import_count = cursor.read_count("imports", 7)?; let mut imports = Vec::new(); reserve_vec(&mut imports, "imports", import_count)?; let mut host_import_schemas = Vec::new(); - // Do not reserve `import_count` here: a V12 payload may contain a large - // number of `None` entries, while a schema-bearing payload is bounded by - // the shared host-schema budget as each element is decoded. + // Do not reserve `import_count` here: a payload may contain a large number + // of `None` entries, while a schema-bearing payload is bounded by the + // shared host-schema budget as each element is decoded. for _ in 0..import_count { let import = HostImport { - name: if has_host_import_schemas { - cursor.read_bounded_string("host import name", MAX_HOST_FUNCTION_NAME_LEN)? - } else { - cursor.read_string()? - }, + name: cursor.read_bounded_string("host import name", MAX_HOST_FUNCTION_NAME_LEN)?, arity: cursor.read_u8()?, return_type: read_value_type(cursor.read_u8()?)?, }; - if has_host_import_schemas { - let schema = read_optional_host_import_schema(&mut cursor)?; - if let Some(schema) = schema.as_ref() - && (schema.name != import.name || schema.arity() != import.arity as usize) - { - return Err(WireError::HostSchemaImportMismatch); - } - host_import_schemas.push(schema); + let schema = read_optional_host_import_schema(&mut cursor)?; + if let Some(schema) = schema.as_ref() + && (schema.name != import.name || schema.arity() != import.arity as usize) + { + return Err(WireError::HostSchemaImportMismatch); } + host_import_schemas.push(schema); imports.push(import); } - if has_host_import_schemas { - crate::host_api::validate_optional_host_import_schemas(&host_import_schemas) - .map_err(|error| WireError::InvalidHostSchemaComplexity(error.to_string()))?; - } + crate::host_api::validate_optional_host_import_schemas(&host_import_schemas) + .map_err(|error| WireError::InvalidHostSchemaComplexity(error.to_string()))?; let type_map = read_type_map(&mut cursor)?; let debug = read_debug_info(&mut cursor)?; let ( @@ -430,11 +411,7 @@ pub fn decode_program(bytes: &[u8]) -> Result { root_callable_bindings, exported_callables, ) = read_callable_metadata(&mut cursor)?; - let named_struct_decls = if version >= VERSION_V13 { - read_named_struct_decls(&mut cursor)? - } else { - HashMap::new() - }; + let named_struct_decls = read_named_struct_decls(&mut cursor)?; if !cursor.is_eof() { return Err(WireError::TrailingBytes); diff --git a/tests/builtins/sqlite_scope_lifecycle_tests.rs b/tests/builtins/sqlite_scope_lifecycle_tests.rs index 5146d931..aaa6dc66 100644 --- a/tests/builtins/sqlite_scope_lifecycle_tests.rs +++ b/tests/builtins/sqlite_scope_lifecycle_tests.rs @@ -60,10 +60,6 @@ impl HostAsyncBridge for TokioHostDriver { poll } - fn cancel_op(&mut self, op_id: HostOpId) { - self.submitted.remove(&op_id); - } - fn request_cancel_op( &mut self, op_id: HostOpId, @@ -76,6 +72,11 @@ impl HostAsyncBridge for TokioHostDriver { fn poll_cancel_op(&mut self, _op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { Poll::Ready(Ok(())) } + + fn cleanup_op(&mut self, op_id: HostOpId, _terminal: vm::HostAsyncOpTerminal) -> VmResult<()> { + self.submitted.remove(&op_id); + Ok(()) + } } fn install_host_driver(vm: &mut Vm) { diff --git a/tests/host_descriptor_effect_tests.rs b/tests/host_descriptor_effect_tests.rs index be9f91a3..632d2508 100644 --- a/tests/host_descriptor_effect_tests.rs +++ b/tests/host_descriptor_effect_tests.rs @@ -1631,57 +1631,6 @@ fn owned_descriptor_keeps_the_take_owned_contract_and_adapter() { ); } -#[test] -fn runtime_owned_pending_descriptors_mark_the_registered_import() { - use vm::host_extension::{ - HostAdapterDescriptor, HostBindingDescriptor, HostBindingKind, HostFunctionDescriptor, - HostModuleDescriptor, - }; - - fn pending_request_descriptor() -> HostFunctionDescriptor { - HostFunctionDescriptor { - schema: HostFunctionSchema::with_return( - "demo::request", - vec![HostParamSchema::value("request", HostTypeSchema::String)], - HostTypeSchema::String, - ), - binding: HostBindingDescriptor { - kind: HostBindingKind::StaticStackRuntimeOwned, - }, - effects: vec![], - adapter: HostAdapterDescriptor::StaticStackRuntimeOwned(noop_host), - resource_types: vec![], - } - } - - let module = HostModuleDescriptor { - name: "demo.pending", - functions: &[pending_request_descriptor], - resources: &[], - }; - let mut registry = vm::HostFunctionRegistry::empty(); - module - .install(&mut registry) - .expect("a runtime-owned pending descriptor installs"); - assert!(registry.contains_name("demo::request")); - - // A mismatched binding/adapter pair still fails closed. - let mismatched = HostFunctionDescriptor { - binding: HostBindingDescriptor { - kind: HostBindingKind::StaticStackRuntimeOwned, - }, - adapter: HostAdapterDescriptor::StaticStack(noop_host), - ..pending_request_descriptor() - }; - let mut target = vm::HostFunctionRegistry::empty(); - let error = HostModuleDescriptor::install_descriptors(&mut target, &[mismatched]) - .expect_err("a binding/adapter mismatch must fail closed"); - assert!( - error.to_string().contains("binding/adapter mismatch"), - "{error}" - ); -} - #[test] fn module_install_from_catalog_keeps_caller_identity_and_restricted_policy() { use vm::host_extension::HostModuleDescriptor; diff --git a/tests/http_async_arch_tests.rs b/tests/http_async_arch_tests.rs index a313d38a..974d7c30 100644 --- a/tests/http_async_arch_tests.rs +++ b/tests/http_async_arch_tests.rs @@ -42,7 +42,6 @@ fn http_hosts_are_macro_owned_async_functions_without_private_drivers() { ("src/builtins/runtime/http/sse.rs", sse.as_str()), ] { for forbidden in [ - "runtime_owned_pending", "submit_host_future", "HostAsyncBridge", "std::thread", diff --git a/tests/http_named_struct_contract_tests.rs b/tests/http_named_struct_contract_tests.rs index 2e538b27..b4cbdcfc 100644 --- a/tests/http_named_struct_contract_tests.rs +++ b/tests/http_named_struct_contract_tests.rs @@ -727,8 +727,22 @@ impl HostAsyncBridge for TokioHostDriver { poll } - fn cancel_op(&mut self, op_id: HostOpId) { + fn request_cancel_op( + &mut self, + op_id: HostOpId, + _reason: vm::operation::OperationCancelReason, + ) -> VmResult<()> { self.submitted.remove(&op_id); + Ok(()) + } + + fn poll_cancel_op(&mut self, _op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn cleanup_op(&mut self, op_id: HostOpId, _terminal: vm::HostAsyncOpTerminal) -> VmResult<()> { + self.submitted.remove(&op_id); + Ok(()) } } diff --git a/tests/io_descriptor_install_tests.rs b/tests/io_descriptor_install_tests.rs index a2a2cfdc..966e35a3 100644 --- a/tests/io_descriptor_install_tests.rs +++ b/tests/io_descriptor_install_tests.rs @@ -406,8 +406,30 @@ fn install_async_driver(vm: &mut Vm) -> SubmittedOps { poll } - fn cancel_op(&mut self, op_id: HostOpId) { + fn request_cancel_op( + &mut self, + op_id: HostOpId, + _reason: vm::operation::OperationCancelReason, + ) -> VmResult<()> { self.submitted.remove(&op_id); + Ok(()) + } + + fn poll_cancel_op( + &mut self, + _op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + + fn cleanup_op( + &mut self, + op_id: HostOpId, + _terminal: vm::HostAsyncOpTerminal, + ) -> VmResult<()> { + self.submitted.remove(&op_id); + Ok(()) } } diff --git a/tests/sqlite_async_host_arch_tests.rs b/tests/sqlite_async_host_arch_tests.rs index 555f5752..03b109f1 100644 --- a/tests/sqlite_async_host_arch_tests.rs +++ b/tests/sqlite_async_host_arch_tests.rs @@ -21,10 +21,7 @@ fn sqlite_host_functions_are_macro_owned_async_functions() { SQLITE_SOURCE.contains("tokio_rusqlite::Connection"), "the SQLite resource must use the maintained Tokio-facing adapter" ); - assert!( - !SQLITE_SOURCE.contains("runtime_owned_pending"), - "SQLite async functions must use macro-owned future submission" - ); + assert_eq!( SQLITE_SOURCE.matches("HostFutureOutput::complete").count(), 2, diff --git a/tests/sqlite_named_struct_tests.rs b/tests/sqlite_named_struct_tests.rs index 7a5678b8..8f9091a1 100644 --- a/tests/sqlite_named_struct_tests.rs +++ b/tests/sqlite_named_struct_tests.rs @@ -53,8 +53,22 @@ impl HostAsyncBridge for TokioHostDriver { poll } - fn cancel_op(&mut self, op_id: HostOpId) { + fn request_cancel_op( + &mut self, + op_id: HostOpId, + _reason: vm::operation::OperationCancelReason, + ) -> VmResult<()> { self.submitted.remove(&op_id); + Ok(()) + } + + fn poll_cancel_op(&mut self, _op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn cleanup_op(&mut self, op_id: HostOpId, _terminal: vm::HostAsyncOpTerminal) -> VmResult<()> { + self.submitted.remove(&op_id); + Ok(()) } } diff --git a/tests/support/async_test_bridge.rs b/tests/support/async_test_bridge.rs index 1cdb2bf7..2e2d4d9b 100644 --- a/tests/support/async_test_bridge.rs +++ b/tests/support/async_test_bridge.rs @@ -66,10 +66,6 @@ impl HostAsyncBridge for TokioTestBridge { poll } - fn cancel_op(&mut self, op_id: HostOpId) { - self.futures.remove(&op_id); - } - fn request_cancel_op( &mut self, op_id: HostOpId, @@ -82,6 +78,11 @@ impl HostAsyncBridge for TokioTestBridge { fn poll_cancel_op(&mut self, _op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { Poll::Ready(Ok(())) } + + fn cleanup_op(&mut self, op_id: HostOpId, _terminal: vm::HostAsyncOpTerminal) -> VmResult<()> { + self.futures.remove(&op_id); + Ok(()) + } } pub(crate) fn install(vm: &mut Vm) { diff --git a/tests/timer_host_tests.rs b/tests/timer_host_tests.rs index 2012e97a..5784705b 100644 --- a/tests/timer_host_tests.rs +++ b/tests/timer_host_tests.rs @@ -281,16 +281,41 @@ impl HostAsyncBridge for ControlledBridge { Ok(()) } - fn poll_op(&mut self, _op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { + fn poll_op(&mut self, op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Err(VmError::HostError(format!( + "unknown external host operation {op_id}" + )))) + } + + fn poll_submitted_op( + &mut self, + _op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { if self.state.lock().expect("bridge state").complete { - Poll::Ready(Ok(CallReturn::one(Value::Bool(false)))) + Poll::Ready(Ok(vm::HostFutureOutput::returning(CallReturn::one( + Value::Bool(false), + )))) } else { Poll::Pending } } - fn cancel_op(&mut self, _op_id: HostOpId) { + fn request_cancel_op( + &mut self, + _op_id: HostOpId, + _reason: vm::operation::OperationCancelReason, + ) -> VmResult<()> { self.state.lock().expect("bridge state").cancellations += 1; + Ok(()) + } + + fn poll_cancel_op(&mut self, _op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn cleanup_op(&mut self, _op_id: HostOpId, _terminal: vm::HostAsyncOpTerminal) -> VmResult<()> { + Ok(()) } } @@ -1112,8 +1137,10 @@ fn backend_installs_independent_bridges_for_waiting_timer_callbacks() { let waker = std::task::Waker::from(Arc::new(NoopWake)); let mut cx = Context::from_waker(&waker); - assert!(matches!(first.callback.poll(&mut cx), Poll::Pending)); - assert!(matches!(second.callback.poll(&mut cx), Poll::Pending)); + let first_poll = first.callback.poll(&mut cx); + assert!(matches!(first_poll, Poll::Pending), "{first_poll:?}"); + let second_poll = second.callback.poll(&mut cx); + assert!(matches!(second_poll, Poll::Pending), "{second_poll:?}"); first_state.lock().expect("first bridge state").complete = true; assert!(matches!( diff --git a/tests/vm/http_host_tests.rs b/tests/vm/http_host_tests.rs index 6e3ecca2..8f994e7b 100644 --- a/tests/vm/http_host_tests.rs +++ b/tests/vm/http_host_tests.rs @@ -50,10 +50,6 @@ impl HostAsyncBridge for TokioHostDriver { poll } - fn cancel_op(&mut self, op_id: HostOpId) { - self.submitted.remove(&op_id); - } - fn request_cancel_op( &mut self, op_id: HostOpId, @@ -66,6 +62,11 @@ impl HostAsyncBridge for TokioHostDriver { fn poll_cancel_op(&mut self, _op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { Poll::Ready(Ok(())) } + + fn cleanup_op(&mut self, op_id: HostOpId, _terminal: vm::HostAsyncOpTerminal) -> VmResult<()> { + self.submitted.remove(&op_id); + Ok(()) + } } fn install_host_driver(vm: &mut Vm) { diff --git a/tests/vm/http_sse_tests.rs b/tests/vm/http_sse_tests.rs index 12a18ba3..e6cdb45c 100644 --- a/tests/vm/http_sse_tests.rs +++ b/tests/vm/http_sse_tests.rs @@ -51,22 +51,23 @@ impl HostAsyncBridge for TokioHostDriver { ) } - fn cancel_op(&mut self, op_id: HostOpId) { - self.submitted.remove(&op_id); - } - fn request_cancel_op( &mut self, op_id: HostOpId, _reason: OperationCancelReason, ) -> VmResult<()> { - self.cancel_op(op_id); + self.submitted.remove(&op_id); Ok(()) } fn poll_cancel_op(&mut self, _op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { Poll::Ready(Ok(())) } + + fn cleanup_op(&mut self, op_id: HostOpId, _terminal: vm::HostAsyncOpTerminal) -> VmResult<()> { + self.submitted.remove(&op_id); + Ok(()) + } } struct AsyncWaitOnce { diff --git a/tests/vm/io_http_coexistence_tests.rs b/tests/vm/io_http_coexistence_tests.rs index b79cd762..8cb66843 100644 --- a/tests/vm/io_http_coexistence_tests.rs +++ b/tests/vm/io_http_coexistence_tests.rs @@ -51,8 +51,22 @@ impl HostAsyncBridge for TokioHostDriver { poll } - fn cancel_op(&mut self, op_id: HostOpId) { + fn request_cancel_op( + &mut self, + op_id: HostOpId, + _reason: vm::operation::OperationCancelReason, + ) -> VmResult<()> { self.submitted.remove(&op_id); + Ok(()) + } + + fn poll_cancel_op(&mut self, _op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn cleanup_op(&mut self, op_id: HostOpId, _terminal: vm::HostAsyncOpTerminal) -> VmResult<()> { + self.submitted.remove(&op_id); + Ok(()) } } diff --git a/tests/vm/vm_async_runtime_tests.rs b/tests/vm/vm_async_runtime_tests.rs index b3e79130..7e47c82c 100644 --- a/tests/vm/vm_async_runtime_tests.rs +++ b/tests/vm/vm_async_runtime_tests.rs @@ -89,12 +89,14 @@ impl HostAsyncBridge for TestAsyncBridge { .poll_op(op_id, cx) } - fn cancel_op(&mut self, op_id: HostOpId) { - self.ops - .lock() - .expect("test async ops lock poisoned") - .pending - .remove(&op_id); + fn poll_submitted_op( + &mut self, + op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Err(VmError::HostError(format!( + "unknown submitted host operation {op_id}" + )))) } fn request_cancel_op( @@ -102,7 +104,11 @@ impl HostAsyncBridge for TestAsyncBridge { op_id: HostOpId, _reason: vm::operation::OperationCancelReason, ) -> Result<(), VmError> { - self.cancel_op(op_id); + self.ops + .lock() + .expect("test async ops lock poisoned") + .pending + .remove(&op_id); Ok(()) } @@ -113,6 +119,19 @@ impl HostAsyncBridge for TestAsyncBridge { ) -> Poll> { Poll::Ready(Ok(())) } + + fn cleanup_op( + &mut self, + op_id: HostOpId, + _terminal: vm::HostAsyncOpTerminal, + ) -> Result<(), VmError> { + self.ops + .lock() + .expect("test async ops lock poisoned") + .pending + .remove(&op_id); + Ok(()) + } } struct RejectingCancelBridge; @@ -130,6 +149,14 @@ impl HostAsyncBridge for RejectingCancelBridge { Poll::Pending } + fn poll_submitted_op( + &mut self, + _op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Pending + } + fn request_cancel_op( &mut self, _op_id: HostOpId, @@ -139,6 +166,14 @@ impl HostAsyncBridge for RejectingCancelBridge { "cancellation request rejected".to_string(), )) } + + fn cleanup_op( + &mut self, + _op_id: HostOpId, + _terminal: vm::HostAsyncOpTerminal, + ) -> Result<(), VmError> { + Ok(()) + } } struct AsyncAddOneFunction { diff --git a/tests/wire/wire_tests.rs b/tests/wire/wire_tests.rs index 3b06786c..dca46f4c 100644 --- a/tests/wire/wire_tests.rs +++ b/tests/wire/wire_tests.rs @@ -68,62 +68,10 @@ fn wire_roundtrip_preserves_constants_and_code() { assert_eq!(decoded.type_map, program.type_map); } -#[test] -fn wire_v11_legacy_imports_decode_without_schema_metadata() { - let import = HostImport { - name: "legacy::import".to_string(), - arity: 0, - return_type: ValueType::Unknown, - }; - let program = Program::with_imports_and_debug( - Vec::new(), - vec![vm::OpCode::Ret as u8], - vec![import.clone()], - None, - ); - let encoded = encode_program(&program).expect("current encoding should succeed"); - let marker_offset = 8 + 4 + 4 + program.code.len() + 4 + 4 + import.name.len() + 2; - assert_eq!(encoded[marker_offset], 0); - let mut legacy = encoded; - legacy.drain(marker_offset..marker_offset + 1); - strip_empty_named_struct_section(&mut legacy); - legacy[4..6].copy_from_slice(&11u16.to_le_bytes()); - - let decoded = decode_program(&legacy).expect("v11 payload should remain readable"); - assert_eq!(decoded.imports, vec![import]); - assert!(decoded.host_import_schemas().is_empty()); -} - -#[test] -fn wire_v11_zero_import_program_decodes_by_version() { - let program = Program::new(Vec::new(), vec![vm::OpCode::Ret as u8]); - let mut encoded = encode_program(&program).expect("current encoding should succeed"); - strip_empty_named_struct_section(&mut encoded); - encoded[4..6].copy_from_slice(&11u16.to_le_bytes()); - - let decoded = decode_program(&encoded).expect("schema-less v11 payload should decode"); - assert_eq!(decoded.code, program.code); - assert!(decoded.imports.is_empty()); - assert!(decoded.host_import_schemas().is_empty()); -} - -fn strip_empty_named_struct_section(encoded: &mut Vec) { - assert!( - encoded.len() >= 4, - "encoded VMBC is too short to contain a named-struct section" - ); - assert_eq!( - &encoded[encoded.len() - 4..], - &[0, 0, 0, 0], - "expected an empty named-struct count trailer on current encode" - ); - encoded.truncate(encoded.len() - 4); -} - fn minimal_vmbc_prefix(constant_count: u32, code: &[u8], import_count: u32) -> Vec { let mut bytes = Vec::new(); bytes.extend_from_slice(b"VMBC"); - bytes.extend_from_slice(&12u16.to_le_bytes()); + bytes.extend_from_slice(&14u16.to_le_bytes()); bytes.extend_from_slice(&0u16.to_le_bytes()); bytes.extend_from_slice(&constant_count.to_le_bytes()); bytes.extend_from_slice(&(code.len() as u32).to_le_bytes()); @@ -149,7 +97,7 @@ fn decode_rejects_oversized_zero_byte_counts_before_allocation() { )); } -fn v12_with_local_schema(schema: &[u8]) -> Vec { +fn current_vmbc_with_local_schema(schema: &[u8]) -> Vec { let mut bytes = minimal_vmbc_prefix(0, &[vm::OpCode::Ret as u8], 0); bytes.extend_from_slice(&[1, 0]); bytes.extend_from_slice(&1u32.to_le_bytes()); @@ -167,10 +115,11 @@ fn v12_with_local_schema(schema: &[u8]) -> Vec { bytes.extend_from_slice(&0u32.to_le_bytes()); bytes.extend_from_slice(&0u32.to_le_bytes()); bytes.extend_from_slice(&0u32.to_le_bytes()); + bytes.extend_from_slice(&0u32.to_le_bytes()); bytes } -fn v12_with_callable_frame_counts(frame_counts: &[u32]) -> Vec { +fn current_vmbc_with_callable_frame_counts(frame_counts: &[u32]) -> Vec { let mut bytes = minimal_vmbc_prefix(0, &[vm::OpCode::Ret as u8], 0); bytes.extend_from_slice(&[0, 0]); // no type map, no debug info bytes.extend_from_slice(&0u32.to_le_bytes()); // script functions @@ -189,10 +138,11 @@ fn v12_with_callable_frame_counts(frame_counts: &[u32]) -> Vec { bytes.extend_from_slice(&0u32.to_le_bytes()); // function regions bytes.extend_from_slice(&0u32.to_le_bytes()); // root callable bindings bytes.extend_from_slice(&0u32.to_le_bytes()); // exported callables + bytes.extend_from_slice(&0u32.to_le_bytes()); // named structs bytes } -fn v12_with_large_type_map(local_count: u32) -> Vec { +fn current_vmbc_with_large_type_map(local_count: u32) -> Vec { let mut bytes = minimal_vmbc_prefix(0, &[vm::OpCode::Ret as u8], 0); bytes.extend_from_slice(&[1, 0]); // type map, strict=false bytes.extend_from_slice(&local_count.to_le_bytes()); @@ -212,6 +162,7 @@ fn v12_with_large_type_map(local_count: u32) -> Vec { bytes.extend_from_slice(&0u32.to_le_bytes()); // function regions bytes.extend_from_slice(&0u32.to_le_bytes()); // root callable bindings bytes.extend_from_slice(&0u32.to_le_bytes()); // exported callables + bytes.extend_from_slice(&0u32.to_le_bytes()); // named structs bytes } @@ -233,7 +184,7 @@ fn wire_roundtrip_preserves_root_resource_schema_for_embedded_decoder() { #[test] fn decode_debits_repeated_callable_frame_counts_from_one_budget() { - let bytes = v12_with_callable_frame_counts(&[40_000; 30]); + let bytes = current_vmbc_with_callable_frame_counts(&[40_000; 30]); assert!( matches!( decode_program(&bytes), @@ -246,7 +197,7 @@ fn decode_debits_repeated_callable_frame_counts_from_one_budget() { #[test] fn decode_rejects_a_single_oversized_callable_frame() { - let bytes = v12_with_callable_frame_counts(&[65_537]); + let bytes = current_vmbc_with_callable_frame_counts(&[65_537]); assert!(matches!( decode_program(&bytes), Err(WireError::LengthTooLarge("callable frame locals", 65_537)) @@ -255,7 +206,7 @@ fn decode_rejects_a_single_oversized_callable_frame() { #[test] fn decode_rejects_oversized_program_frame_count_from_type_map() { - let bytes = v12_with_large_type_map(65_537); + let bytes = current_vmbc_with_large_type_map(65_537); assert!(matches!( decode_program(&bytes), Err(WireError::LengthTooLarge("type map locals", 65_537)) @@ -281,7 +232,7 @@ fn decode_rejects_oversized_nested_schema_counts_before_allocation() { (14, "schema object fields"), (15, "schema callable params"), ] { - let bytes = v12_with_local_schema(&schema_with_oversized_count(tag, TOO_MANY)); + let bytes = current_vmbc_with_local_schema(&schema_with_oversized_count(tag, TOO_MANY)); assert!(matches!( decode_program(&bytes), Err(WireError::LengthTooLarge(actual, count)) @@ -295,14 +246,14 @@ fn decode_rejects_oversized_resource_schema_key_before_allocation() { const TOO_MANY: u32 = 16 * 1024 * 1024 + 1; let mut schema = vec![17]; schema.extend_from_slice(&TOO_MANY.to_le_bytes()); - let bytes = v12_with_local_schema(&schema); + let bytes = current_vmbc_with_local_schema(&schema); assert!(matches!( decode_program(&bytes), Err(WireError::LengthTooLarge("string", count)) if count == TOO_MANY as usize )); } -fn v12_with_oversized_import_schema_param_count(count: u32) -> Vec { +fn current_vmbc_with_oversized_import_schema_param_count(count: u32) -> Vec { let mut bytes = minimal_vmbc_prefix(0, &[vm::OpCode::Ret as u8], 1); bytes.extend_from_slice(&1u32.to_le_bytes()); bytes.push(b'h'); @@ -325,7 +276,7 @@ fn v12_with_oversized_import_schema_param_count(count: u32) -> Vec { #[test] fn decode_rejects_oversized_import_schema_parameter_count_before_allocation() { const TOO_MANY: u32 = 1_000_001; - let bytes = v12_with_oversized_import_schema_param_count(TOO_MANY); + let bytes = current_vmbc_with_oversized_import_schema_param_count(TOO_MANY); assert!(matches!( decode_program(&bytes), Err(WireError::LengthTooLarge("host import schema parameters", count)) @@ -420,20 +371,6 @@ fn decode_rejects_invalid_magic_version_and_truncation() { Err(WireError::UnsupportedVersion(99)) )); - let mut old_version = encoded.clone(); - old_version[4..6].copy_from_slice(&9u16.to_le_bytes()); - assert!(matches!( - decode_program(&old_version), - Err(WireError::UnsupportedVersion(9)) - )); - - let mut previous_version = encoded.clone(); - previous_version[4..6].copy_from_slice(&10u16.to_le_bytes()); - assert!(matches!( - decode_program(&previous_version), - Err(WireError::UnsupportedVersion(10)) - )); - let truncated = &encoded[..encoded.len() - 1]; assert!(matches!( decode_program(truncated), @@ -489,7 +426,7 @@ fn validate_accepts_known_good_program() { } #[test] -fn callable_metadata_roundtrips_vmbc_v12() { +fn callable_metadata_roundtrips_current_vmbc() { let compiled = vm::compile_source_for_repl( r#" fn add_one(value: int) -> int { value + 1 } @@ -850,7 +787,7 @@ fn literal_string_builtin_indices_are_appended_and_publicly_resolved() { } // --------------------------------------------------------------------------- -// Milestone 6: CallScript wire support (VMBC V12) +// CallScript wire support // --------------------------------------------------------------------------- #[test] @@ -859,7 +796,7 @@ fn call_script_roundtrips_validation_and_disassembly() { code.extend_from_slice(&7u32.to_le_bytes()); code.push(2); code.push(vm::OpCode::Ret as u8); - // The V12 validator resolves the prototype id against the callable + // The validator resolves the prototype id against the callable // metadata, so the fixture carries a matching prototype (id 7, arity 2, // script-function target) plus one script function boundary. let program = Program::new(vec![], code).with_callable_metadata( @@ -1066,22 +1003,14 @@ fn validate_rejects_call_script_targeting_host_import_prototype() { } #[test] -fn call_script_wire_version_is_v14_and_v11_accepts_schema_less_program() { +fn call_script_wire_version_is_v14() { let program = Program::new(vec![], vec![vm::OpCode::Ret as u8]); let encoded = encode_program(&program).expect("encode should succeed"); assert_eq!(u16::from_le_bytes([encoded[4], encoded[5]]), 14); - - let mut old = encoded; - strip_empty_named_struct_section(&mut old); - old[4..6].copy_from_slice(&11u16.to_le_bytes()); - decode_program(&old).expect("schema-less v11 program should decode"); } #[test] -fn call_script_no_script_program_code_bytes_unchanged_by_version_bump() { - // Version bumps must not alter instruction bytes for programs without - // script calls: encode a plain arithmetic program and verify the - // embedded code section is exactly the assembler output. +fn current_vmbc_preserves_code_bytes_without_script_calls() { let mut bc = BytecodeBuilder::new(); bc.ldc(0); bc.ldc(1); @@ -1095,22 +1024,6 @@ fn call_script_no_script_program_code_bytes_unchanged_by_version_bump() { assert_eq!(decoded.constants, program.constants); } -#[test] -fn v12_trailing_zero_count_is_not_a_named_struct_table() { - let program = Program::new(Vec::new(), vec![vm::OpCode::Ret as u8]); - let mut encoded = encode_program(&program).expect("current encoding should succeed"); - strip_empty_named_struct_section(&mut encoded); - encoded[4..6].copy_from_slice(&12u16.to_le_bytes()); - decode_program(&encoded).expect("clean v12 without a named-struct section should decode"); - - let mut garbage = encoded; - garbage.extend_from_slice(&0u32.to_le_bytes()); - assert!( - matches!(decode_program(&garbage), Err(WireError::TrailingBytes)), - "v12 must not treat a 4-byte zero trailer as an empty named-struct table" - ); -} - #[test] fn v14_roundtrip_preserves_guest_named_struct_payload() { let compiled = compile_source( @@ -1134,15 +1047,3 @@ fn v14_roundtrip_preserves_guest_named_struct_payload() { "VMBC v14 should preserve guest struct decls" ); } - -#[test] -fn v13_artifact_requires_recompilation_after_catalog_revision() { - let program = Program::new(Vec::new(), vec![vm::OpCode::Ret as u8]); - let mut encoded = encode_program(&program).expect("current program should encode"); - encoded[4..6].copy_from_slice(&13u16.to_le_bytes()); - - assert!(matches!( - decode_program(&encoded), - Err(WireError::UnsupportedVersion(13)) - )); -} From 167304181bbd49f30c66740da3995cc23f3a013e Mon Sep 17 00:00:00 2001 From: fffonion Date: Sat, 19 Sep 2026 20:17:53 +0800 Subject: [PATCH 13/23] refactor(host): keep domain contracts current-only --- build.rs | 95 +- .../fixtures/sqlite-host-catalog.fixture | 1284 +++++++++++++++++ crates/rustscript/tests/lsp_resource_types.rs | 6 +- docs/host-sdk-descriptors.md | 4 +- docs/sqlite.md | 19 +- pd-vm-wasm/src/runtime.rs | 16 +- src/builtins/runtime/host.rs | 4 +- src/builtins/runtime/host_modules.rs | 3 +- src/builtins/runtime/io/mod.rs | 28 +- src/builtins/runtime/mod.rs | 8 +- src/builtins/runtime/sqlite.rs | 175 ++- src/builtins/runtime/sqlite_schema.rs | 116 +- src/cli.rs | 6 +- src/compiler/pipeline.rs | 4 +- src/lib.rs | 9 +- src/vm/async_host/stream.rs | 37 +- src/vm/host.rs | 4 +- src/vm/tests.rs | 68 - tests/builtins/io_async_tests.rs | 101 +- tests/builtins_tests.rs | 4 +- tests/http_async_arch_tests.rs | 84 -- tests/pr24_resource_tests.rs | 75 +- tests/sqlite_async_host_arch_tests.rs | 63 - tests/sqlite_named_struct_tests.rs | 2 +- tests/standard_host_descriptor_arch_tests.rs | 1238 ++-------------- tests/typed_host_no_dynamic_contract_tests.rs | 42 +- tests/vm/http_sse_tests.rs | 51 +- 27 files changed, 1780 insertions(+), 1766 deletions(-) create mode 100644 crates/rustscript/tests/fixtures/sqlite-host-catalog.fixture delete mode 100644 tests/http_async_arch_tests.rs delete mode 100644 tests/sqlite_async_host_arch_tests.rs diff --git a/build.rs b/build.rs index ae0c8a3c..daf568bc 100644 --- a/build.rs +++ b/build.rs @@ -166,18 +166,20 @@ fn main() { // The SQLite namespace is optional: its builtin module links rusqlite, // which is not available on every target or without the `sqlite` feature. - // When the feature is off (or the target is wasm32, where rusqlite's + // When the feature is off (or the target family is wasm, where rusqlite's // bundled build is unsupported), drop the namespace and its static // catalog IDs so the generated catalog, dispatch, and compiler namespace // surface stay consistent and feature-clean. + let target_family = env::var("CARGO_CFG_TARGET_FAMILY").expect("missing target family"); let sqlite_enabled = env::var_os("CARGO_FEATURE_SQLITE").is_some() - && env::var("CARGO_CFG_TARGET_ARCH").as_deref() != Ok("wasm32"); + && !target_family + .split(',') + .any(|family| family.trim() == "wasm"); if !sqlite_enabled { namespaces.retain(|namespace| namespace.namespace != "sqlite"); catalog.retain(|entry| !entry.source_name.starts_with("sqlite::")); } - let target_family = env::var("CARGO_CFG_TARGET_FAMILY").expect("missing target family"); let mut host_sources = vec![ SourceSpec { path: "src/builtins/runtime/host.rs".to_string(), @@ -206,8 +208,7 @@ fn main() { }); } let async_enabled = env::var_os("CARGO_FEATURE_ASYNC").is_some(); - let target_arch = env::var("CARGO_CFG_TARGET_ARCH").expect("missing target architecture"); - let builtin_sources = builtin_source_specs(&namespaces, async_enabled, &target_arch); + let builtin_sources = builtin_source_specs(&namespaces, async_enabled, &target_family); let core_sources = [SourceSpec { path: "src/builtins/runtime/core.rs".to_string(), module: "core".to_string(), @@ -279,8 +280,11 @@ fn write_generated_file(path: &Path, contents: &str) { .unwrap_or_else(|err| panic!("failed to write {}: {err}", path.display())); } -pub(crate) fn select_io_source_path(async_enabled: bool, target_arch: &str) -> &'static str { - if target_arch == "wasm32" { +pub(crate) fn select_io_source_path(async_enabled: bool, target_family: &str) -> &'static str { + if target_family + .split(',') + .any(|family| family.trim() == "wasm") + { "src/builtins/runtime/io_wasm.rs" } else if async_enabled { "src/builtins/runtime/io/async_io.rs" @@ -292,13 +296,13 @@ pub(crate) fn select_io_source_path(async_enabled: bool, target_arch: &str) -> & fn builtin_source_specs( namespaces: &[NamespaceDecl], async_enabled: bool, - target_arch: &str, + target_family: &str, ) -> Vec { namespaces .iter() .map(|namespace| { let path = if namespace.module == "io" { - select_io_source_path(async_enabled, target_arch).to_string() + select_io_source_path(async_enabled, target_family).to_string() } else { format!("src/builtins/runtime/{}.rs", namespace.module) }; @@ -2261,11 +2265,7 @@ fn find_matching_paren(source: &str) -> usize { #[cfg(test)] mod tests { - use super::{ - HostExecutionKind, NamespaceDecl, SourceCategory, builtin_source_specs, - http_transport_enabled, parse_source_file, select_io_source_path, - }; - use std::path::Path; + use super::{http_transport_enabled, select_io_source_path}; #[test] fn http_transport_predicate_matches_source_and_catalog_boundary() { @@ -2277,88 +2277,33 @@ mod tests { assert!(!http_transport_enabled(false, "wasm")); } - fn io_namespace() -> NamespaceDecl { - NamespaceDecl { - namespace: "io".to_string(), - module: "io".to_string(), - docs: "I/O".to_string(), - runtime_supported_on_wasm: false, - } - } - #[test] fn io_source_selection_matches_runtime_module_cfg() { assert_eq!( - select_io_source_path(false, "x86_64"), + select_io_source_path(false, "unix"), "src/builtins/runtime/io/blocking.rs" ); assert_eq!( - select_io_source_path(true, "x86_64"), + select_io_source_path(true, "unix"), "src/builtins/runtime/io/async_io.rs" ); assert_eq!( - select_io_source_path(false, "aarch64"), + select_io_source_path(false, "windows"), "src/builtins/runtime/io/blocking.rs" ); assert_eq!( - select_io_source_path(true, "aarch64"), + select_io_source_path(true, "windows"), "src/builtins/runtime/io/async_io.rs" ); assert_eq!( - select_io_source_path(false, "wasm32"), + select_io_source_path(false, "wasm"), "src/builtins/runtime/io_wasm.rs" ); assert_eq!( - select_io_source_path(true, "wasm32"), + select_io_source_path(true, "wasm"), "src/builtins/runtime/io_wasm.rs" ); } - - #[test] - fn selected_io_source_drives_generated_metadata_input() { - let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); - let namespace = io_namespace(); - for (async_enabled, target_arch) in [ - (false, "x86_64"), - (true, "x86_64"), - (false, "wasm32"), - (true, "wasm32"), - ] { - let specs = - builtin_source_specs(std::slice::from_ref(&namespace), async_enabled, target_arch); - let spec = specs - .iter() - .find(|spec| spec.category == SourceCategory::NamespacedBuiltin) - .expect("the IO namespace must produce a source spec"); - assert_eq!( - spec.path, - select_io_source_path(async_enabled, target_arch), - "metadata must use the same source selected by the runtime module" - ); - let source = std::fs::read_to_string(manifest_dir.join(&spec.path)) - .expect("selected IO source must be readable"); - let open_marker = if async_enabled && target_arch != "wasm32" { - "async fn builtin_io_open" - } else { - "fn builtin_io_open" - }; - assert!( - source.contains(open_marker), - "selected source must provide the expected IO implementation" - ); - let callables = parse_source_file(&manifest_dir.join(&spec.path), spec, 0); - let open = callables - .iter() - .find(|callable| callable.name == "io::open") - .expect("selected IO source must contain io::open"); - let expected_execution = if async_enabled || target_arch == "wasm32" { - HostExecutionKind::MaySuspend - } else { - HostExecutionKind::Sync - }; - assert_eq!(open.host_execution, expected_execution); - } - } } #[cfg(test)] diff --git a/crates/rustscript/tests/fixtures/sqlite-host-catalog.fixture b/crates/rustscript/tests/fixtures/sqlite-host-catalog.fixture new file mode 100644 index 00000000..63fcc87a --- /dev/null +++ b/crates/rustscript/tests/fixtures/sqlite-host-catalog.fixture @@ -0,0 +1,1284 @@ +{ + "resources": [ + { + "key": "sqlite.connection", + "description": "An open SQLite connection" + } + ], + "structs": [ + { + "name": "SqliteOpenOptions", + "fields": [ + { + "name": "path", + "ty": { + "Optional": "String" + } + }, + { + "name": "mode", + "ty": { + "Optional": "String" + } + }, + { + "name": "root", + "ty": { + "Optional": "String" + } + }, + { + "name": "limits", + "ty": { + "Optional": { + "Named": { + "name": "SqliteLimits", + "fields": [ + { + "name": "max_connections", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_statements", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_rows", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_columns", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_result_bytes", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_statement_bytes", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_parameters", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_parameter_bytes", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_pending_operations", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_transaction_ms", + "ty": { + "Optional": "Int" + } + }, + { + "name": "busy_timeout_ms", + "ty": { + "Optional": "Int" + } + } + ] + } + } + } + } + ], + "description": "" + }, + { + "name": "SqliteLimits", + "fields": [ + { + "name": "max_connections", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_statements", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_rows", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_columns", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_result_bytes", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_statement_bytes", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_parameters", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_parameter_bytes", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_pending_operations", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_transaction_ms", + "ty": { + "Optional": "Int" + } + }, + { + "name": "busy_timeout_ms", + "ty": { + "Optional": "Int" + } + } + ], + "description": "" + }, + { + "name": "SqliteValue", + "fields": [ + { + "name": "kind", + "ty": "String" + }, + { + "name": "int_value", + "ty": { + "Optional": "Int" + } + }, + { + "name": "float_value", + "ty": { + "Optional": "Float" + } + }, + { + "name": "text_value", + "ty": { + "Optional": "String" + } + }, + { + "name": "blob_value", + "ty": { + "Optional": "Bytes" + } + } + ], + "description": "" + }, + { + "name": "SqliteExecuteResult", + "fields": [ + { + "name": "rows_affected", + "ty": "Int" + }, + { + "name": "last_insert_rowid", + "ty": "Int" + } + ], + "description": "" + }, + { + "name": "SqliteQueryResult", + "fields": [ + { + "name": "columns", + "ty": { + "Array": "String" + } + }, + { + "name": "rows", + "ty": { + "Array": { + "Named": { + "name": "SqliteRow", + "fields": [ + { + "name": "cells", + "ty": { + "Array": { + "Named": { + "name": "SqliteValue", + "fields": [ + { + "name": "kind", + "ty": "String" + }, + { + "name": "int_value", + "ty": { + "Optional": "Int" + } + }, + { + "name": "float_value", + "ty": { + "Optional": "Float" + } + }, + { + "name": "text_value", + "ty": { + "Optional": "String" + } + }, + { + "name": "blob_value", + "ty": { + "Optional": "Bytes" + } + } + ] + } + } + } + } + ] + } + } + } + }, + { + "name": "truncated", + "ty": "Bool" + }, + { + "name": "next_cursor", + "ty": { + "Optional": "Int" + } + } + ], + "description": "" + }, + { + "name": "SqliteRow", + "fields": [ + { + "name": "cells", + "ty": { + "Array": { + "Named": { + "name": "SqliteValue", + "fields": [ + { + "name": "kind", + "ty": "String" + }, + { + "name": "int_value", + "ty": { + "Optional": "Int" + } + }, + { + "name": "float_value", + "ty": { + "Optional": "Float" + } + }, + { + "name": "text_value", + "ty": { + "Optional": "String" + } + }, + { + "name": "blob_value", + "ty": { + "Optional": "Bytes" + } + } + ] + } + } + } + } + ], + "description": "" + }, + { + "name": "SqliteStatement", + "fields": [ + { + "name": "sql", + "ty": "String" + }, + { + "name": "params", + "ty": { + "Optional": { + "Array": { + "Named": { + "name": "SqliteValue", + "fields": [ + { + "name": "kind", + "ty": "String" + }, + { + "name": "int_value", + "ty": { + "Optional": "Int" + } + }, + { + "name": "float_value", + "ty": { + "Optional": "Float" + } + }, + { + "name": "text_value", + "ty": { + "Optional": "String" + } + }, + { + "name": "blob_value", + "ty": { + "Optional": "Bytes" + } + } + ] + } + } + } + } + }, + { + "name": "query", + "ty": { + "Optional": "Bool" + } + }, + { + "name": "limits", + "ty": { + "Optional": { + "Named": { + "name": "SqliteLimits", + "fields": [ + { + "name": "max_connections", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_statements", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_rows", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_columns", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_result_bytes", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_statement_bytes", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_parameters", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_parameter_bytes", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_pending_operations", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_transaction_ms", + "ty": { + "Optional": "Int" + } + }, + { + "name": "busy_timeout_ms", + "ty": { + "Optional": "Int" + } + } + ] + } + } + } + } + ], + "description": "" + }, + { + "name": "SqliteTransactionResult", + "fields": [ + { + "name": "kind", + "ty": "String" + }, + { + "name": "execute", + "ty": { + "Optional": { + "Named": { + "name": "SqliteExecuteResult", + "fields": [ + { + "name": "rows_affected", + "ty": "Int" + }, + { + "name": "last_insert_rowid", + "ty": "Int" + } + ] + } + } + } + }, + { + "name": "query", + "ty": { + "Optional": { + "Named": { + "name": "SqliteQueryResult", + "fields": [ + { + "name": "columns", + "ty": { + "Array": "String" + } + }, + { + "name": "rows", + "ty": { + "Array": { + "Named": { + "name": "SqliteRow", + "fields": [ + { + "name": "cells", + "ty": { + "Array": { + "Named": { + "name": "SqliteValue", + "fields": [ + { + "name": "kind", + "ty": "String" + }, + { + "name": "int_value", + "ty": { + "Optional": "Int" + } + }, + { + "name": "float_value", + "ty": { + "Optional": "Float" + } + }, + { + "name": "text_value", + "ty": { + "Optional": "String" + } + }, + { + "name": "blob_value", + "ty": { + "Optional": "Bytes" + } + } + ] + } + } + } + } + ] + } + } + } + }, + { + "name": "truncated", + "ty": "Bool" + }, + { + "name": "next_cursor", + "ty": { + "Optional": "Int" + } + } + ] + } + } + } + } + ], + "description": "" + } + ], + "functions": [ + { + "name": "sqlite::open", + "params": [ + { + "name": "options", + "ty": { + "Named": { + "name": "SqliteOpenOptions", + "fields": [ + { + "name": "path", + "ty": { + "Optional": "String" + } + }, + { + "name": "mode", + "ty": { + "Optional": "String" + } + }, + { + "name": "root", + "ty": { + "Optional": "String" + } + }, + { + "name": "limits", + "ty": { + "Optional": { + "Named": { + "name": "SqliteLimits", + "fields": [ + { + "name": "max_connections", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_statements", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_rows", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_columns", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_result_bytes", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_statement_bytes", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_parameters", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_parameter_bytes", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_pending_operations", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_transaction_ms", + "ty": { + "Optional": "Int" + } + }, + { + "name": "busy_timeout_ms", + "ty": { + "Optional": "Int" + } + } + ] + } + } + } + } + ] + } + }, + "passing": "Value" + } + ], + "return_type": { + "Resource": "sqlite.connection" + }, + "description": "" + }, + { + "name": "sqlite::execute", + "params": [ + { + "name": "connection", + "ty": { + "Resource": "sqlite.connection" + }, + "passing": "Borrow" + }, + { + "name": "sql", + "ty": "String", + "passing": "Value" + }, + { + "name": "params", + "ty": { + "Array": { + "Named": { + "name": "SqliteValue", + "fields": [ + { + "name": "kind", + "ty": "String" + }, + { + "name": "int_value", + "ty": { + "Optional": "Int" + } + }, + { + "name": "float_value", + "ty": { + "Optional": "Float" + } + }, + { + "name": "text_value", + "ty": { + "Optional": "String" + } + }, + { + "name": "blob_value", + "ty": { + "Optional": "Bytes" + } + } + ] + } + } + }, + "passing": "Value" + } + ], + "return_type": { + "Named": { + "name": "SqliteExecuteResult", + "fields": [ + { + "name": "rows_affected", + "ty": "Int" + }, + { + "name": "last_insert_rowid", + "ty": "Int" + } + ] + } + }, + "description": "" + }, + { + "name": "sqlite::query", + "params": [ + { + "name": "connection", + "ty": { + "Resource": "sqlite.connection" + }, + "passing": "Borrow" + }, + { + "name": "sql", + "ty": "String", + "passing": "Value" + }, + { + "name": "params", + "ty": { + "Array": { + "Named": { + "name": "SqliteValue", + "fields": [ + { + "name": "kind", + "ty": "String" + }, + { + "name": "int_value", + "ty": { + "Optional": "Int" + } + }, + { + "name": "float_value", + "ty": { + "Optional": "Float" + } + }, + { + "name": "text_value", + "ty": { + "Optional": "String" + } + }, + { + "name": "blob_value", + "ty": { + "Optional": "Bytes" + } + } + ] + } + } + }, + "passing": "Value" + }, + { + "name": "limits", + "ty": { + "Named": { + "name": "SqliteLimits", + "fields": [ + { + "name": "max_connections", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_statements", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_rows", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_columns", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_result_bytes", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_statement_bytes", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_parameters", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_parameter_bytes", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_pending_operations", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_transaction_ms", + "ty": { + "Optional": "Int" + } + }, + { + "name": "busy_timeout_ms", + "ty": { + "Optional": "Int" + } + } + ] + } + }, + "passing": "Value" + } + ], + "return_type": { + "Named": { + "name": "SqliteQueryResult", + "fields": [ + { + "name": "columns", + "ty": { + "Array": "String" + } + }, + { + "name": "rows", + "ty": { + "Array": { + "Named": { + "name": "SqliteRow", + "fields": [ + { + "name": "cells", + "ty": { + "Array": { + "Named": { + "name": "SqliteValue", + "fields": [ + { + "name": "kind", + "ty": "String" + }, + { + "name": "int_value", + "ty": { + "Optional": "Int" + } + }, + { + "name": "float_value", + "ty": { + "Optional": "Float" + } + }, + { + "name": "text_value", + "ty": { + "Optional": "String" + } + }, + { + "name": "blob_value", + "ty": { + "Optional": "Bytes" + } + } + ] + } + } + } + } + ] + } + } + } + }, + { + "name": "truncated", + "ty": "Bool" + }, + { + "name": "next_cursor", + "ty": { + "Optional": "Int" + } + } + ] + } + }, + "description": "" + }, + { + "name": "sqlite::transaction", + "params": [ + { + "name": "connection", + "ty": { + "Resource": "sqlite.connection" + }, + "passing": "Borrow" + }, + { + "name": "statements", + "ty": { + "Array": { + "Named": { + "name": "SqliteStatement", + "fields": [ + { + "name": "sql", + "ty": "String" + }, + { + "name": "params", + "ty": { + "Optional": { + "Array": { + "Named": { + "name": "SqliteValue", + "fields": [ + { + "name": "kind", + "ty": "String" + }, + { + "name": "int_value", + "ty": { + "Optional": "Int" + } + }, + { + "name": "float_value", + "ty": { + "Optional": "Float" + } + }, + { + "name": "text_value", + "ty": { + "Optional": "String" + } + }, + { + "name": "blob_value", + "ty": { + "Optional": "Bytes" + } + } + ] + } + } + } + } + }, + { + "name": "query", + "ty": { + "Optional": "Bool" + } + }, + { + "name": "limits", + "ty": { + "Optional": { + "Named": { + "name": "SqliteLimits", + "fields": [ + { + "name": "max_connections", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_statements", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_rows", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_columns", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_result_bytes", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_statement_bytes", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_parameters", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_parameter_bytes", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_pending_operations", + "ty": { + "Optional": "Int" + } + }, + { + "name": "max_transaction_ms", + "ty": { + "Optional": "Int" + } + }, + { + "name": "busy_timeout_ms", + "ty": { + "Optional": "Int" + } + } + ] + } + } + } + } + ] + } + } + }, + "passing": "Value" + } + ], + "return_type": { + "Array": { + "Named": { + "name": "SqliteTransactionResult", + "fields": [ + { + "name": "kind", + "ty": "String" + }, + { + "name": "execute", + "ty": { + "Optional": { + "Named": { + "name": "SqliteExecuteResult", + "fields": [ + { + "name": "rows_affected", + "ty": "Int" + }, + { + "name": "last_insert_rowid", + "ty": "Int" + } + ] + } + } + } + }, + { + "name": "query", + "ty": { + "Optional": { + "Named": { + "name": "SqliteQueryResult", + "fields": [ + { + "name": "columns", + "ty": { + "Array": "String" + } + }, + { + "name": "rows", + "ty": { + "Array": { + "Named": { + "name": "SqliteRow", + "fields": [ + { + "name": "cells", + "ty": { + "Array": { + "Named": { + "name": "SqliteValue", + "fields": [ + { + "name": "kind", + "ty": "String" + }, + { + "name": "int_value", + "ty": { + "Optional": "Int" + } + }, + { + "name": "float_value", + "ty": { + "Optional": "Float" + } + }, + { + "name": "text_value", + "ty": { + "Optional": "String" + } + }, + { + "name": "blob_value", + "ty": { + "Optional": "Bytes" + } + } + ] + } + } + } + } + ] + } + } + } + }, + { + "name": "truncated", + "ty": "Bool" + }, + { + "name": "next_cursor", + "ty": { + "Optional": "Int" + } + } + ] + } + } + } + } + ] + } + } + }, + "description": "" + }, + { + "name": "sqlite::close", + "params": [ + { + "name": "connection", + "ty": { + "Resource": "sqlite.connection" + }, + "passing": "TakeOwned" + } + ], + "return_type": "Null", + "description": "" + } + ] +} \ No newline at end of file diff --git a/crates/rustscript/tests/lsp_resource_types.rs b/crates/rustscript/tests/lsp_resource_types.rs index 0e6e1f4f..b1029eb8 100644 --- a/crates/rustscript/tests/lsp_resource_types.rs +++ b/crates/rustscript/tests/lsp_resource_types.rs @@ -55,7 +55,11 @@ fn read_framed_message(reader: &mut impl BufRead) -> Option { impl RpcClient { fn spawn() -> Self { - Self::spawn_with_args(&[]) + let catalog = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/sqlite-host-catalog.fixture" + ); + Self::spawn_with_args(&["--catalog", catalog]) } fn spawn_with_args(args: &[&str]) -> Self { diff --git a/docs/host-sdk-descriptors.md b/docs/host-sdk-descriptors.md index 8fa2ce72..d017f87a 100644 --- a/docs/host-sdk-descriptors.md +++ b/docs/host-sdk-descriptors.md @@ -290,9 +290,7 @@ let mut modules: Vec = vec![ /* always-present modules */ ]; #[cfg(all(feature = "http-client", not(target_family = "wasm")))] modules.push(super::http::http_host_module()); modules.push(super::io::io_host_module()); -// The SQLite catalog surface is composed in every build; the `sqlite` feature -// only selects whether its adapters are the real host functions or the -// fail-closed stubs, so there is no gate here. +#[cfg(all(feature = "sqlite", not(target_family = "wasm")))] modules.push(super::sqlite_schema::sqlite_standard_host_module()); modules.push(super::timer::timer_host_module()); modules.sort_by_key(|module| module.name); diff --git a/docs/sqlite.md b/docs/sqlite.md index 988ee103..00c7729c 100644 --- a/docs/sqlite.md +++ b/docs/sqlite.md @@ -19,17 +19,10 @@ serializes work on the connection and owns the blocking SQLite execution thread. ## Compiler and editor catalog boundary -The typed SQLite catalog is a schema-only surface and does not construct a VM or link -`rusqlite`. `sqlite_host_catalog` and the SQLite entries in `standard_host_catalog` remain -available whenever the `runtime` feature is compiled, including builds without the `sqlite` -feature. Catalog-aware compiler callers and the LSP use these declarations for named-struct -field access and exact host signatures. - -The `sqlite` feature controls the executable SQLite module, generated SQLite namespace and -callables, the `rusqlite` and `tokio-rusqlite` dependencies, and SQLite registration exports. A -runtime build without that feature can inspect the editor/compiler contract but has no SQLite -implementation to bind; execution requires a build with `sqlite` enabled, an async host bridge, -and the SQLite module registered. +The `sqlite` feature publishes the complete SQLite module: its typed catalog, generated namespace +and callables, `rusqlite` and `tokio-rusqlite` dependencies, and registration exports. Builds +without that feature, and wasm-family builds, omit the SQLite namespace and catalog entirely. +Execution also requires an async host bridge and the SQLite module registered. ## Open options (`SqliteOpenOptions`) @@ -213,7 +206,9 @@ handler interrupts a transaction after its configured deadline so the transactio ## Resource lifecycle -`sqlite::close(db)` consumes the connection, awaits adapter close, and removes the VM resource. +`sqlite::close(db)` consumes the connection, begins adapter close before suspension, and removes +the VM resource after close completes. Cancelling a blocked close leaves its resource-owned close +lifecycle available for a later close call to poll to completion. VM reset interrupts active SQLite work and remains pending while the resource polls `tokio-rusqlite`'s `Connection::close`; the connection permit is released only after the adapter confirms that queued and running work has drained and the connection has closed. Cancelling an diff --git a/pd-vm-wasm/src/runtime.rs b/pd-vm-wasm/src/runtime.rs index 4f0afc5d..7ff0d392 100644 --- a/pd-vm-wasm/src/runtime.rs +++ b/pd-vm-wasm/src/runtime.rs @@ -1,12 +1,12 @@ use std::cell::RefCell; use std::collections::{HashMap, HashSet}; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(target_family = "wasm"))] use std::sync::OnceLock; use std::sync::{Arc, Mutex}; use std::task::{Context, Poll, Wake, Waker}; -#[cfg(all(not(target_arch = "wasm32"), test))] +#[cfg(all(not(target_family = "wasm"), test))] use std::time::Duration; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(target_family = "wasm"))] use std::time::Instant; use serde::Deserialize; @@ -370,7 +370,7 @@ fn noop_waker() -> Waker { Waker::from(Arc::new(NoopWake)) } -#[cfg(target_arch = "wasm32")] +#[cfg(target_family = "wasm")] #[link(wasm_import_module = "env")] unsafe extern "C" { #[link_name = "pd_playground_now_ms"] @@ -378,11 +378,11 @@ unsafe extern "C" { } fn current_time_ms() -> f64 { - #[cfg(target_arch = "wasm32")] + #[cfg(target_family = "wasm")] unsafe { imported_now_ms() } - #[cfg(not(target_arch = "wasm32"))] + #[cfg(not(target_family = "wasm"))] { static START: OnceLock = OnceLock::new(); START.get_or_init(Instant::now).elapsed().as_secs_f64() * 1_000.0 @@ -1078,12 +1078,12 @@ pub(crate) fn run_source_with_flavor(source: &str, flavor: SourceFlavor) -> RunR ); } Poll::Pending => { - #[cfg(not(target_arch = "wasm32"))] + #[cfg(not(target_family = "wasm"))] { std::thread::sleep(Duration::from_millis(1)); continue; } - #[cfg(target_arch = "wasm32")] + #[cfg(target_family = "wasm")] { let output = drain_output(&output_lines); let stack = vm.stack().iter().map(format_value).collect::>(); diff --git a/src/builtins/runtime/host.rs b/src/builtins/runtime/host.rs index d729d05b..31a64b66 100644 --- a/src/builtins/runtime/host.rs +++ b/src/builtins/runtime/host.rs @@ -49,9 +49,9 @@ fn sleep_duration(millis: i64) -> VmResult { #[pd_host_function(name = "runtime::sleep")] fn runtime_sleep_impl(ms: i64) -> VmResult { let duration = sleep_duration(ms)?; - #[cfg(not(target_arch = "wasm32"))] + #[cfg(not(target_family = "wasm"))] std::thread::sleep(duration); - #[cfg(target_arch = "wasm32")] + #[cfg(target_family = "wasm")] let _ = duration; Ok(true) } diff --git a/src/builtins/runtime/host_modules.rs b/src/builtins/runtime/host_modules.rs index 05814294..af578741 100644 --- a/src/builtins/runtime/host_modules.rs +++ b/src/builtins/runtime/host_modules.rs @@ -165,8 +165,7 @@ pub fn standard_host_modules() -> &'static [StandardHostModule] { #[cfg(all(feature = "http-client", not(target_family = "wasm")))] modules.push(super::http::http_host_module()); modules.push(super::io::io_host_module()); - // The SQLite catalog surface is composed in every build; the feature - // only selects whether the adapters are the real host functions. + #[cfg(all(feature = "sqlite", not(target_family = "wasm")))] modules.push(super::sqlite_schema::sqlite_standard_host_module()); modules.push(super::timer::timer_host_module()); modules.sort_by_key(|module| module.name); diff --git a/src/builtins/runtime/io/mod.rs b/src/builtins/runtime/io/mod.rs index 3778b961..66ca2bf4 100644 --- a/src/builtins/runtime/io/mod.rs +++ b/src/builtins/runtime/io/mod.rs @@ -18,13 +18,13 @@ //! contract and the target it compiles for cannot drift. use super::borrow_arg; -#[cfg(all(feature = "async", not(target_arch = "wasm32")))] +#[cfg(all(feature = "async", not(target_family = "wasm")))] use super::{CallOutcome, CaptureAsyncHostContext, return_one}; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(target_family = "wasm"))] use crate::vm::Vm; /// The synchronous pending-call channel used only by the wasm32 stub backend. -#[cfg(target_arch = "wasm32")] +#[cfg(target_family = "wasm")] pub(super) use super::HostCallResult; /// The canonical catalog key of the `io.file` resource type. @@ -45,7 +45,7 @@ pub(crate) const IO_FILE_DESCRIPTION: &str = "An open file handle"; // rejects every IO call regardless of policy, so the type and its accessors // exist only where they are consulted. -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(target_family = "wasm"))] #[derive(Clone, Debug, PartialEq, Eq)] pub struct IoPolicy { pub allowed_roots: Vec, @@ -55,7 +55,7 @@ pub struct IoPolicy { pub max_write_bytes: usize, } -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(target_family = "wasm"))] impl Default for IoPolicy { fn default() -> Self { Self { @@ -69,13 +69,13 @@ impl Default for IoPolicy { } /// I/O host configuration owned by the I/O host implementation. -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(target_family = "wasm"))] pub trait IoHostExt { fn configure_io(&mut self, policy: IoPolicy); fn clear_io_configuration(&mut self); } -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(target_family = "wasm"))] impl IoHostExt for Vm { fn configure_io(&mut self, mut policy: IoPolicy) { policy.allowed_roots.sort(); @@ -92,7 +92,7 @@ impl IoHostExt for Vm { } } -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(target_family = "wasm"))] pub(super) fn io_policy(vm: &Vm) -> Option { vm.host .get_module_state::() @@ -102,22 +102,22 @@ pub(super) fn io_policy(vm: &Vm) -> Option { // ---- cfg-selected implementations ----------------------------------------- -#[cfg(all(feature = "async", not(target_arch = "wasm32")))] +#[cfg(all(feature = "async", not(target_family = "wasm")))] mod async_io; -#[cfg(all(not(feature = "async"), not(target_arch = "wasm32")))] +#[cfg(all(not(feature = "async"), not(target_family = "wasm")))] mod blocking; /// The wasm32 backend lives beside the native ones in the runtime module and is /// pulled in here so it inherits this module's contracts, catalog, ownership /// list, and resource metadata. -#[cfg(target_arch = "wasm32")] +#[cfg(target_family = "wasm")] #[path = "../io_wasm.rs"] mod wasm; -#[cfg(all(feature = "async", not(target_arch = "wasm32")))] +#[cfg(all(feature = "async", not(target_family = "wasm")))] pub(crate) use async_io::*; -#[cfg(all(not(feature = "async"), not(target_arch = "wasm32")))] +#[cfg(all(not(feature = "async"), not(target_family = "wasm")))] pub(crate) use blocking::*; -#[cfg(target_arch = "wasm32")] +#[cfg(target_family = "wasm")] pub(crate) use wasm::*; // ---- guest contracts ------------------------------------------------------- diff --git a/src/builtins/runtime/mod.rs b/src/builtins/runtime/mod.rs index b5d8b671..356e70bc 100644 --- a/src/builtins/runtime/mod.rs +++ b/src/builtins/runtime/mod.rs @@ -33,8 +33,9 @@ mod map_iter; mod math; pub(crate) mod print; pub(crate) mod regex; -#[cfg(all(feature = "sqlite", not(target_arch = "wasm32")))] +#[cfg(all(feature = "sqlite", not(target_family = "wasm")))] pub(crate) mod sqlite; +#[cfg(all(feature = "sqlite", not(target_family = "wasm")))] pub(crate) mod sqlite_schema; pub(crate) mod standard_composition; mod timer; @@ -45,7 +46,7 @@ pub use jit::{ jit_host_catalog, register_jit_builtin_module, register_jit_builtin_module_from_catalog, }; pub use regex::{DEFAULT_REGEX_CACHE_CAPACITY, RegexCache, RegexCacheVmExt}; -#[cfg(all(feature = "sqlite", not(target_arch = "wasm32")))] +#[cfg(all(feature = "sqlite", not(target_family = "wasm")))] pub use sqlite::{register_sqlite_builtin_module, register_sqlite_builtin_module_from_catalog}; pub use timer::{ DEFAULT_MAX_PENDING_TIMERS, DEFAULT_MAX_RUNNING_TIMERS, OwnedTimerCallback, TIMER_CALLBACK_ARG, @@ -75,6 +76,7 @@ pub fn io_host_catalog() -> Arc { /// Returns the editor/compiler catalog for the SQLite host extension, derived /// from the standard SQLite host module descriptors. +#[cfg(all(feature = "sqlite", not(target_family = "wasm")))] pub fn sqlite_host_catalog() -> Arc { static CATALOG: OnceLock> = OnceLock::new(); Arc::clone(CATALOG.get_or_init(|| { @@ -116,7 +118,7 @@ pub(crate) use context::{RuntimeContext, RuntimeContextConfig, STREAM_EMIT_NAME} pub use error::{RuntimeError, RuntimeErrorCode, RuntimeResult}; #[allow(unused_imports)] pub(crate) use event::{EventLimits, EventPayload}; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(not(target_family = "wasm"))] pub use io::{IoHostExt, IoPolicy}; pub use standard_composition::standard_composition; pub use typed::HostCallResult; diff --git a/src/builtins/runtime/sqlite.rs b/src/builtins/runtime/sqlite.rs index 02606032..e980dd6f 100644 --- a/src/builtins/runtime/sqlite.rs +++ b/src/builtins/runtime/sqlite.rs @@ -394,14 +394,11 @@ impl CaptureAsyncHostContext for SqliteOpenContext { #[derive(Clone)] pub(super) struct SqliteConnectionContext { - handle: ResourceHandle, connection: tokio_rusqlite::Connection, - interrupt: Arc, limits: SqliteLimits, allow_unsafe_sql: bool, closed: Arc, in_flight: Arc, - close_lifecycle: Arc, } impl SqliteConnectionContext { @@ -458,6 +455,33 @@ impl CaptureAsyncHostContext for SqliteConnectionContext { } } +#[derive(Clone)] +pub(super) struct SqliteCloseContext { + handle: ResourceHandle, + close_lifecycle: Arc, +} + +impl CaptureAsyncHostContext for SqliteCloseContext { + fn capture(_vm: &mut Vm) -> VmResult { + Err(VmError::HostError( + "SQLite close context requires call arguments".to_string(), + )) + } + + fn capture_with_args(vm: &mut Vm, args: &[Value]) -> VmResult { + let db_id = match args.first() { + Some(Value::Int(value)) => *value, + Some(_) => return Err(VmError::TypeMismatch("int")), + None => { + return Err(VmError::HostError( + "missing SQLite database argument".to_string(), + )); + } + }; + capture_close_connection(vm, db_id) + } +} + /// The default SQLite embedding policy used when no policy has been /// configured through [`SqliteHostExt::configure_sqlite`]. This is the /// value `SqlitePolicy::default()` produces, constructed explicitly so it can @@ -568,13 +592,36 @@ fn lookup_connection(vm: &mut Vm, handle_id: i64) -> VmResult VmResult { + let handle = sqlite_handle(handle_id)?; + let token = vm + .execution_scope() + .resources() + .typed::(handle) + .map_err(|error| VmError::HostError(format!("unknown SQLite database: {error}")))?; + let resource = vm + .execution_scope() + .resources() + .get::(&token) + .map_err(|error| VmError::HostError(format!("SQLite database borrow failed: {error}")))?; + resource + .close_lifecycle + .begin( + resource.connection.clone(), + resource.interrupt.as_ref(), + resource.closed.as_ref(), + ) + .map_err(VmError::HostError)?; + Ok(SqliteCloseContext { + handle, close_lifecycle: Arc::clone(&resource.close_lifecycle), }) } @@ -1600,30 +1647,9 @@ pub(super) async fn builtin_sqlite_transaction_impl( /// Closes the adapter connection, then removes its VM resource. #[pd_host_function(name = "sqlite::close", contract = super::sqlite_schema::sqlite_close_contract)] pub(super) async fn builtin_sqlite_close_impl( - #[pd_host_context] context: SqliteConnectionContext, + #[pd_host_context] context: SqliteCloseContext, _db_id: i64, ) -> VmResult> { - let lease = context.begin_operation()?; - if context.closed.swap(true, Ordering::AcqRel) { - return Err(VmError::HostError( - "SQLite database is already closed".to_string(), - )); - } - context.interrupt.interrupt(); - let _ = context - .connection - .call(move |_connection| { - drop(lease); - Ok::<(), VmError>(()) - }) - .await; - if let Err(message) = context.close_lifecycle.begin( - context.connection.clone(), - context.interrupt.as_ref(), - context.closed.as_ref(), - ) { - return Err(VmError::HostError(message)); - } if let Err(message) = std::future::poll_fn(|cx| context.close_lifecycle.poll(cx)).await { return Err(VmError::HostError(message)); } @@ -1981,6 +2007,91 @@ mod tests { ); } + #[tokio::test] + async fn canceled_blocked_close_can_be_reentered_and_reclaims_the_resource() { + let limits = SqliteLimits::default(); + let options = OpenOptions { + path: ":memory:".to_string(), + mode: OpenMode::Memory, + root: None, + limits, + allow_unsafe_sql: false, + }; + let (connection, interrupt) = open_connection(&options) + .await + .expect("adapter connection should open"); + let (release_tx, release_rx) = mpsc::sync_channel(0); + let blocker_connection = connection.clone(); + let mut blocker = Box::pin(blocker_connection.call(move |_connection| { + release_rx + .recv() + .expect("blocked adapter call should be released"); + Ok::<(), VmError>(()) + })); + let mut cx = Context::from_waker(Waker::noop()); + assert!(matches!(blocker.as_mut().poll(&mut cx), Poll::Pending)); + + let open_connections = Arc::new(AtomicUsize::new(1)); + let closed = Arc::new(AtomicBool::new(false)); + let resource = SqliteResource { + connection, + interrupt, + limits, + allow_unsafe_sql: false, + closed: Arc::clone(&closed), + in_flight: Arc::new(AtomicUsize::new(0)), + close_lifecycle: Arc::new(SqliteCloseLifecycle::new()), + _connection_permit: ConnectionCountPermit { + open_connections: Arc::clone(&open_connections), + }, + }; + let program = crate::compile_source("null;") + .expect("test program should compile") + .program; + let mut vm = Vm::new(program); + let token = vm + .execution_scope() + .push_resource(resource) + .expect("SQLite resource should insert"); + let handle_id = handle_value(token.handle()); + + let context = capture_close_connection(&mut vm, handle_id) + .expect("the first close should capture its resource"); + assert!( + closed.load(Ordering::Acquire), + "close capture must begin the resource-owned lifecycle before suspension" + ); + let mut close = Box::pin(builtin_sqlite_close_impl(context, handle_id)); + assert!(matches!(close.as_mut().poll(&mut cx), Poll::Pending)); + drop(close); + assert_eq!(vm.host_context().resource_count(), 1); + assert_eq!(open_connections.load(Ordering::Acquire), 1); + + let context = capture_close_connection(&mut vm, handle_id) + .expect("a cancelled close should remain available for re-entry"); + let close = builtin_sqlite_close_impl(context, handle_id); + release_tx + .send(()) + .expect("blocked adapter call should still be waiting"); + blocker.await.expect("blocked adapter call should finish"); + let output = tokio::time::timeout(Duration::from_secs(5), close) + .await + .expect("re-entered close should finish") + .expect("re-entered close should succeed"); + match output { + HostFutureOutput::VmCompletion(completion) => { + completion(&mut vm).expect("close completion should remove the resource"); + } + HostFutureOutput::Return(()) => { + panic!("close must remove its resource on the VM thread") + } + HostFutureOutput::VmContinuation(_) => panic!("close must not install a continuation"), + } + + assert_eq!(vm.host_context().resource_count(), 0); + assert_eq!(open_connections.load(Ordering::Acquire), 0); + } + #[tokio::test] async fn canceled_host_future_holds_operation_slot_until_adapter_closure_finishes() { let nonce = SystemTime::now() @@ -2006,19 +2117,16 @@ mod tests { limits, allow_unsafe_sql: false, }; - let (connection, interrupt) = open_connection(&options) + let (connection, _interrupt) = open_connection(&options) .await .expect("adapter connection should open"); let in_flight = Arc::new(AtomicUsize::new(0)); let context = SqliteConnectionContext { - handle: ResourceHandle::encode(1, 0, 1).expect("test handle should encode"), connection: connection.clone(), - interrupt, limits, allow_unsafe_sql: false, closed: Arc::new(AtomicBool::new(false)), in_flight: Arc::clone(&in_flight), - close_lifecycle: Arc::new(SqliteCloseLifecycle::new()), }; let mut operation = Box::pin(builtin_sqlite_execute_impl( context, @@ -2064,19 +2172,16 @@ mod tests { limits, allow_unsafe_sql: false, }; - let (connection, interrupt) = open_connection(&options) + let (connection, _interrupt) = open_connection(&options) .await .expect("adapter connection should open"); let in_flight = Arc::new(AtomicUsize::new(0)); let context = SqliteConnectionContext { - handle: ResourceHandle::encode(1, 0, 1).expect("test handle should encode"), connection: connection.clone(), - interrupt, limits, allow_unsafe_sql: false, closed: Arc::new(AtomicBool::new(false)), in_flight: Arc::clone(&in_flight), - close_lifecycle: Arc::new(SqliteCloseLifecycle::new()), }; builtin_sqlite_execute_impl(context.clone(), 1, String::new(), Arc::new(Vec::new())) diff --git a/src/builtins/runtime/sqlite_schema.rs b/src/builtins/runtime/sqlite_schema.rs index c9c5705b..9bd52136 100644 --- a/src/builtins/runtime/sqlite_schema.rs +++ b/src/builtins/runtime/sqlite_schema.rs @@ -1,16 +1,8 @@ //! SQLite guest contracts and the SQLite host catalog surface. //! -//! This module is deliberately not feature-gated: the standard guest catalog -//! declares the SQLite surface for every build, exactly as it did before the -//! descriptors, so a program compiled against the standard catalog always -//! resolves the same imports. The concrete adapters and the connection -//! resource type live in the feature-gated [`super::sqlite`] module; when that -//! module is compiled out, the surface is still declared from the same -//! contract functions with an adapter that fails closed. -//! //! Every named struct and every function schema below is the single source of -//! the SQLite guest contract: the feature-gated host functions attach these -//! contracts directly, and the catalog is derived from them. +//! the SQLite guest contract: the host functions attach these contracts +//! directly, and the catalog is derived from them. use crate::host_api::{ HostFunctionSchema, HostParamPassing, HostParamSchema, HostStructField, HostStructSchema, @@ -23,33 +15,9 @@ pub(super) const SQLITE_CONNECTION_KEY: &str = "sqlite.connection"; /// The canonical `sqlite.connection` resource description. pub(super) const SQLITE_CONNECTION_DESCRIPTION: &str = "An open SQLite connection"; -/// Resource type declaration used when the SQLite host module is compiled out. -/// -/// The concrete scope resource is [`super::sqlite`]'s `SqliteResource`; this -/// marker only carries the guest-visible declaration for the catalog, so the -/// disabled build keeps the same resource key and description. -#[cfg(not(all(feature = "sqlite", not(target_arch = "wasm32"))))] -pub(super) struct SqliteConnectionResourceMarker; - -#[cfg(not(all(feature = "sqlite", not(target_arch = "wasm32"))))] -impl crate::vm::resource::HostResource for SqliteConnectionResourceMarker {} - -#[cfg(not(all(feature = "sqlite", not(target_arch = "wasm32"))))] -impl crate::host_extension::HostResourceType for SqliteConnectionResourceMarker { - const KEY: &'static str = SQLITE_CONNECTION_KEY; - const DESCRIPTION: &'static str = SQLITE_CONNECTION_DESCRIPTION; -} - /// The canonical declaration for the `sqlite.connection` resource type. pub(super) fn sqlite_connection_resource() -> HostResourceTypeMeta { - #[cfg(all(feature = "sqlite", not(target_arch = "wasm32")))] - { - super::sqlite::concrete_sqlite_connection_resource() - } - #[cfg(not(all(feature = "sqlite", not(target_arch = "wasm32"))))] - { - HostResourceTypeMeta::of::() - } + super::sqlite::concrete_sqlite_connection_resource() } /// The `sqlite.connection` resource key, taken from its single declaration. @@ -335,77 +303,9 @@ pub(super) const SQLITE_NAMED_STRUCTS: &[(&str, &str)] = &[ // coincide; parameter labels, result cells, and mixed transaction outputs use // the typed named structs declared in `super`. -/// Fails closed when the SQLite host module is not compiled into the build. -#[cfg(not(all(feature = "sqlite", not(target_arch = "wasm32"))))] -fn sqlite_module_unavailable( - _vm: &mut crate::vm::Vm, - _args: &[crate::vm::Value], -) -> crate::vm::VmResult { - Err(crate::vm::VmError::HostError( - "the SQLite host module is not compiled into this build".to_string(), - )) -} - -/// One SQLite catalog entry for builds without the SQLite host module. -#[cfg(not(all(feature = "sqlite", not(target_arch = "wasm32"))))] -fn sqlite_unavailable_descriptor( - contract: fn() -> HostFunctionSchema, -) -> crate::host_extension::HostFunctionDescriptor { - crate::host_extension::HostFunctionDescriptor { - schema: contract(), - binding: crate::host_extension::HostBindingDescriptor { - kind: crate::host_extension::HostBindingKind::StaticStack, - }, - effects: crate::host_extension::guest_resource_effects(&contract()), - adapter: crate::host_extension::HostAdapterDescriptor::StaticStack( - sqlite_module_unavailable, - ), - resource_types: Vec::new(), - } -} - -#[cfg(not(all(feature = "sqlite", not(target_arch = "wasm32"))))] -fn sqlite_open_unavailable_descriptor() -> crate::host_extension::HostFunctionDescriptor { - sqlite_unavailable_descriptor(sqlite_open_contract) -} - -#[cfg(not(all(feature = "sqlite", not(target_arch = "wasm32"))))] -fn sqlite_execute_unavailable_descriptor() -> crate::host_extension::HostFunctionDescriptor { - sqlite_unavailable_descriptor(sqlite_execute_contract) -} - -#[cfg(not(all(feature = "sqlite", not(target_arch = "wasm32"))))] -fn sqlite_query_unavailable_descriptor() -> crate::host_extension::HostFunctionDescriptor { - sqlite_unavailable_descriptor(sqlite_query_contract) -} - -#[cfg(not(all(feature = "sqlite", not(target_arch = "wasm32"))))] -fn sqlite_transaction_unavailable_descriptor() -> crate::host_extension::HostFunctionDescriptor { - sqlite_unavailable_descriptor(sqlite_transaction_contract) -} - -#[cfg(not(all(feature = "sqlite", not(target_arch = "wasm32"))))] -fn sqlite_close_unavailable_descriptor() -> crate::host_extension::HostFunctionDescriptor { - sqlite_unavailable_descriptor(sqlite_close_contract) -} - /// Every SQLite catalog function this build owns. -pub(super) const SQLITE_FUNCTIONS: &[fn() -> HostFunctionDescriptor] = { - #[cfg(all(feature = "sqlite", not(target_arch = "wasm32")))] - { - super::sqlite::SQLITE_CATALOG_FUNCTIONS - } - #[cfg(not(all(feature = "sqlite", not(target_arch = "wasm32"))))] - { - &[ - sqlite_open_unavailable_descriptor, - sqlite_execute_unavailable_descriptor, - sqlite_query_unavailable_descriptor, - sqlite_transaction_unavailable_descriptor, - sqlite_close_unavailable_descriptor, - ] - } -}; +pub(super) const SQLITE_FUNCTIONS: &[fn() -> HostFunctionDescriptor] = + super::sqlite::SQLITE_CATALOG_FUNCTIONS; /// The SQLite catalog surface for this build. pub(super) fn sqlite_catalog_module() -> crate::host_extension::HostModuleDescriptor { @@ -413,12 +313,6 @@ pub(super) fn sqlite_catalog_module() -> crate::host_extension::HostModuleDescri } /// The standard `sqlite` host module. -/// -/// The guest catalog surface is part of the standard catalog in **every** -/// build, exactly as before the descriptors: a program compiled against the -/// standard catalog always resolves the same SQLite imports. The features only -/// select the adapters — with the SQLite host module compiled in they are the -/// real host functions, otherwise they fail closed at the runtime boundary. pub(super) fn sqlite_standard_host_module() -> super::host_modules::StandardHostModule { use super::host_modules::StandardHostModule; diff --git a/src/cli.rs b/src/cli.rs index d6aa2831..24b65b02 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1637,11 +1637,11 @@ mod tests { fn cli_build_features_report_compiled_capabilities() { let features = super::cli_build_features(); - #[cfg(all(feature = "sqlite", not(target_arch = "wasm32")))] + #[cfg(all(feature = "sqlite", not(target_family = "wasm")))] let mut modules = vec!["bytes", "io", "re", "json", "jit", "math"]; - #[cfg(not(all(feature = "sqlite", not(target_arch = "wasm32"))))] + #[cfg(not(all(feature = "sqlite", not(target_family = "wasm"))))] let modules = ["bytes", "io", "re", "json", "jit", "math"]; - #[cfg(all(feature = "sqlite", not(target_arch = "wasm32")))] + #[cfg(all(feature = "sqlite", not(target_family = "wasm")))] modules.push("sqlite"); assert_eq!( features, diff --git a/src/compiler/pipeline.rs b/src/compiler/pipeline.rs index 2dbb21a1..64941d42 100644 --- a/src/compiler/pipeline.rs +++ b/src/compiler/pipeline.rs @@ -1854,12 +1854,12 @@ where T: Send + 'static, F: FnOnce() -> T + Send + 'static, { - #[cfg(target_arch = "wasm32")] + #[cfg(target_family = "wasm")] { f() } - #[cfg(not(target_arch = "wasm32"))] + #[cfg(not(target_family = "wasm"))] { const COMPILER_STACK_SIZE: usize = 32 * 1024 * 1024; let handle = std::thread::Builder::new() diff --git a/src/lib.rs b/src/lib.rs index 4081914d..454187ec 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,7 +24,7 @@ pub mod vmbc; pub use assembler::{AsmParseError, Assembler, AssemblerError, BytecodeBuilder, assemble}; #[cfg(feature = "runtime")] pub use builtins::runtime::print::{PrintHostFunction, PrintlnHostFunction, format_value}; -#[cfg(all(feature = "runtime", feature = "sqlite", not(target_arch = "wasm32")))] +#[cfg(all(feature = "runtime", feature = "sqlite", not(target_family = "wasm")))] pub use builtins::runtime::sqlite::{SqliteHostExt, SqliteLimits, SqlitePolicy}; #[cfg(feature = "runtime")] pub use builtins::runtime::{ @@ -34,9 +34,10 @@ pub use builtins::runtime::{ installed_timer_counts, register_owned_timer, register_timer_builtin_module, register_timer_builtin_module_from_catalog, timer_host_catalog, }; -#[cfg(all(feature = "runtime", feature = "sqlite", not(target_arch = "wasm32")))] +#[cfg(all(feature = "runtime", feature = "sqlite", not(target_family = "wasm")))] pub use builtins::runtime::{ register_sqlite_builtin_module, register_sqlite_builtin_module_from_catalog, + sqlite_host_catalog, }; #[cfg(feature = "runtime")] pub(crate) fn install_default_host_functions(registry: &mut vm::HostFunctionRegistry) { @@ -69,10 +70,10 @@ pub use builtins::runtime::{ pub use builtins::runtime::{ DEFAULT_REGEX_CACHE_CAPACITY, RegexCache, RegexCacheVmExt, StandardHostModule, io_host_catalog, jit_host_catalog, register_jit_builtin_module, register_jit_builtin_module_from_catalog, - sqlite_host_catalog, standard_catalog_modules, standard_composition, standard_host_catalog, + standard_catalog_modules, standard_composition, standard_host_catalog, standard_host_catalog_fingerprint, standard_host_modules, }; -#[cfg(all(feature = "runtime", not(target_arch = "wasm32")))] +#[cfg(all(feature = "runtime", not(target_family = "wasm")))] pub use builtins::runtime::{IoHostExt, IoPolicy}; pub use builtins::{ BUILTIN_CATALOG, BuiltinFunction, BuiltinNamespaceMemberSpec, BuiltinNamespaceSpec, diff --git a/src/vm/async_host/stream.rs b/src/vm/async_host/stream.rs index a5e7b144..530736ee 100644 --- a/src/vm/async_host/stream.rs +++ b/src/vm/async_host/stream.rs @@ -67,8 +67,7 @@ pub(crate) struct HostStreamAdmissionError { /// The VM always validates the callback's callable provenance and arity before /// installing a driver. When its metadata is [`TypeSchema::Callable`], it also /// validates a map or Named argument and a map, Named, or Object result. HTTP SSE -/// additionally requires the exact `SseCallbackAction` named type or a matching -/// `{ action: string }` object rather than an arbitrary map. Scripts receive +/// additionally requires the exact `SseCallbackAction` named type. Scripts receive /// ordinary callback items and a final value; they never receive a stream /// handle or a producer poll API. /// @@ -162,8 +161,7 @@ pub(crate) struct HostStreamContinuation { pub(crate) parent_ip: usize, } -/// HTTP SSE callback results retain the existing named action/object runtime -/// compatibility, while callback inputs use the exact `SseEvent` named schema. +/// HTTP SSE callbacks use exact named input and result schemas. #[cfg(feature = "http-client")] fn sse_callback_input_schema(params: &[TypeSchema]) -> bool { matches!( @@ -174,16 +172,10 @@ fn sse_callback_input_schema(params: &[TypeSchema]) -> bool { #[cfg(feature = "http-client")] fn sse_callback_action_result_schema(result: &TypeSchema) -> bool { - match result { - TypeSchema::Named(name, args) => name == "SseCallbackAction" && args.is_empty(), - TypeSchema::Object(fields) => { - fields.len() == 1 - && fields - .get("action") - .is_some_and(|ty| matches!(ty, TypeSchema::String)) - } - _ => false, - } + matches!( + result, + TypeSchema::Named(name, args) if name == "SseCallbackAction" && args.is_empty() + ) } impl Vm { @@ -193,8 +185,8 @@ impl Vm { /// always validates that `callback` is a callable owned by this VM and has /// arity one. When its metadata is [`TypeSchema::Callable`], the VM also /// validates a map or Named argument and a map, Named, or Object result. HTTP SSE - /// uses [`Self::validate_sse_callback_value`] for the exact - /// `SseCallbackAction` named/object contract rather than an arbitrary map. + /// uses [`Self::validate_sse_callback_value`] for the exact named + /// `SseCallbackAction` contract. /// The VM then owns the callback and driver until completion, cancellation, /// reset, or error; removing the driver drops it to release producer /// resources. @@ -311,12 +303,15 @@ impl Vm { else { return Ok(()); }; - if let Some(TypeSchema::Callable { params, result, .. }) = &prototype.schema - && (!sse_callback_input_schema(params) || !sse_callback_action_result_schema(result)) - { - return Err(VmError::TypeMismatch("fn(SseEvent) -> SseCallbackAction")); + match &prototype.schema { + Some(TypeSchema::Callable { params, result, .. }) + if sse_callback_input_schema(params) + && sse_callback_action_result_schema(result) => + { + Ok(()) + } + _ => Err(VmError::TypeMismatch("fn(SseEvent) -> SseCallbackAction")), } - Ok(()) } pub(crate) fn cancel_callable_stream_with_reason( diff --git a/src/vm/host.rs b/src/vm/host.rs index 25646fb9..f5cbcf8e 100644 --- a/src/vm/host.rs +++ b/src/vm/host.rs @@ -3267,11 +3267,11 @@ impl Vm { match self.poll_waiting_host_op(&mut cx) { Poll::Ready(result) => return result, Poll::Pending => { - #[cfg(not(target_arch = "wasm32"))] + #[cfg(not(target_family = "wasm"))] { std::thread::sleep(std::time::Duration::from_millis(1)); } - #[cfg(target_arch = "wasm32")] + #[cfg(target_family = "wasm")] { return Err(VmError::HostError( "blocking host-op wait is unsupported on wasm32 runtime".to_string(), diff --git a/src/vm/tests.rs b/src/vm/tests.rs index ac0f35eb..87256fc1 100644 --- a/src/vm/tests.rs +++ b/src/vm/tests.rs @@ -72,74 +72,6 @@ fn legacy_and_marker_only_schema_absence_share_cache_identity() { ); } -#[cfg(not(target_arch = "wasm32"))] -#[test] -fn builtin_pending_completion_uses_declared_return_type() { - struct PendingBridge; - - impl HostAsyncBridge for PendingBridge { - fn submit_op(&mut self, _op_id: HostOpId, _future: HostFuture) -> VmResult<()> { - Ok(()) - } - - fn poll_op( - &mut self, - _op_id: HostOpId, - _cx: &mut Context<'_>, - ) -> Poll> { - Poll::Pending - } - - fn poll_submitted_op( - &mut self, - _op_id: HostOpId, - _cx: &mut Context<'_>, - ) -> Poll> { - Poll::Pending - } - - fn cleanup_op(&mut self, _op_id: HostOpId, _terminal: HostAsyncOpTerminal) -> VmResult<()> { - Ok(()) - } - } - - let builtin = BuiltinFunction::from_namespaced_name("io::exists") - .expect("io::exists builtin should be available"); - let mut bytecode = BytecodeBuilder::new(); - bytecode.ldc(0); - bytecode.call(builtin.call_index(), 1); - bytecode.ret(); - - let mut vm = Vm::new(Program::new(vec![Value::string(".")], bytecode.finish())); - vm.set_async_bridge(Box::new(PendingBridge)) - .expect("pending bridge should install"); - let VmStatus::Waiting(op_id) = vm.run().expect("builtin should enter pending") else { - panic!("io::exists should suspend"); - }; - let error = vm - .complete_host_op(op_id, CallReturn::one(Value::string("wrong"))) - .expect_err("pending completion must reject the wrong type"); - assert!(matches!(error, VmError::TypeMismatch("bool")), "{error:?}"); - - let mut bytecode = BytecodeBuilder::new(); - bytecode.ldc(0); - bytecode.call(builtin.call_index(), 1); - bytecode.ret(); - let mut vm = Vm::new(Program::new(vec![Value::string(".")], bytecode.finish())); - vm.set_async_bridge(Box::new(PendingBridge)) - .expect("pending bridge should install"); - let VmStatus::Waiting(op_id) = vm.run().expect("builtin should enter pending") else { - panic!("io::exists should suspend"); - }; - vm.complete_host_op(op_id, CallReturn::one(Value::Bool(true))) - .expect("matching pending completion should be accepted"); - assert_eq!( - vm.resume().expect("resume after completion"), - VmStatus::Halted - ); - assert_eq!(vm.stack(), &[Value::Bool(true)]); -} - #[test] fn pending_completion_validates_scalar_resource_and_collection_schemas() { let make_schema = |name: &str, return_type: HostTypeSchema| { diff --git a/tests/builtins/io_async_tests.rs b/tests/builtins/io_async_tests.rs index 9d5f3f4e..ff1c6710 100644 --- a/tests/builtins/io_async_tests.rs +++ b/tests/builtins/io_async_tests.rs @@ -1,6 +1,10 @@ +use std::task::{Context, Poll}; use std::time::{SystemTime, UNIX_EPOCH}; -use vm::{Value, Vm, VmError, VmStatus, compile_source}; +use vm::{ + BuiltinFunction, BytecodeBuilder, CallReturn, HostAsyncBridge, HostFuture, HostOpId, Program, + Value, Vm, VmError, VmResult, VmStatus, compile_source, +}; use super::vm_reset::reset_for_reuse_to_ready; @@ -23,6 +27,61 @@ fn run_source(source: &str) -> Result, VmError> { } } +#[test] +fn builtin_pending_completion_uses_declared_return_type() { + struct PendingBridge; + + impl HostAsyncBridge for PendingBridge { + fn submit_op(&mut self, _op_id: HostOpId, _future: HostFuture) -> VmResult<()> { + Ok(()) + } + + fn poll_op( + &mut self, + _op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Pending + } + } + + let builtin = BuiltinFunction::from_namespaced_name("io::exists") + .expect("io::exists builtin should be available"); + let mut bytecode = BytecodeBuilder::new(); + bytecode.ldc(0); + bytecode.call(builtin.call_index(), 1); + bytecode.ret(); + + let mut vm = Vm::new(Program::new(vec![Value::string(".")], bytecode.finish())); + vm.set_async_bridge(Box::new(PendingBridge)) + .expect("pending bridge should install"); + let VmStatus::Waiting(op_id) = vm.run().expect("builtin should enter pending") else { + panic!("io::exists should suspend"); + }; + let error = vm + .complete_host_op(op_id, CallReturn::one(Value::string("wrong"))) + .expect_err("pending completion must reject the wrong type"); + assert!(matches!(error, VmError::TypeMismatch("bool")), "{error:?}"); + + let mut bytecode = BytecodeBuilder::new(); + bytecode.ldc(0); + bytecode.call(builtin.call_index(), 1); + bytecode.ret(); + let mut vm = Vm::new(Program::new(vec![Value::string(".")], bytecode.finish())); + vm.set_async_bridge(Box::new(PendingBridge)) + .expect("pending bridge should install"); + let VmStatus::Waiting(op_id) = vm.run().expect("builtin should enter pending") else { + panic!("io::exists should suspend"); + }; + vm.complete_host_op(op_id, CallReturn::one(Value::Bool(true))) + .expect("matching pending completion should be accepted"); + assert_eq!( + vm.resume().expect("resume after completion"), + VmStatus::Halted + ); + assert_eq!(vm.stack(), &[Value::Bool(true)]); +} + #[test] fn async_io_round_trips_file_operations_through_host_driver() { let nonce = SystemTime::now() @@ -96,46 +155,6 @@ fn async_io_popen_reads_through_tokio_process_pipe() { assert_eq!(stack.last(), Some(&Value::string("async-process"))); } -#[test] -fn io_implementations_use_only_generic_async_and_inline_sync_lifecycles() { - let async_source = include_str!("../../src/builtins/runtime/io/async_io.rs"); - let blocking_source = include_str!("../../src/builtins/runtime/io/blocking.rs"); - - for forbidden in [ - "AtomicBool", - "AtomicU32", - "std::thread", - "thread::Builder", - "JoinHandle", - "runtime::Builder", - "spawn_blocking", - "submit_host_future", - "HostAsyncBridge", - "HostOperation", - "IoOperationLease", - "active_operations", - "close_waker", - "close_scheduled", - "close_future", - "owner_alive", - "process_id:", - "try_lock()", - "wake_by_ref()", - "OperationSpec", - "schedule_io_task", - "worker_done", - ] { - assert!( - !async_source.contains(forbidden), - "async IO must not contain `{forbidden}` lifecycle machinery" - ); - assert!( - !blocking_source.contains(forbidden), - "blocking IO must be synchronous inline code without `{forbidden}`" - ); - } -} - #[cfg(unix)] struct ProcessGroupGuard { parent: Option, diff --git a/tests/builtins_tests.rs b/tests/builtins_tests.rs index f605eb63..9e29fc98 100644 --- a/tests/builtins_tests.rs +++ b/tests/builtins_tests.rs @@ -11,7 +11,7 @@ mod async_test_bridge; #[path = "builtins/io_builtin_edge_tests.rs"] mod io_builtin_edge_tests; -#[cfg(all(not(feature = "async"), not(target_arch = "wasm32")))] +#[cfg(all(not(feature = "async"), not(target_family = "wasm")))] #[path = "builtins/io_scope_lifecycle_tests.rs"] mod io_scope_lifecycle_tests; @@ -19,7 +19,7 @@ mod io_scope_lifecycle_tests; #[path = "builtins/io_async_tests.rs"] mod io_async_tests; -#[cfg(feature = "sqlite")] +#[cfg(all(feature = "sqlite", not(target_family = "wasm")))] #[path = "builtins/sqlite_scope_lifecycle_tests.rs"] mod sqlite_scope_lifecycle_tests; diff --git a/tests/http_async_arch_tests.rs b/tests/http_async_arch_tests.rs deleted file mode 100644 index 974d7c30..00000000 --- a/tests/http_async_arch_tests.rs +++ /dev/null @@ -1,84 +0,0 @@ -#![cfg(all(feature = "http-client", not(target_family = "wasm")))] - -use std::fs; -use std::path::PathBuf; - -fn source(path: &str) -> String { - fs::read_to_string(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(path)) - .unwrap_or_else(|error| panic!("failed to read {path}: {error}")) -} - -#[test] -fn http_hosts_are_macro_owned_async_functions_without_private_drivers() { - let module = source("src/builtins/runtime/http/mod.rs"); - let request = source("src/builtins/runtime/http/request.rs"); - let sse = source("src/builtins/runtime/http/sse.rs"); - let cargo = source("Cargo.toml"); - - assert!( - module.contains("async fn builtin_http_client_request"), - "the buffered host implementation must be an async macro function" - ); - assert!( - sse.contains("async fn builtin_http_client_sse"), - "the SSE host implementation must be an async macro function" - ); - assert!( - request.contains("hyper_util::client::legacy::Client"), - "HTTP transport and pooling must be owned by hyper-util" - ); - assert!( - module.contains("client: request::HttpClient"), - "the cloneable Hyper client must live in per-VM HTTP module state" - ); - assert!( - cargo.contains("\"client-legacy\"") && cargo.contains("\"http1\""), - "the HTTP feature must enable Hyper's maintained pooled client" - ); - - for (path, text) in [ - ("src/builtins/runtime/http/mod.rs", module.as_str()), - ("src/builtins/runtime/http/request.rs", request.as_str()), - ("src/builtins/runtime/http/sse.rs", sse.as_str()), - ] { - for forbidden in [ - "submit_host_future", - "HostAsyncBridge", - "std::thread", - "JoinHandle", - "tokio::runtime", - "runtime_block_on", - "HostOperation", - "HostResource", - "AtomicWaker", - ] { - assert!( - !text.contains(forbidden), - "{path} must not contain HTTP/SSE-owned async plumbing: {forbidden}" - ); - } - } - - for forbidden in [ - "HttpRequestResource", - "HttpResponseResource", - "SseStreamResource", - "SseScopeOperation", - "BufferedRequestShared", - "SseShared", - ] { - assert!( - !module.contains(forbidden) && !request.contains(forbidden) && !sse.contains(forbidden), - "transient HTTP/SSE operation or resource state remains: {forbidden}" - ); - } - - assert!( - sse.contains("impl HostStreamDriver for SseStreamDriver"), - "SSE may retain only the generic callable-stream continuation driver" - ); - assert!( - sse.contains("submit_callable_stream"), - "SSE callback re-entry must use the generic callable-stream continuation" - ); -} diff --git a/tests/pr24_resource_tests.rs b/tests/pr24_resource_tests.rs index 65d4c980..7527fc7a 100644 --- a/tests/pr24_resource_tests.rs +++ b/tests/pr24_resource_tests.rs @@ -97,18 +97,23 @@ fn standard_catalog_contains_exact_resource_close_schemas() { ); assert_eq!(io_close.return_type, HostTypeSchema::Bool); - let sqlite_close = catalog - .function("sqlite::close") - .expect("standard catalog must contain sqlite::close"); - assert_eq!( - sqlite_close.params, - vec![HostParamSchema::with_passing( - "connection", - HostTypeSchema::Resource(key("sqlite.connection")), - HostParamPassing::TakeOwned, - )] - ); - assert_eq!(sqlite_close.return_type, HostTypeSchema::Null); + #[cfg(all(feature = "sqlite", not(target_family = "wasm")))] + { + let sqlite_close = catalog + .function("sqlite::close") + .expect("standard catalog must contain sqlite::close"); + assert_eq!( + sqlite_close.params, + vec![HostParamSchema::with_passing( + "connection", + HostTypeSchema::Resource(key("sqlite.connection")), + HostParamPassing::TakeOwned, + )] + ); + assert_eq!(sqlite_close.return_type, HostTypeSchema::Null); + } + #[cfg(not(all(feature = "sqlite", not(target_family = "wasm"))))] + assert!(catalog.function("sqlite::close").is_none()); } #[test] @@ -130,28 +135,38 @@ fn standard_catalog_resolves_open_then_close_with_nominal_ownership() { assert_eq!(io_close.passing, vec![HostParamPassing::TakeOwned]); assert_eq!(io_close.return_type, TypeSchema::Bool); - let connection = resolver - .resolve("sqlite::open", &[TypeSchema::Unknown]) - .expect("standard sqlite::open resolves"); - assert_eq!( - connection.return_type, - TypeSchema::Resource(key("sqlite.connection")), - "sqlite::open must produce the SQLite resource key" + #[cfg(all(feature = "sqlite", not(target_family = "wasm")))] + { + let connection = resolver + .resolve("sqlite::open", &[TypeSchema::Unknown]) + .expect("standard sqlite::open resolves"); + assert_eq!( + connection.return_type, + TypeSchema::Resource(key("sqlite.connection")), + "sqlite::open must produce the SQLite resource key" + ); + let sqlite_close = resolver + .resolve( + "sqlite::close", + &[TypeSchema::Resource(key("sqlite.connection"))], + ) + .expect("standard sqlite::close resolves"); + assert_eq!(sqlite_close.passing, vec![HostParamPassing::TakeOwned]); + assert_eq!(sqlite_close.return_type, TypeSchema::Null); + } + #[cfg(not(all(feature = "sqlite", not(target_family = "wasm"))))] + assert!( + resolver + .resolve("sqlite::open", &[TypeSchema::Unknown]) + .is_err() ); - let sqlite_close = resolver - .resolve( - "sqlite::close", - &[TypeSchema::Resource(key("sqlite.connection"))], - ) - .expect("standard sqlite::close resolves"); - assert_eq!(sqlite_close.passing, vec![HostParamPassing::TakeOwned]); - assert_eq!(sqlite_close.return_type, TypeSchema::Null); } #[test] fn default_source_compilation_accepts_open_use_close_ownership_flow() { let options = CompileSourceFileOptions::new().with_host_api_catalog(vm::standard_host_catalog()); + #[cfg(all(feature = "sqlite", not(target_family = "wasm")))] let source = r#" use io; use sqlite; @@ -160,6 +175,12 @@ fn default_source_compilation_accepts_open_use_close_ownership_flow() { let connection = sqlite::open({}); sqlite::close(connection); "#; + #[cfg(not(all(feature = "sqlite", not(target_family = "wasm"))))] + let source = r#" + use io; + let file = io::open("Cargo.toml", "r"); + io::close(file); + "#; compile_source_with_flavor_and_options(source, vm::SourceFlavor::RustScript, options) .expect("standard catalog must compile normal open/use/close ownership flow"); } diff --git a/tests/sqlite_async_host_arch_tests.rs b/tests/sqlite_async_host_arch_tests.rs deleted file mode 100644 index 03b109f1..00000000 --- a/tests/sqlite_async_host_arch_tests.rs +++ /dev/null @@ -1,63 +0,0 @@ -#![cfg(all(feature = "sqlite", not(target_arch = "wasm32")))] -//! Architecture guard for the macro-owned async SQLite host adapter. - -const SQLITE_SOURCE: &str = include_str!("../src/builtins/runtime/sqlite.rs"); - -#[test] -fn sqlite_host_functions_are_macro_owned_async_functions() { - for function in ["open", "execute", "query", "transaction", "close"] { - let signature = format!("async fn builtin_sqlite_{function}_impl"); - assert!( - SQLITE_SOURCE.contains(&signature), - "sqlite::{function} must be an ordinary async #[pd_host_function]" - ); - } - - assert!( - SQLITE_SOURCE.contains("CaptureAsyncHostContext"), - "SQLite calls must capture owned VM context before async submission" - ); - assert!( - SQLITE_SOURCE.contains("tokio_rusqlite::Connection"), - "the SQLite resource must use the maintained Tokio-facing adapter" - ); - - assert_eq!( - SQLITE_SOURCE.matches("HostFutureOutput::complete").count(), - 2, - "only open insertion and close removal may require terminal VM completion" - ); -} - -#[test] -fn sqlite_host_owns_no_threads_or_custom_operation_driver() { - let implementation = SQLITE_SOURCE - .split_once("\n#[cfg(test)]\nmod tests") - .map_or(SQLITE_SOURCE, |(implementation, _tests)| implementation); - for forbidden in [ - "std::thread", - "thread::", - "JoinHandle", - "std::sync::mpsc", - "crossbeam_channel", - "AtomicWaker", - "RawWaker", - "HostOperation", - "OperationSpec", - "OperationOutcome", - "OperationCancelReason", - "SqliteWorker", - "SqliteOpDriver", - "SqliteOpShared", - "schedule_operation", - "register_scoped_operation_completion", - "completion mailbox", - "close_waker", - "quiescence_waker", - ] { - assert!( - !implementation.contains(forbidden), - "SQLite host source must not contain custom scheduling token `{forbidden}`" - ); - } -} diff --git a/tests/sqlite_named_struct_tests.rs b/tests/sqlite_named_struct_tests.rs index 8f9091a1..33e77296 100644 --- a/tests/sqlite_named_struct_tests.rs +++ b/tests/sqlite_named_struct_tests.rs @@ -1,4 +1,4 @@ -#![cfg(feature = "sqlite")] +#![cfg(all(feature = "sqlite", not(target_family = "wasm")))] //! SQLite fixed-shape host maps are named structs at the catalog/compiler //! boundary. Runtime values remain maps; positional params, row cells, and //! transaction results use named wrappers. diff --git a/tests/standard_host_descriptor_arch_tests.rs b/tests/standard_host_descriptor_arch_tests.rs index 3c23d575..0aeb2178 100644 --- a/tests/standard_host_descriptor_arch_tests.rs +++ b/tests/standard_host_descriptor_arch_tests.rs @@ -1,1015 +1,31 @@ -//! Architecture guard for the standard host descriptor surface. -//! -//! These tests prove, from the production sources and the runtime composition, -//! that: -//! -//! * every standard `#[pd_host_function]` has **exactly one** descriptor owner -//! (a module ownership list), and every ownership list belongs to a module -//! composed in this build by [`vm::standard_host_modules`]; -//! * no migrated module keeps a hand-written parallel catalog, adapter table, -//! or registry glue; -//! * the standard guest catalog is derived from the module descriptors, and its -//! fingerprint is byte-for-byte the published one; -//! * feature gates deterministically include or exclude whole modules; -//! * every resource key in the derived catalog has a declaration, and the typed -//! named-struct set is preserved. +//! Runtime contract tests for standard host descriptor composition. -use std::collections::{BTreeMap, BTreeSet}; -use std::fs; -use std::path::{Path, PathBuf}; +use std::collections::BTreeSet; -/// Whether this build composes the HTTP host module (and therefore its guest -/// surface). const HTTP_SURFACE_ENABLED: bool = cfg!(all(feature = "http-client", not(target_family = "wasm"))); +const SQLITE_SURFACE_ENABLED: bool = cfg!(all(feature = "sqlite", not(target_family = "wasm"))); -/// Fingerprint of the published standard host catalog **with** the HTTP -/// surface. -/// -/// The standard catalog is the merge of the current composed module surfaces, -/// so the golden depends on the composed set. The HTTP surface intentionally -/// omits transport-only resources; the default build (no `http-client`) -/// composes one module fewer and must reproduce -/// [`STANDARD_CATALOG_FINGERPRINT_NO_HTTP`]. -const STANDARD_CATALOG_FINGERPRINT: &str = "8aad996e0b2f010b"; -/// Fingerprint of the published standard host catalog **without** the HTTP -/// surface: the `--workspace` default build and every wasm build. -const STANDARD_CATALOG_FINGERPRINT_NO_HTTP: &str = "8afd8a69c58f02bd"; +const STANDARD_CATALOG_FINGERPRINT_HTTP_SQLITE: &str = "8aad996e0b2f010b"; +const STANDARD_CATALOG_FINGERPRINT_SQLITE: &str = "8afd8a69c58f02bd"; +const STANDARD_CATALOG_FINGERPRINT_HTTP: &str = "7b558c161322fc76"; +const STANDARD_CATALOG_FINGERPRINT_BASE: &str = "8254b00d727494b4"; const IO_CATALOG_FINGERPRINT: &str = "a16730bd11bf5e10"; +#[cfg(all(feature = "sqlite", not(target_family = "wasm")))] const SQLITE_CATALOG_FINGERPRINT: &str = "dce61460a46c421a"; const JIT_CATALOG_FINGERPRINT: &str = "ae81318e8a018681"; const TIMER_CATALOG_FINGERPRINT: &str = "7ecc0517cea3570b"; #[cfg(all(feature = "http-client", not(target_family = "wasm")))] const HTTP_CATALOG_FINGERPRINT: &str = "329e09ffbdd82a6f"; -/// The standard catalog fingerprint this build must reproduce exactly. fn standard_catalog_fingerprint() -> &'static str { - if HTTP_SURFACE_ENABLED { - STANDARD_CATALOG_FINGERPRINT - } else { - STANDARD_CATALOG_FINGERPRINT_NO_HTTP + match (HTTP_SURFACE_ENABLED, SQLITE_SURFACE_ENABLED) { + (true, true) => STANDARD_CATALOG_FINGERPRINT_HTTP_SQLITE, + (false, true) => STANDARD_CATALOG_FINGERPRINT_SQLITE, + (true, false) => STANDARD_CATALOG_FINGERPRINT_HTTP, + (false, false) => STANDARD_CATALOG_FINGERPRINT_BASE, } } -/// Resource keys the standard catalog always publishes. -const STANDARD_RESOURCE_KEYS: &[&str] = &["io.file", "sqlite.connection"]; - -/// HTTP uses macro-owned futures and does not publish transient resources. -const HTTP_RESOURCE_KEYS: &[&str] = &[]; - -/// Named structs the standard catalog always declares. -const STANDARD_NAMED_STRUCTS: &[&str] = &[ - "JitConfig", - "SqliteValue", - "SqliteQueryResult", - "SqliteTransactionResult", -]; - -/// Named structs the HTTP module declares when it is composed. -const HTTP_NAMED_STRUCTS: &[&str] = &[ - "HttpRequest", - "HttpResponse", - "SseEvent", - "SseSummary", - "SseCallbackAction", -]; - -/// Source files that must contain no hand-written catalog or registry glue for -/// their migrated host module. -const MIGRATED_MODULE_FILES: &[&str] = &[ - "src/builtins/runtime/http/mod.rs", - "src/builtins/runtime/http/sse.rs", - "src/builtins/runtime/io/mod.rs", - "src/builtins/runtime/io/async_io.rs", - "src/builtins/runtime/io/blocking.rs", - "src/builtins/runtime/io_wasm.rs", - "src/builtins/runtime/jit.rs", - "src/builtins/runtime/sqlite.rs", - "src/builtins/runtime/timer.rs", -]; - -fn manifest_root() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) -} - -fn relative(path: &Path) -> String { - path.strip_prefix(manifest_root()) - .expect("source under manifest root") - .to_string_lossy() - .replace('\\', "/") -} - -fn collect_rs(dir: &Path, out: &mut Vec) { - for entry in fs::read_dir(dir).unwrap_or_else(|e| panic!("read {}: {e}", dir.display())) { - let path = entry.expect("readable entry").path(); - if fs::metadata(&path).expect("source metadata").is_dir() { - collect_rs(&path, out); - } else if path.extension().is_some_and(|ext| ext == "rs") { - out.push(path); - } - } -} - -fn production_sources() -> Vec { - let mut files = Vec::new(); - collect_rs(&manifest_root().join("src/builtins"), &mut files); - files.sort(); - files -} - -/// Replaces one literal or comment with whitespace while preserving both line -/// numbers and byte offsets, so the raw text stays addressable. -fn blank_literal(out: &mut String, source: &str, start: usize, end: usize) { - for byte in &source.as_bytes()[start..end] { - out.push(if *byte == b'\n' { '\n' } else { ' ' }); - } -} - -/// Returns the exclusive end of a character literal, if one starts at `start`. -/// -/// A lifetime (`'static`, `'_, `'a`) is deliberately not a literal: treating it -/// as one would make the scan swallow every byte up to the next apostrophe. -fn char_literal_end(bytes: &[u8], start: usize) -> Option { - let mut cursor = start + 1; - loop { - match bytes.get(cursor)? { - b'\\' => cursor = cursor.checked_add(2)?, - b'\'' => return Some(cursor + 1), - b'\n' => return None, - _ => cursor += 1, - } - if cursor - start > 8 { - return None; - } - } -} - -fn quoted_literal_end(bytes: &[u8], start: usize, quote: u8) -> usize { - let mut cursor = start + 1; - while cursor < bytes.len() { - match bytes[cursor] { - b'\\' => cursor = cursor.saturating_add(2), - value if value == quote => return cursor + 1, - _ => cursor += 1, - } - } - bytes.len() -} - -/// Removes comments so the guard inspects Rust tokens rather than prose. -fn strip_comments(source: &str) -> String { - let bytes = source.as_bytes(); - let mut out = String::with_capacity(source.len()); - let mut cursor = 0usize; - while cursor < bytes.len() { - match bytes[cursor] { - b'"' => { - let end = quoted_literal_end(bytes, cursor, b'"'); - blank_literal(&mut out, source, cursor, end); - cursor = end; - } - b'/' if bytes.get(cursor + 1) == Some(&b'/') => { - let start = cursor; - while cursor < bytes.len() && bytes[cursor] != b'\n' { - cursor += 1; - } - blank_literal(&mut out, source, start, cursor); - } - b'/' if bytes.get(cursor + 1) == Some(&b'*') => { - let start = cursor; - cursor += 2; - while cursor < bytes.len() && bytes.get(cursor..cursor + 2) != Some(b"*/") { - cursor += 1; - } - cursor = (cursor + 2).min(bytes.len()); - blank_literal(&mut out, source, start, cursor); - } - b'\'' => match char_literal_end(bytes, cursor) { - // A character literal. - Some(end) => { - blank_literal(&mut out, source, cursor, end); - cursor = end; - } - // A lifetime: keep the quote so `'static` stays visible. - None => { - out.push('\''); - cursor += 1; - } - }, - _ => { - let character = source[cursor..] - .chars() - .next() - .expect("cursor must point inside source"); - out.push(character); - cursor += character.len_utf8(); - } - } - } - out -} - -/// Production code of one source file: comments removed and unit-test -/// fixtures dropped. -fn production_code(path: &Path) -> String { - strip_comments(&production_raw(path)) -} - -/// Production source of one file with its unit-test items blanked. -/// -/// Attribute parsing needs the raw text: a `#[pd_host_function(name = ...)]` -/// value *is* a string literal, so the comment/literal-stripped form cannot be -/// used to read it. -/// -/// Test items are replaced by whitespace instead of being truncated, so the -/// production text keeps its byte offsets and line numbers: diagnostics are -/// reported as `file:line`, and the comment stripper asserts the stripped form -/// has the same length. Truncating at the first `#[cfg(test)]` marker instead -/// would hide every production declaration that follows a test-only item, -/// which is exactly the shape `src/builtins/runtime/http/sse.rs` has. -fn production_raw(path: &Path) -> String { - let raw = fs::read_to_string(path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())); - strip_test_items(&raw) -} - -/// The marker that introduces a unit-test item. -const CFG_TEST_ATTRIBUTE: &str = "#[cfg(test)]"; - -/// Blanks every `#[cfg(test)]` item in `source` while preserving byte offsets -/// and line numbers. -fn strip_test_items(source: &str) -> String { - let mut out = String::with_capacity(source.len()); - let mut cursor = 0usize; - while let Some(offset) = source[cursor..].find(CFG_TEST_ATTRIBUTE) { - let start = cursor + offset; - out.push_str(&source[cursor..start]); - let end = test_item_end(source, start); - blank_literal(&mut out, source, start, end); - cursor = end; - } - out.push_str(&source[cursor..]); - out -} - -/// Returns the exclusive end of the item or statement introduced by the -/// `#[cfg(test)]` attribute starting at `attr_start`. -/// -/// This is a bounded lexer over the raw bytes rather than a full Rust parser: -/// it tracks bracket depth, skips string literals and comments, and stops at -/// the first top-level statement terminator (`;`) or at the closing brace that -/// returns to the enclosing block. That covers every shape the marker takes in -/// this crate: a constant, a free function, a method inside an `impl`, a -/// `static`, an `if` statement inside a function body, and a whole -/// `mod tests { ... }`. -fn test_item_end(source: &str, attr_start: usize) -> usize { - let bytes = source.as_bytes(); - let mut cursor = attr_start; - let mut depth = 0usize; - while cursor < bytes.len() { - if let Some(end) = raw_string_end(bytes, cursor) { - cursor = end; - continue; - } - match bytes[cursor] { - b'"' => cursor = quoted_literal_end(bytes, cursor, b'"'), - b'/' if bytes.get(cursor + 1) == Some(&b'/') => { - while cursor < bytes.len() && bytes[cursor] != b'\n' { - cursor += 1; - } - } - b'/' if bytes.get(cursor + 1) == Some(&b'*') => { - cursor += 2; - while cursor < bytes.len() && bytes.get(cursor..cursor + 2) != Some(b"*/") { - cursor += 1; - } - cursor = (cursor + 2).min(bytes.len()); - } - b'{' | b'(' | b'[' => { - depth += 1; - cursor += 1; - } - b'}' => { - if depth == 0 { - // The attributed item sits in a block that ends here. - return cursor + 1; - } - depth -= 1; - cursor += 1; - if depth == 0 { - // The attributed item closed with its own group. - return cursor; - } - } - b')' | b']' => { - depth = depth.saturating_sub(1); - cursor += 1; - if depth == 0 && bytes.get(cursor) == Some(&b';') { - return cursor + 1; - } - } - b';' if depth == 0 => return cursor + 1, - _ => cursor += 1, - } - } - bytes.len() -} - -/// Returns the exclusive end of a raw string/byte-string literal, if one starts -/// at `start`. -/// -/// Raw strings carry unbalanced `{`/`}` and quotes verbatim (a test fixture -/// shell script is one), so the depth scan must skip them as a unit. -fn raw_string_end(bytes: &[u8], start: usize) -> Option { - let mut cursor = start; - if bytes.get(cursor) == Some(&b'b') { - if bytes.get(cursor + 1) != Some(&b'r') { - return None; - } - cursor += 1; - } - if bytes.get(cursor) != Some(&b'r') { - return None; - } - cursor += 1; - let mut hashes = 0usize; - while bytes.get(cursor) == Some(&b'#') { - hashes += 1; - cursor += 1; - } - if bytes.get(cursor) != Some(&b'"') { - return None; - } - let content_start = cursor + 1; - let mut candidate = content_start; - while candidate < bytes.len() { - if bytes[candidate] == b'"' - && bytes - .get(candidate + 1..candidate + 1 + hashes) - .is_some_and(|suffix| suffix.iter().all(|byte| *byte == b'#')) - { - return Some(candidate + 1 + hashes); - } - candidate += 1; - } - None -} - -/// One standard `#[pd_host_function]` declaration found in the sources. -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] -struct StandardHostFunction { - name: String, - descriptor: String, - file: String, -} - -fn standard_host_functions() -> Vec { - let mut found = Vec::new(); - for path in production_sources() { - let code = production_code(&path); - let raw = production_raw(&path); - assert_eq!( - code.len(), - raw.len(), - "{}: comment stripping must preserve byte offsets", - relative(&path) - ); - let file = relative(&path); - let mut cursor = 0usize; - while let Some(offset) = code[cursor..].find("#[pd_host_function(") { - let attribute_start = cursor + offset; - let body_start = attribute_start + "#[pd_host_function(".len(); - let Some(body_end) = code[body_start..].find(']') else { - break; - }; - // A `name = "..."` value is a string literal, so it is read from the - // raw attribute body; the attribute may span several lines. - let attribute = &raw[attribute_start..body_start + body_end]; - let line = code[..attribute_start].lines().count(); - let name = attribute - .split("name =") - .nth(1) - .and_then(|tail| tail.trim_start().strip_prefix('"')) - .and_then(|tail| tail.split('"').next()) - .unwrap_or_else(|| panic!("{file}:{line}: unparsable host function name")) - .to_string(); - cursor = body_start + body_end; - let Some(ident) = raw[cursor..].lines().find_map(|candidate| { - let candidate = candidate.trim(); - let (_, tail) = candidate.split_once("fn ")?; - let ident: String = tail - .chars() - .take_while(|c| c.is_alphanumeric() || *c == '_') - .collect(); - (!ident.is_empty()).then_some(ident) - }) else { - panic!("{file}:{line}: no function follows the attribute"); - }; - let descriptor = match ident.strip_suffix("_impl") { - Some(prefix) => format!("{prefix}_descriptor"), - None => format!("{ident}_descriptor"), - }; - found.push(StandardHostFunction { - name, - descriptor, - file: file.clone(), - }); - } - } - found -} - -/// Descriptor factories referenced by an ownership list, keyed by the -/// declaration count. An identifier introduced twice by two files of the same -/// module (the mutually exclusive IO backends) is one owner, not two. -fn declared_ownership() -> BTreeMap> { - /// The type that introduces an ownership list. - const TYPE: &str = "HostFunctionDescriptor]"; - /// The list opener, which rustfmt may place on the following line. - const OPENER: &str = "&["; - - let mut owners: BTreeMap> = BTreeMap::new(); - for path in production_sources() { - let code = production_code(&path); - let file = relative(&path); - let mut remaining = code.as_str(); - while let Some(type_at) = remaining.find(TYPE) { - let after_type = &remaining[type_at + TYPE.len()..]; - let Some(open_at) = after_type.find(OPENER) else { - break; - }; - let body_start = type_at + TYPE.len() + open_at + OPENER.len(); - let Some(close) = remaining[body_start..].find(']') else { - break; - }; - for entry in remaining[body_start..body_start + close].split(',') { - let entry = entry.trim(); - if entry.is_empty() { - continue; - } - // An ownership list may name a descriptor through the module - // that declares it (`sse::builtin_http_client_sse_descriptor`); - // the owner is the descriptor alone. - let entry = entry.rsplit("::").next().unwrap_or(entry).trim(); - if entry.is_empty() { - continue; - } - owners - .entry(entry.to_string()) - .or_default() - .push(file.clone()); - } - remaining = &remaining[body_start + close + 1..]; - } - } - owners -} - -/// A standard host module named by a source file, with the descriptor -/// factories the file's ownership lists declare for it. -#[derive(Debug, Default, PartialEq, Eq)] -struct OwnershipSource { - /// Module names the file declares through `StandardHostModule { name: ... }`, - /// `descriptor_only_module(...)`, or `catalog_module(...)`. - modules: BTreeSet, - /// Descriptor factories the file's `&[fn() -> HostFunctionDescriptor]` - /// lists name, normalized to the descriptor's own ident. - entries: BTreeSet, -} - -/// Source files that supply the ownership list of a module declared in another -/// file, mapped to the module they belong to. -/// -/// The guard cannot derive that link from the descriptor lists alone: the -/// native SQLite adapter lives in `sqlite.rs` while the module that owns it is -/// declared in `sqlite_schema.rs`. Stating it here keeps it checkable — every -/// module named here must be composed in this build. -const SHARED_MODULE_SOURCES: &[(&str, &str)] = &[("src/builtins/runtime/sqlite.rs", "sqlite")]; - -/// Module names a source file declares as standard host modules. -fn declared_host_modules(source: &str) -> BTreeSet { - let code = strip_comments(source); - let mut names = BTreeSet::new(); - for marker in [ - "StandardHostModule {", - "descriptor_only_module(", - "catalog_module(", - ] { - let mut cursor = 0usize; - while let Some(offset) = code[cursor..].find(marker) { - let after = cursor + offset + marker.len(); - cursor = after; - // The marker is found in the comment-stripped text (so prose does - // not match), but the module name is a string literal and must be - // read from the raw source at the same offset. - if let Some(name) = first_string_literal(&source[after..]) { - names.insert(name); - } - } - } - names -} - -/// The first string literal in `source`, if one starts before any other token. -fn first_string_literal(source: &str) -> Option { - let bytes = source.as_bytes(); - let mut cursor = 0usize; - while cursor < bytes.len() { - match bytes[cursor] { - b'"' => { - let end = quoted_literal_end(bytes, cursor, b'"'); - return Some(source[cursor + 1..end.saturating_sub(1)].to_string()); - } - b' ' | b'\t' | b'\r' | b'\n' => cursor += 1, - b'/' if bytes.get(cursor + 1) == Some(&b'/') => { - while cursor < bytes.len() && bytes[cursor] != b'\n' { - cursor += 1; - } - } - _ => return None, - } - } - None -} - -/// Ownership lists that no module composed in this build owns. -/// -/// `sources` maps each file that declares an ownership list to the modules it -/// declares and the descriptors it lists, `composed` maps every composed -/// module name to the guest names it owns, `guest_names` maps a descriptor -/// factory to the name its `#[pd_host_function]` declares (absent for the -/// descriptors a module builds by hand, which stay outside this check). -fn unknown_ownership_entries( - sources: &BTreeMap, - guest_names: &BTreeMap, - composed: &BTreeMap>, - disabled_modules: &BTreeSet, - shared_sources: &[(&str, &str)], -) -> Vec { - let mut offenders = Vec::new(); - for (file, source) in sources { - let mut owners: BTreeSet = source.modules.clone(); - if owners.is_empty() { - match shared_sources.iter().find(|(path, _)| path == file) { - Some((_, module)) => { - owners.insert((*module).to_string()); - } - None => { - // A module this build's feature gates turn off keeps its - // sources in the tree; the guard cannot see the gate from - // the module name alone, so the disabled set is the - // exemption. - offenders.push(format!( - "{file} declares an ownership list but no standard host module" - )); - continue; - } - } - } - for owner in &owners { - if !composed.contains_key(owner) && !disabled_modules.contains(owner) { - offenders.push(format!( - "{file} declares an ownership list for `{owner}`, which no build composes" - )); - } - } - // Every list entry the guard can resolve to a declaration must be owned - // by the module the declaring file belongs to. - for entry in &source.entries { - let Some(name) = guest_names.get(entry) else { - continue; - }; - // A module this build's gate turns off keeps its own list intact - // and cannot be cross-checked against the composition. - if owners.iter().any(|owner| disabled_modules.contains(owner)) { - continue; - } - if owners.iter().any(|owner| { - composed - .get(owner) - .is_some_and(|owned| owned.contains(name)) - }) { - continue; - } - offenders.push(format!( - "{file} lists `{entry}` (`{name}`) for {owners:?}, which does not own it" - )); - } - } - offenders -} - -/// Focused scanner tests: the production scan must drop test items without -/// hiding the production declarations that follow them. -#[test] -fn production_scan_drops_test_items_without_hiding_later_declarations() { - let source = "\ -#[cfg(test)] -const _: () = assert!(SSE_CHANNEL_CAPACITY == 1); - -/// Production declaration below a test-only constant. -#[pd_host_function(name = \"demo::late\")] -fn builtin_demo_late() -> i64 { - 0 -} -\n"; - let stripped = strip_test_items(source); - assert_eq!( - stripped.len(), - source.len(), - "blanking a test item must preserve byte offsets" - ); - assert_eq!( - stripped.lines().count(), - source.lines().count(), - "blanking a test item must preserve line numbers" - ); - assert!( - !stripped.contains("assert!(SSE_CHANNEL_CAPACITY"), - "the test item must not survive the production scan" - ); - assert!( - stripped.contains("#[pd_host_function(name = \"demo::late\")]"), - "a production declaration after a test item must stay visible" - ); -} - -/// Every shape a `#[cfg(test)]` marker takes in this crate is dropped, and the -/// production statements around it survive. -#[test] -fn production_scan_drops_test_items_in_every_declaration_shape() { - let source = "\ -fn body() { - #[cfg(test)] - if FAIL_NEXT.swap(false, Ordering::AcqRel) { - return Err(\"injected\"); - } - work(); -} - -#[cfg(test)] -static FAIL_NEXT: AtomicBool = AtomicBool::new(false); - -impl Thing { - #[cfg(test)] - fn helper(&self) -> u32 { - 1 - } - - fn production(&self) -> u32 { - 2 - } -} - -#[cfg(test)] -use super::{fixture_a, fixture_b}; -\n"; - let stripped = strip_test_items(source); - assert_eq!(stripped.len(), source.len(), "offsets must be preserved"); - for survivor in ["fn body()", "work();", "fn production(&self)", "impl Thing"] { - assert!(stripped.contains(survivor), "`{survivor}` must survive"); - } - for dropped in ["injected", "static FAIL_NEXT", "fn helper", "fixture_a"] { - assert!( - !stripped.contains(dropped), - "test-only `{dropped}` must be dropped" - ); - } -} - -/// A raw string inside a test item may carry unbalanced braces and quotes; the -/// scan must skip it as a unit instead of reading its bytes as code. -#[test] -fn production_scan_skips_raw_strings_inside_test_items() { - let source = "\ -#[cfg(test)] -mod fixtures { - const SCRIPT: &str = r#\" - if [ -n \"$child\" ]; then { echo 'unbalanced' \"$child\" > '{}' - \"#; -} - -/// Production declaration after a raw-string fixture. -#[pd_host_function(name = \"demo::raw\")] -fn builtin_demo_raw() {} -\n"; - let stripped = strip_test_items(source); - assert_eq!(stripped.len(), source.len(), "offsets must be preserved"); - assert!( - !stripped.contains("unbalanced"), - "the raw-string fixture must be dropped" - ); - assert!( - stripped.contains("#[pd_host_function(name = \"demo::raw\")]"), - "a production declaration after a raw-string fixture must stay visible" - ); -} - -/// The discriminating regression: a test-only constant early in a file used to -/// truncate the scan, hiding the `http::client::sse` declaration that follows -/// it. Ownership must still be scanned exactly once. -#[test] -fn sse_stream_declaration_is_scanned_and_owned_exactly_once() { - let file = "src/builtins/runtime/http/sse.rs"; - let path = manifest_root().join(file); - let raw = production_raw(&path); - assert_eq!( - raw.len(), - fs::read_to_string(&path).expect("sse source").len(), - "{file}: the production scan must preserve byte offsets" - ); - assert!( - raw.contains("#[pd_host_function(name = \"http::client::sse\""), - "{file}: the production host declaration must survive the test-item scan" - ); - - let functions = standard_host_functions(); - let sse: Vec<&StandardHostFunction> = functions - .iter() - .filter(|function| function.name == "http::client::sse") - .collect(); - assert_eq!( - sse.len(), - 1, - "the SSE stream host function must be discovered exactly once: {sse:?}" - ); - assert_eq!(sse[0].file, file); - assert_eq!(sse[0].descriptor, "builtin_http_client_sse_descriptor"); - - let owners = declared_ownership(); - let claimants = owners - .get("builtin_http_client_sse_descriptor") - .expect("the SSE stream descriptor must have an ownership list"); - assert_eq!( - claimants, - &vec!["src/builtins/runtime/http/mod.rs".to_string()], - "SSE ownership must be declared exactly once" - ); -} - -/// Every file that declares an ownership list, with the modules it declares -/// and the descriptors it lists. -fn ownership_sources() -> BTreeMap { - let mut sources: BTreeMap = BTreeMap::new(); - for (descriptor, files) in declared_ownership() { - for file in files { - sources - .entry(file) - .or_default() - .entries - .insert(descriptor.clone()); - } - } - for file in sources.keys().cloned().collect::>() { - let path = manifest_root().join(&file); - let source = fs::read_to_string(&path) - .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); - // The module declarations are read from the whole file; the entries - // come from the production scan above. - let modules = declared_host_modules(&source); - sources.entry(file).or_default().modules = modules; - } - sources -} - -/// Every guest name each module composed in this build owns. -fn composed_module_ownership() -> BTreeMap> { - vm::standard_host_modules() - .iter() - .map(|module| { - let owned = module - .owned_descriptors() - .into_iter() - .map(|descriptor| descriptor.schema.name) - .collect(); - (module.name.to_string(), owned) - }) - .collect() -} - -/// Module names this build's feature gates turn off. -fn disabled_standard_modules() -> BTreeSet { - let mut disabled = BTreeSet::new(); - if !HTTP_SURFACE_ENABLED { - disabled.insert("http".to_string()); - } - disabled -} - -/// Every ownership list must belong to a module `standard_host_modules()` -/// composes in this build. -#[test] -fn every_ownership_list_belongs_to_a_composed_standard_module() { - let sources = ownership_sources(); - assert!( - sources.len() > 10, - "the standard ownership lists must be discovered from the sources, found {}", - sources.len() - ); - let guest_names: BTreeMap = standard_host_functions() - .into_iter() - .map(|function| (function.descriptor, function.name)) - .collect(); - let composed = composed_module_ownership(); - assert!( - composed.values().map(BTreeSet::len).sum::() > 100, - "the composed standard modules must own the discovered functions" - ); - - let offenders = unknown_ownership_entries( - &sources, - &guest_names, - &composed, - &disabled_standard_modules(), - SHARED_MODULE_SOURCES, - ); - assert!( - offenders.is_empty(), - "every ownership list must belong to a module composed in this build:\n{}", - offenders.join("\n") - ); -} - -/// The check above is only evidence if it can fail. -#[test] -fn ownership_check_reports_lists_no_composed_module_owns() { - let composed = BTreeMap::from([ - ("io".to_string(), BTreeSet::from(["io::open".to_string()])), - ( - "sqlite".to_string(), - BTreeSet::from(["sqlite::open".to_string()]), - ), - ]); - let disabled = BTreeSet::new(); - - // A file that declares no module and no shared-module entry is an orphan. - let orphan = BTreeMap::from([( - "src/builtins/runtime/orphan.rs".to_string(), - OwnershipSource { - modules: BTreeSet::new(), - entries: BTreeSet::new(), - }, - )]); - let offenders = unknown_ownership_entries( - &orphan, - &BTreeMap::new(), - &composed, - &disabled, - SHARED_MODULE_SOURCES, - ); - assert_eq!(offenders.len(), 1, "{offenders:?}"); - - // A module the composition does not include is reported. - let uncomposed = BTreeMap::from([( - "src/builtins/runtime/ghost.rs".to_string(), - OwnershipSource { - modules: BTreeSet::from(["ghost".to_string()]), - entries: BTreeSet::new(), - }, - )]); - let offenders = unknown_ownership_entries( - &uncomposed, - &BTreeMap::new(), - &composed, - &disabled, - SHARED_MODULE_SOURCES, - ); - assert_eq!(offenders.len(), 1, "{offenders:?}"); - assert!(offenders[0].contains("ghost"), "{offenders:?}"); - - // A list entry the guard can resolve must be owned by the same module. - let misfiled = BTreeMap::from([( - "src/builtins/runtime/jit.rs".to_string(), - OwnershipSource { - modules: BTreeSet::from(["jit".to_string()]), - entries: BTreeSet::from(["builtin_io_open_descriptor".to_string()]), - }, - )]); - let names = BTreeMap::from([( - "builtin_io_open_descriptor".to_string(), - "io::open".to_string(), - )]); - let mut composed = composed.clone(); - composed.insert( - "jit".to_string(), - BTreeSet::from(["jit::get_config".to_string()]), - ); - let offenders = unknown_ownership_entries( - &misfiled, - &names, - &composed, - &disabled, - SHARED_MODULE_SOURCES, - ); - assert_eq!(offenders.len(), 1, "{offenders:?}"); - assert!(offenders[0].contains("io::open"), "{offenders:?}"); - - // A feature-disabled module and a shared-module source are both accepted. - let shared = BTreeMap::from([( - "src/builtins/runtime/sqlite.rs".to_string(), - OwnershipSource { - modules: BTreeSet::new(), - entries: BTreeSet::new(), - }, - )]); - assert!( - unknown_ownership_entries( - &shared, - &BTreeMap::new(), - &composed, - &disabled, - SHARED_MODULE_SOURCES, - ) - .is_empty() - ); - let gated = BTreeMap::from([( - "src/builtins/runtime/http/mod.rs".to_string(), - OwnershipSource { - modules: BTreeSet::from(["http".to_string()]), - entries: BTreeSet::new(), - }, - )]); - assert!( - unknown_ownership_entries( - &gated, - &BTreeMap::new(), - &composed, - &BTreeSet::from(["http".to_string()]), - SHARED_MODULE_SOURCES, - ) - .is_empty() - ); -} - -#[test] -fn every_standard_host_function_has_exactly_one_descriptor_owner() { - let functions = standard_host_functions(); - assert!( - functions.len() > 100, - "the standard host inventory must be discovered from the sources, found {}", - functions.len() - ); - let owners = declared_ownership(); - assert!( - !owners.is_empty(), - "the standard host modules must declare explicit ownership lists" - ); - - let mut offenders = Vec::new(); - for function in &functions { - match owners.get(&function.descriptor) { - None => offenders.push(format!( - "{}: `{}` ({} in {}) has no descriptor owner", - function.file, function.name, function.descriptor, function.file - )), - Some(claimants) => { - let unique: BTreeSet<&String> = claimants.iter().collect(); - if unique.len() != 1 { - offenders.push(format!( - "{}: `{}` is owned by more than one module: {claimants:?}", - function.file, function.name - )); - } - } - } - } - assert!( - offenders.is_empty(), - "every standard #[pd_host_function] must have exactly one descriptor owner:\n{}", - offenders.join("\n") - ); -} - -#[test] -fn migrated_modules_keep_no_hand_written_catalog_or_registry_glue() { - let forbidden = [ - "HostApiBuilder::new()", - "builder.resource(", - "builder.function(", - "builder.named_struct(", - "register_exact_static(", - "register_exact_owned(", - "ADAPTER_CONTRACTS", - ]; - let mut offenders = Vec::new(); - for file in MIGRATED_MODULE_FILES { - let path = manifest_root().join(file); - if !path.is_file() { - continue; - } - let code = production_code(&path); - for symbol in forbidden { - if code.contains(symbol) { - offenders.push(format!("{file} → {symbol}")); - } - } - } - assert!( - offenders.is_empty(), - "migrated host modules must derive their catalog and registry bindings \ - from descriptors:\n{}", - offenders.join("\n") - ); -} - #[test] fn standard_catalog_is_derived_from_one_descriptor_per_module() { let modules = vm::standard_host_modules(); @@ -1020,31 +36,27 @@ fn standard_catalog_is_derived_from_one_descriptor_per_module() { assert_eq!( names.len(), unique.len(), - "standard host module names must be unique: {names:?}" + "standard host module names must be unique" ); let mut sorted = names.clone(); sorted.sort_unstable(); assert_eq!( names, sorted, - "the standard host module aggregation must be deterministic (sorted by name)" + "standard host module order must be deterministic" ); for module in modules { assert!( !module.owned.is_empty(), - "module '{}' must own at least one descriptor", + "module '{}' must own descriptors", module.name ); let Some(surface) = module.catalog_module() else { - assert!( - module.catalog_module().is_none(), - "a descriptor-only module must not publish a catalog surface" - ); continue; }; assert!( !surface.functions.is_empty(), - "module '{}' published an empty catalog surface", + "module '{}' published an empty surface", module.name ); for factory in surface.functions { @@ -1062,31 +74,27 @@ fn standard_catalog_is_derived_from_one_descriptor_per_module() { } let catalog = vm::standard_host_catalog(); - let merged_owner = vm::standard_catalog_modules(); + let merged = vm::standard_catalog_modules(); assert_eq!( - merged_owner.len(), + merged.len(), modules .iter() - .filter(|m| m.catalog_module().is_some()) - .count(), - "every catalog surface must be merged exactly once" + .filter(|module| module.catalog_module().is_some()) + .count() ); assert_eq!( catalog.functions().len(), - merged_owner + merged .iter() .map(|module| module.functions.len()) - .sum::(), - "the standard catalog must be the merge of the module surfaces" + .sum::() ); assert_eq!( catalog.fingerprint().to_string(), - standard_catalog_fingerprint(), - "the standard catalog fingerprint must be byte-for-byte unchanged for the composed module set" + standard_catalog_fingerprint() ); } -/// The HTTP catalog fingerprint check, absent when the feature is disabled. #[cfg(all(feature = "http-client", not(target_family = "wasm")))] fn http_catalog_surface() -> Option<(&'static str, &'static str, vm::HostApiFingerprint)> { Some(( @@ -1101,9 +109,23 @@ fn http_catalog_surface() -> Option<(&'static str, &'static str, vm::HostApiFing None } +#[cfg(all(feature = "sqlite", not(target_family = "wasm")))] +fn sqlite_catalog_surface() -> Option<(&'static str, &'static str, vm::HostApiFingerprint)> { + Some(( + "sqlite", + SQLITE_CATALOG_FINGERPRINT, + vm::sqlite_host_catalog().fingerprint(), + )) +} + +#[cfg(not(all(feature = "sqlite", not(target_family = "wasm"))))] +fn sqlite_catalog_surface() -> Option<(&'static str, &'static str, vm::HostApiFingerprint)> { + None +} + #[test] fn module_catalogs_keep_their_published_fingerprints() { - let mut surfaces: Vec<(&str, &str, vm::HostApiFingerprint)> = vec![ + let mut surfaces = vec![ ( "io", IO_CATALOG_FINGERPRINT, @@ -1119,18 +141,14 @@ fn module_catalogs_keep_their_published_fingerprints() { TIMER_CATALOG_FINGERPRINT, vm::timer_host_catalog().fingerprint(), ), - ( - "sqlite", - SQLITE_CATALOG_FINGERPRINT, - vm::sqlite_host_catalog().fingerprint(), - ), ]; surfaces.extend(http_catalog_surface()); + surfaces.extend(sqlite_catalog_surface()); for (name, golden, fingerprint) in surfaces { assert_eq!( fingerprint.to_string(), golden, - "the `{name}` host catalog fingerprint must be byte-for-byte unchanged" + "`{name}` catalog fingerprint changed" ); } } @@ -1138,93 +156,76 @@ fn module_catalogs_keep_their_published_fingerprints() { #[test] fn every_catalog_resource_key_has_a_declaration() { let catalog = vm::standard_host_catalog(); - let mut undeclared = Vec::new(); - for resource in catalog.resources() { - if resource.description.trim().is_empty() { - undeclared.push(resource.key.to_string()); - } - } assert!( - undeclared.is_empty(), - "every guest resource key must come from a HostResourceType declaration: {undeclared:?}" + catalog + .resources() + .iter() + .all(|resource| !resource.description.trim().is_empty()) ); - let expected: BTreeSet = [ - STANDARD_RESOURCE_KEYS, - if HTTP_SURFACE_ENABLED { - HTTP_RESOURCE_KEYS - } else { - &[] - }, - ] - .concat() - .into_iter() - .map(str::to_string) - .collect(); - let actual: BTreeSet = catalog + let mut expected = BTreeSet::from(["io.file"]); + if SQLITE_SURFACE_ENABLED { + expected.insert("sqlite.connection"); + } + let actual: BTreeSet<&str> = catalog .resources() .iter() - .map(|resource| resource.key.to_string()) + .map(|resource| resource.key.as_str()) .collect(); - assert_eq!( - actual, expected, - "the standard resource set must be unchanged for the composed module set" - ); - if !HTTP_SURFACE_ENABLED { - for key in HTTP_RESOURCE_KEYS { - assert!( - !actual.contains(*key), - "`{key}` must not leak into a build without the HTTP module" - ); - } - } + assert_eq!(actual, expected); } #[test] -fn typed_named_struct_contract_is_preserved() { +fn typed_named_struct_contract_follows_composed_modules() { let catalog = vm::standard_host_catalog(); let declared: BTreeSet<&str> = catalog .structs() .iter() .map(|schema| schema.name.as_str()) .collect(); - for name in STANDARD_NAMED_STRUCTS { - assert!(declared.contains(name), "named struct `{name}` is missing"); + + assert!(declared.contains("JitConfig")); + for name in [ + "SqliteValue", + "SqliteQueryResult", + "SqliteTransactionResult", + ] { + assert_eq!(declared.contains(name), SQLITE_SURFACE_ENABLED, "{name}"); + } + for name in [ + "HttpRequest", + "HttpResponse", + "SseEvent", + "SseSummary", + "SseCallbackAction", + ] { + assert_eq!(declared.contains(name), HTTP_SURFACE_ENABLED, "{name}"); } - for name in HTTP_NAMED_STRUCTS { + + if SQLITE_SURFACE_ENABLED { + assert_eq!( + catalog + .struct_named("SqliteQueryResult") + .expect("SqliteQueryResult") + .fields + .len(), + 4 + ); assert_eq!( - declared.contains(name), - HTTP_SURFACE_ENABLED, - "named struct `{name}` must follow the HTTP module's composition gate" + catalog + .struct_named("SqliteValue") + .expect("SqliteValue") + .fields + .len(), + 5 ); } - // The typed shapes themselves are unchanged, not just the names. - let query = catalog - .struct_named("SqliteQueryResult") - .expect("SqliteQueryResult"); - assert_eq!( - query.fields.len(), - 4, - "SqliteQueryResult keeps columns/rows/truncated/next_cursor" - ); - let value = catalog.struct_named("SqliteValue").expect("SqliteValue"); - assert_eq!( - value.fields.len(), - 5, - "SqliteValue keeps kind plus four typed payload slots" - ); - - // No public standard function regressed to a dynamic return. - let mut dynamic = Vec::new(); - for function in catalog.functions() { - if function.return_type == vm::HostTypeSchema::Unknown { - dynamic.push(function.name.clone()); - } - } assert!( - dynamic.is_empty(), - "a public host function must not return `unknown`: {dynamic:?}" + catalog + .functions() + .iter() + .all(|function| function.return_type != vm::HostTypeSchema::Unknown) ); } @@ -1237,66 +238,41 @@ fn feature_gates_select_whole_modules_deterministically() { for always in ["io", "jit", "timer", "regex", "core", "math"] { assert!(names.contains(always), "`{always}` must always be composed"); } + assert_eq!(names.contains("http"), HTTP_SURFACE_ENABLED); + assert_eq!(names.contains("sqlite"), SQLITE_SURFACE_ENABLED); - #[cfg(all(feature = "http-client", not(target_family = "wasm")))] - assert!( - names.contains("http"), - "the http module must be composed when the feature is enabled" - ); - #[cfg(not(all(feature = "http-client", not(target_family = "wasm"))))] - assert!( - !names.contains("http"), - "the http module must be excluded when the feature is disabled" - ); - - // The SQLite catalog surface is composed in every build: the feature only - // selects whether its adapters are the real host functions. - assert!( - names.contains("sqlite"), - "the sqlite catalog surface must be composed in every build" - ); - - // A gated module must not leak into the derived guest catalog. let catalog = vm::standard_host_catalog(); - let has_http_function = catalog + let has_http = catalog .functions() .iter() .any(|function| function.name.starts_with("http::")); - assert_eq!( - has_http_function, - cfg!(all(feature = "http-client", not(target_family = "wasm"))), - "the guest catalog must follow the same gate as the module" - ); - assert!( - catalog - .functions() - .iter() - .any(|function| function.name.starts_with("sqlite::")), - "the SQLite guest imports must resolve in every build" - ); + let has_sqlite = catalog + .functions() + .iter() + .any(|function| function.name.starts_with("sqlite::")); + assert_eq!(has_http, HTTP_SURFACE_ENABLED); + assert_eq!(has_sqlite, SQLITE_SURFACE_ENABLED); } #[test] -fn legacy_registration_apis_stay_public_for_downstream_migration() { - // The builder and the low-level registry stay usable: the descriptor path - // is the preferred authoring surface, not the only one. +fn low_level_registration_apis_remain_executable() { let mut builder = vm::HostApiBuilder::new(); builder.resource(vm::ResourceTypeSchema::new( - vm::ResourceTypeKey::new("demo.legacy").expect("key"), - "A legacy resource", + vm::ResourceTypeKey::new("demo.resource").expect("key"), + "A demo resource", )); - builder.named_struct(vm::HostStructSchema::new("LegacyPoint", vec![])); + builder.named_struct(vm::HostStructSchema::new("DemoPoint", vec![])); builder.function(vm::HostFunctionSchema::with_return( - "demo::legacy", + "demo::call", vec![], vm::HostTypeSchema::Int, )); - let catalog = builder.build().expect("the legacy builder still builds"); + let catalog = builder.build().expect("catalog builds"); let mut registry = vm::HostFunctionRegistry::empty(); - registry.register_static_stack("demo::legacy", 0, |_vm, _args| { + registry.register_static_stack("demo::call", 0, |_vm, _args| { Ok(vm::CallOutcome::Return(vm::CallReturn::None)) }); - assert!(registry.contains_name("demo::legacy")); + assert!(registry.contains_name("demo::call")); assert_eq!(catalog.functions().len(), 1); } diff --git a/tests/typed_host_no_dynamic_contract_tests.rs b/tests/typed_host_no_dynamic_contract_tests.rs index f5928b27..09cae030 100644 --- a/tests/typed_host_no_dynamic_contract_tests.rs +++ b/tests/typed_host_no_dynamic_contract_tests.rs @@ -7,7 +7,7 @@ use std::collections::BTreeSet; not(target_family = "wasm") ))] use vm::http_host_catalog; -#[cfg(feature = "runtime")] +#[cfg(all(feature = "runtime", feature = "sqlite", not(target_family = "wasm")))] use vm::sqlite_host_catalog; #[cfg(feature = "runtime")] use vm::{HostApiCatalog, jit_host_catalog, standard_host_catalog, timer_host_catalog}; @@ -324,44 +324,6 @@ fn affected_public_host_catalogs_have_no_reachable_map_or_unknown() { assert_no_public_dynamic_schema("timer", &timer_host_catalog()); #[cfg(all(feature = "http-client", not(target_family = "wasm")))] assert_no_public_dynamic_schema("http", &http_host_catalog()); - #[cfg(all(feature = "sqlite", not(target_arch = "wasm32")))] + #[cfg(all(feature = "sqlite", not(target_family = "wasm")))] assert_no_public_dynamic_schema("sqlite", &sqlite_host_catalog()); } - -#[cfg(all(feature = "runtime", not(feature = "sqlite")))] -#[test] -fn standard_catalog_keeps_sqlite_editor_schema_without_sqlite_runtime() { - let sqlite = sqlite_host_catalog(); - assert!( - sqlite.function("sqlite::query").is_some(), - "the standalone editor/compiler catalog must retain SQLite declarations" - ); - let query = sqlite - .function("sqlite::query") - .expect("SQLite query declaration"); - let value = sqlite - .struct_named("SqliteValue") - .expect("SQLite value named struct"); - assert_eq!( - query.params[2].ty, - HostTypeSchema::Array(Box::new(value.as_type())), - "SQLite query params must stay typed without the runtime feature" - ); - assert_no_public_dynamic_schema("sqlite", &sqlite); - - let catalog = standard_host_catalog(); - assert!( - catalog.function("sqlite::open").is_some(), - "the editor/compiler catalog must retain SQLite schema declarations" - ); - assert!( - catalog.struct_named("SqliteOpenOptions").is_some(), - "the editor/compiler catalog must retain SQLite named structs" - ); - assert!( - vm::default_host_callables() - .iter() - .all(|callable| !callable.name.starts_with("sqlite::")), - "the executable default host surface must remain feature-gated" - ); -} diff --git a/tests/vm/http_sse_tests.rs b/tests/vm/http_sse_tests.rs index e6cdb45c..42a79c1f 100644 --- a/tests/vm/http_sse_tests.rs +++ b/tests/vm/http_sse_tests.rs @@ -557,6 +557,29 @@ fn sse_rejects_wrong_callback_schema_and_invalid_timeout_before_permit_admission ) .is_err()); + let compiled = compile_source( + r#" + use http; + http::client::sse( + {method: "GET", url: "http://127.0.0.1:1/events"}, + |item| {action: "continue"} + ); + "#, + ) + .expect("the structural callback reaches runtime schema validation"); + let mut vm = Vm::new(compiled.program); + vm.configure_http(config(1)).unwrap(); + HostFunctionRegistry::new().bind_vm_cached(&mut vm).unwrap(); + let error = vm + .run() + .expect_err("an object callback result must be rejected"); + assert!( + error + .to_string() + .contains("fn(SseEvent) -> SseCallbackAction"), + "{error}" + ); + for (timeout, expected) in [("0", "positive"), ("-1", "positive")] { let source = format!( r#" @@ -1001,9 +1024,10 @@ async fn sse_reset_releases_the_connection_permit_before_reuse() { ]); let source = format!( r#"use http; + fn callback(item: SseEvent) -> SseCallbackAction {{ {{action: "continue"}} }} http::client::sse( {{"method":"GET","url":"http://127.0.0.1:{port}/events"}}, - |item| {{action: "continue"}} + callback );"# ); let compiled = compile_source(&source).unwrap(); @@ -1053,11 +1077,12 @@ async fn sse_reset_while_callback_waits_retires_stream_to_quiescence() { let source = format!( r#"use http; fn async_wait() -> bool; + fn callback(item: SseEvent) -> SseCallbackAction {{ + {{action: if async_wait() => {{ "continue" }} else => {{ "continue" }} }} + }} http::client::sse( {{"method":"GET","url":"http://127.0.0.1:{port}/events"}}, - |item| {{ - action: if async_wait() => {{ "continue" }} else => {{ "continue" }} - }} + callback );"# ); let compiled = compile_source(&source).unwrap(); @@ -1290,7 +1315,9 @@ async fn sse_total_deadline_releases_the_connection_permit_for_reuse() { first.join().unwrap(); }); let source = format!( - r#"use http; http::client::sse({{"method":"GET","url":"http://127.0.0.1:{port}/events"}}, |item| {{action:"continue"}});"# + r#"use http; + fn callback(item: SseEvent) -> SseCallbackAction {{ {{action: "continue"}} }} + http::client::sse({{"method":"GET","url":"http://127.0.0.1:{port}/events"}}, callback);"# ); let compiled = compile_source(&source).unwrap(); let mut vm = Vm::new(compiled.program); @@ -1348,11 +1375,12 @@ async fn sse_callback_stop_after_deadline_fails_and_releases_permit_without_anot r#" use http; fn async_wait() -> bool; + fn callback(item: SseEvent) -> SseCallbackAction {{ + {{action: if async_wait() => {{ "stop" }} else => {{ "stop" }} }} + }} http::client::sse( {{"method":"GET","url":"http://127.0.0.1:{port}/events"}}, - |item| {{ - action: if async_wait() => {{ "stop" }} else => {{ "stop" }} - }} + callback ); "# ); @@ -1421,11 +1449,12 @@ async fn sse_callback_continue_after_deadline_fails_before_another_network_poll( r#" use http; fn async_wait() -> bool; + fn callback(item: SseEvent) -> SseCallbackAction {{ + {{action: if async_wait() => {{ "continue" }} else => {{ "continue" }} }} + }} http::client::sse( {{"method":"GET","url":"http://127.0.0.1:{port}/events"}}, - |item| {{ - action: if async_wait() => {{ "continue" }} else => {{ "continue" }} - }} + callback ); "# ); From 0f1e81d9db9dcea1285bbd7124cbcd31e1179217 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sat, 19 Sep 2026 20:56:56 +0800 Subject: [PATCH 14/23] fix(build): gate sqlite catalog on wasm --- Cargo.toml | 4 +-- docs/host-sdk-descriptors.md | 14 +++++----- tests/wire/catalog_build_validation_tests.rs | 7 ++--- tests/wire/catalog_contract_tests.rs | 28 ++++++++++---------- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 47b9f756..f2e25396 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,8 +87,6 @@ cranelift-jit = { version = "0.129.1", optional = true } cranelift-module = { version = "0.129.1", optional = true } cranelift-native = { version = "0.129.1", optional = true } pd-host-function = { path = "./pd-host-function", version = "0.1.0" } -rusqlite = { version = "0.40.1", default-features = false, features = ["bundled", "hooks", "limits"], optional = true } -tokio-rusqlite = { version = "0.8", optional = true } edge_abi = { package = "pd-edge-abi", version = "0.1.1", default-features = false, optional = true } futures-channel = "0.3" paste = "1" @@ -104,6 +102,8 @@ http-body-util = { version = "0.1", optional = true } hyper = { version = "1", default-features = false, features = ["client", "http1"], optional = true } hyper-rustls = { version = "0.27.7", default-features = false, features = ["http1", "ring", "tls12", "webpki-roots"], optional = true } hyper-util = { version = "0.1", default-features = false, features = ["client-legacy", "http1", "tokio"], optional = true } +rusqlite = { version = "0.40.1", default-features = false, features = ["bundled", "hooks", "limits"], optional = true } +tokio-rusqlite = { version = "0.8", optional = true } tower-service = { version = "0.3", optional = true } url = { version = "2", optional = true } tokio = { version = "1", features = ["rt-multi-thread", "net", "time", "sync", "fs", "io-util", "process", "macros"], optional = true } diff --git a/docs/host-sdk-descriptors.md b/docs/host-sdk-descriptors.md index d017f87a..9ee02dc5 100644 --- a/docs/host-sdk-descriptors.md +++ b/docs/host-sdk-descriptors.md @@ -303,13 +303,13 @@ Each module declares two things: - `owned` — **every** descriptor the module owns, including functions still dispatched through the generated namespaced-builtin path. -`tests/standard_host_descriptor_arch_tests.rs` proves that the two agree: every -standard `#[pd_host_function]` has exactly one descriptor owner, every ownership -list is declared by a file that belongs to a module `standard_host_modules()` -composes (a module this build's gates turn off is the only exemption, and it is -stated in the guard), no gated module leaks into the derived catalog, and any -intentional catalog revision updates the fingerprint format plus artifact ABI -with explicit prior-artifact rejection coverage. +`tests/standard_host_descriptor_arch_tests.rs` checks the composed descriptors +and catalogs directly: module names are unique and deterministically ordered, +every published function belongs to its module's ownership list, and the merged +catalog preserves the expected module and aggregate fingerprints. It also checks +that resource declarations and named structs follow the selected modules, public +returns stay typed, and native-only HTTP and SQLite surfaces do not appear when +their build gates are off. ## 7. Compatibility window diff --git a/tests/wire/catalog_build_validation_tests.rs b/tests/wire/catalog_build_validation_tests.rs index bc60c3c9..23a7a336 100644 --- a/tests/wire/catalog_build_validation_tests.rs +++ b/tests/wire/catalog_build_validation_tests.rs @@ -55,9 +55,10 @@ fn parse_catalog_source_accepts_the_checked_in_catalog() { )) .expect("read authoritative catalog"); let entries = parse_catalog_source(&source, "catalog.rs"); - // The SQLite namespace is optional (mirrors the build.rs feature filter): - // when the feature is off, the generated catalog excludes it. - #[cfg(not(feature = "sqlite"))] + // The SQLite namespace is native-only (mirrors the build.rs filter): when + // the feature is off or the target family is wasm, the generated catalog + // excludes it. + #[cfg(not(all(feature = "sqlite", not(target_family = "wasm"))))] let entries: Vec<_> = entries .into_iter() .filter(|entry| !entry.source_name.starts_with("sqlite::")) diff --git a/tests/wire/catalog_contract_tests.rs b/tests/wire/catalog_contract_tests.rs index 42ae876a..0faee5f1 100644 --- a/tests/wire/catalog_contract_tests.rs +++ b/tests/wire/catalog_contract_tests.rs @@ -27,9 +27,9 @@ const EXTENSION_BLOCK_END: u16 = 0xFF8F; const SPECIAL_CALL_BLOCK_START: u16 = 0xFF90; const SPECIAL_CALL_BLOCK_END: u16 = 0xFFA1; const ORDINARY_BLOCK_START: u16 = 0xFFA2; -#[cfg(feature = "sqlite")] +#[cfg(all(feature = "sqlite", not(target_family = "wasm")))] const SQLITE_RESERVED_TOP_START: u16 = 0xFFFC; -#[cfg(feature = "sqlite")] +#[cfg(all(feature = "sqlite", not(target_family = "wasm")))] const SQLITE_RESERVED_TOP_END: u16 = u16::MAX; /// Reserved sentinel gap inside the special-call block (see the catalog docs @@ -84,15 +84,15 @@ fn parse_catalog(source: &str) -> Vec { feature_gate: parts[4].to_string(), }); } - // The SQLite namespace is optional (mirrors the build.rs feature filter): - // when the feature is off, the generated catalog excludes it, so the - // parsed raw catalog must agree. - #[cfg(not(feature = "sqlite"))] + // The SQLite namespace is native-only (mirrors the build.rs filter): when + // the feature is off or the target family is wasm, the generated catalog + // excludes it, so the parsed raw catalog must agree. + #[cfg(not(all(feature = "sqlite", not(target_family = "wasm"))))] entries.retain(|entry| !entry.source_name.starts_with("sqlite::")); entries } -#[cfg(feature = "sqlite")] +#[cfg(all(feature = "sqlite", not(target_family = "wasm")))] #[test] fn sqlite_top_u16_ids_are_explicitly_reserved_for_frozen_entries() { let entries = parse_catalog(&catalog_source()); @@ -279,10 +279,10 @@ fn checked_in_nostd_mirror_matches_std_catalog() { }; let id = u16::from_str_radix(hex.trim().trim_start_matches("0x"), 16) .unwrap_or_else(|err| panic!("mirror const {const_name} has invalid id: {err}")); - // The SQLite namespace is optional (mirrors the build.rs feature - // filter): when the feature is off, the mirror's sqlite consts are - // excluded from the sync contract. - #[cfg(not(feature = "sqlite"))] + // The SQLite namespace is native-only (mirrors the build.rs filter): + // when the feature is off or the target family is wasm, the mirror's + // sqlite consts are excluded from the sync contract. + #[cfg(not(all(feature = "sqlite", not(target_family = "wasm"))))] if const_name.starts_with("SQLITE_") { continue; } @@ -387,9 +387,9 @@ fn appending_or_reordering_catalog_entries_does_not_renumber_existing_ids() { // Appending a new entry at the next free ordinary ID (append-only // allocation) must not renumber any existing entry. When the optional - // SQLite namespace is enabled the ordinary block (0xFFA2..=0xFFFF) is - // exactly full, so there is nothing to append and the property is - // trivially preserved. + // SQLite namespace is enabled on a native target the ordinary block + // (0xFFA2..=0xFFFF) is exactly full, so there is nothing to append and the + // property is trivially preserved. let mut used: Vec = entries.iter().map(|entry| entry.id).collect(); used.sort_unstable(); let Some(next_free) = From 31392782f9dd131727cbbeb4884185c254378d51 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sat, 19 Sep 2026 21:42:47 +0800 Subject: [PATCH 15/23] fix(tests): complete pending async bridge stub --- tests/builtins/io_async_tests.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/builtins/io_async_tests.rs b/tests/builtins/io_async_tests.rs index ff1c6710..eba71879 100644 --- a/tests/builtins/io_async_tests.rs +++ b/tests/builtins/io_async_tests.rs @@ -43,6 +43,22 @@ fn builtin_pending_completion_uses_declared_return_type() { ) -> Poll> { Poll::Pending } + + fn poll_submitted_op( + &mut self, + _op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Pending + } + + fn cleanup_op( + &mut self, + _op_id: HostOpId, + _terminal: vm::HostAsyncOpTerminal, + ) -> VmResult<()> { + Ok(()) + } } let builtin = BuiltinFunction::from_namespaced_name("io::exists") From 2994d7f681fb634123278ebad266c6ce5b6babb9 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sun, 20 Sep 2026 03:14:39 +0800 Subject: [PATCH 16/23] Enforce Tokio process runtime lease --- docs/host-sdk-descriptors.md | 26 +++ src/builtins/runtime/io/async_io.rs | 305 +++++++++++++++++++++++----- src/vm/execution_scope.rs | 34 ++-- src/vm/host.rs | 15 ++ src/vm/host_runtime.rs | 3 + src/vm/mod.rs | 36 ++-- src/vm/operation/registry.rs | 2 +- src/vm/resource/close.rs | 3 + src/vm/resource/error.rs | 36 ++++ src/vm/resource/table.rs | 109 +++++++--- tests/builtins/io_async_tests.rs | 41 +++- tests/builtins_tests.rs | 1 + tests/vm/execution_scope_tests.rs | 84 ++++++++ 13 files changed, 572 insertions(+), 123 deletions(-) diff --git a/docs/host-sdk-descriptors.md b/docs/host-sdk-descriptors.md index 9ee02dc5..ac5b3c5d 100644 --- a/docs/host-sdk-descriptors.md +++ b/docs/host-sdk-descriptors.md @@ -205,6 +205,32 @@ return `HostFutureOutput` and use `HostFutureOutput::continue_with`; value mapping preserves that continuation. The HTTP SSE builtin uses this only to transfer an opened Hyper response into the generic callable-stream driver. +### Tokio runtime lease for process-backed IO + +With the `async` feature, `io::popen` and `io::close` use Tokio process and pipe +types exclusively. The embedding Tokio runtime that spawns a process-backed IO +resource must remain alive and actively driven until `io::close` completes or a +VM reset has been polled to quiescence. This includes Windows process pipes: do +not shut the runtime down while a process pipe operation or process cleanup is +still active. + +Process close and reset cleanup must be polled while a live Tokio runtime is +current. If reset cleanup is polled outside such a context, it returns the +retryable resource error `runtime_required` immediately. The child, its pipes, +the closing execution scope, and the pending reset remain owned by the VM; the +VM cannot be reused. Re-enter and drive the runtime, then poll the same reset +again. `Vm::clear_async_bridge` also rejects an execution scope with live +resources or operations, and rejects a pending reset, so the bridge can be +released only after scope quiescence. + +`Vm`, process IO resource, and process IO handle `Drop` paths are strictly +nonblocking. They issue only immediate best-effort direct-child kill requests +and, on Unix, a process-group signal. They do not poll cleanup futures, wait for +the leader, or run Windows tree termination. Abrupt Drop therefore has no +eventual-reap guarantee. Embeddings that require deterministic tree termination +and leader reaping must finish explicit `io::close` or reset while the runtime +lease is valid. + What the contract does and does not change: - The contract **replaces only the guest schema**. The adapter, binding class, and diff --git a/src/builtins/runtime/io/async_io.rs b/src/builtins/runtime/io/async_io.rs index b60ec0a2..fe114304 100644 --- a/src/builtins/runtime/io/async_io.rs +++ b/src/builtins/runtime/io/async_io.rs @@ -4,6 +4,12 @@ //! handles remain typed execution-scope resources because they span guest //! calls; transient reads, writes, flushes, and closes rely on the generic //! submitted-future lifecycle. +//! +//! Process resources borrow the embedding's Tokio runtime as a runtime lease: +//! the runtime that spawns them must remain alive and actively driven until an +//! explicit close or reset reaches quiescence. Abrupt Drop only sends immediate +//! best-effort kill signals; it never waits and provides no eventual-reap +//! guarantee. use std::future::Future; use std::io; @@ -43,17 +49,55 @@ impl Drop for IoHandle { fn drop(&mut self) { match self { Self::PopenRead { child, .. } | Self::PopenWrite { child, .. } => { - let _ = start_terminate_child_tree(child); + let _ = start_terminate_child_tree_for_drop(child); } Self::File(_) => {} } } } +type ProcessTreeCleanupFuture = Pin> + Send + 'static>>; +type PreparedProcessCleanup = ( + Option, + bool, + bool, + Option, +); + +#[cfg(windows)] +fn prepare_process_cleanup( + child: &mut Child, + _terminate_tree: fn(u32) -> io::Result<()>, +) -> PreparedProcessCleanup { + let pid = child.id().unwrap_or(0); + ( + Some(Box::pin(terminate_process_id(pid))), + false, + false, + None, + ) +} + +#[cfg(not(windows))] +fn prepare_process_cleanup( + child: &mut Child, + terminate_tree: fn(u32) -> io::Result<()>, +) -> PreparedProcessCleanup { + ( + None, + true, + true, + start_terminate_child_tree_with(child, terminate_tree).err(), + ) +} + /// The typed resource stored in the execution scope for one async IO handle. struct IoResource { handle: Arc>>, process_tree_terminator: fn(u32) -> io::Result<()>, + process_tree_cleanup: Option, + process_tree_cleanup_complete: bool, + leader_kill_started: bool, deferred_process_cleanup_error: Option, } @@ -61,7 +105,10 @@ impl IoResource { fn new(handle: IoHandle) -> Self { Self { handle: Arc::new(Mutex::new(Some(handle))), - process_tree_terminator: terminate_process_id, + process_tree_terminator: signal_process_tree_for_drop, + process_tree_cleanup: None, + process_tree_cleanup_complete: false, + leader_kill_started: false, deferred_process_cleanup_error: None, } } @@ -74,6 +121,9 @@ impl IoResource { Self { handle: Arc::new(Mutex::new(Some(handle))), process_tree_terminator, + process_tree_cleanup: None, + process_tree_cleanup_complete: false, + leader_kill_started: false, deferred_process_cleanup_error: None, } } @@ -93,18 +143,26 @@ impl IoResource { fn begin_close_after_operations_quiesce(&mut self) -> ResourceResult { let process_tree_terminator = self.process_tree_terminator; let slot = self.exclusive_handle_slot()?; - let (progress, cleanup_error) = match slot.as_mut() { + let (progress, process_cleanup) = match slot.as_mut() { None => (CloseProgress::Ready, None), Some(IoHandle::File(_)) => { slot.take(); (CloseProgress::Ready, None) } - Some(IoHandle::PopenRead { child, .. }) | Some(IoHandle::PopenWrite { child, .. }) => { - let cleanup_error = - start_terminate_child_tree_with(child, process_tree_terminator).err(); - (CloseProgress::Pending, cleanup_error) - } + Some(IoHandle::PopenRead { child, .. }) | Some(IoHandle::PopenWrite { child, .. }) => ( + CloseProgress::Pending, + Some(prepare_process_cleanup(child, process_tree_terminator)), + ), }; + let ( + process_tree_cleanup, + process_tree_cleanup_complete, + leader_kill_started, + cleanup_error, + ) = process_cleanup.unwrap_or((None, true, true, None)); + self.process_tree_cleanup = process_tree_cleanup; + self.process_tree_cleanup_complete = process_tree_cleanup_complete; + self.leader_kill_started = leader_kill_started; self.deferred_process_cleanup_error = cleanup_error; Ok(progress) } @@ -112,6 +170,9 @@ impl IoResource { fn poll_process_close(&mut self, cx: &mut Context<'_>) -> Poll> { let Self { handle, + process_tree_cleanup, + process_tree_cleanup_complete, + leader_kill_started, deferred_process_cleanup_error, .. } = self; @@ -128,17 +189,29 @@ impl IoResource { let poll = match slot.as_mut() { None | Some(IoHandle::File(_)) => Poll::Ready(Ok(())), Some(IoHandle::PopenRead { child, .. }) | Some(IoHandle::PopenWrite { child, .. }) => { - match child.try_wait() { - Ok(Some(_)) => Poll::Ready(Ok(())), - Err(error) => Poll::Ready(Err(error)), - Ok(None) if tokio::runtime::Handle::try_current().is_err() => Poll::Pending, - Ok(None) => { - let mut wait = Box::pin(child.wait()); - match wait.as_mut().poll(cx) { - Poll::Pending => Poll::Pending, - Poll::Ready(result) => Poll::Ready(result.map(|_| ())), - } - } + if tokio::runtime::Handle::try_current().is_err() { + return Poll::Ready(Err(runtime_required_resource_error())); + } + if !*process_tree_cleanup_complete { + let cleanup = process_tree_cleanup + .as_mut() + .expect("pending process tree cleanup retains its future"); + let tree_result = match cleanup.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(result) => result, + }; + process_tree_cleanup.take(); + *process_tree_cleanup_complete = true; + let leader_result = child.start_kill().or_else(ignore_already_exited); + *leader_kill_started = true; + *deferred_process_cleanup_error = + combine_process_cleanup_results(tree_result, leader_result).err(); + } + debug_assert!(*leader_kill_started); + let mut wait = Box::pin(child.wait()); + match wait.as_mut().poll(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(result) => Poll::Ready(result.map(|_| ())), } } }; @@ -162,9 +235,9 @@ impl IoResource { impl Drop for IoResource { fn drop(&mut self) { if let Some(mutex) = Arc::get_mut(&mut self.handle) - && let Some(mut handle) = mutex.get_mut().take() + && let Some(handle) = mutex.get_mut().as_mut() { - let _ = start_close_io_handle(&mut handle); + start_close_io_handle_for_drop(handle); } } } @@ -197,11 +270,20 @@ fn process_resource_error(error: io::Error) -> ResourceError { ) } -fn start_close_io_handle(handle: &mut IoHandle) -> ResourceResult<()> { +fn runtime_required_resource_error() -> ResourceError { + ResourceError::new( + ResourceErrorCode::RuntimeRequired, + "io::resource", + "process cleanup requires polling from a live Tokio runtime", + ) + .with_retryable_disposition() +} + +fn start_close_io_handle_for_drop(handle: &mut IoHandle) { match handle { - IoHandle::File(_) => Ok(()), + IoHandle::File(_) => {} IoHandle::PopenRead { child, .. } | IoHandle::PopenWrite { child, .. } => { - start_terminate_child_tree(child).map_err(process_resource_error) + let _ = start_terminate_child_tree_for_drop(child); } } } @@ -226,6 +308,16 @@ async fn close_shared_io_handle_with( } async fn close_io_handle(handle: &mut IoHandle) -> VmResult<()> { + if matches!( + handle, + IoHandle::PopenRead { .. } | IoHandle::PopenWrite { .. } + ) && tokio::runtime::Handle::try_current().is_err() + { + return Err(VmError::HostError( + "RuntimeRequired: io::close process cleanup must be polled from a live Tokio runtime" + .to_string(), + )); + } match handle { IoHandle::File(file) => { file.get_mut() @@ -259,16 +351,17 @@ async fn kill_and_reap_child(child: &mut Child) -> io::Result<()> { } } -async fn terminate_process_tree_and_leader( +async fn terminate_process_tree_and_leader( terminate_tree: Tree, terminate_leader: Leader, ) -> io::Result<()> where - Tree: FnOnce() -> io::Result<()>, + Tree: FnOnce() -> TreeFuture, + TreeFuture: Future>, Leader: FnOnce() -> LeaderFuture, LeaderFuture: Future>, { - let tree_result = terminate_tree(); + let tree_result = terminate_tree().await; let leader_result = terminate_leader().await; combine_process_cleanup_results(tree_result, leader_result) } @@ -290,8 +383,8 @@ fn combine_process_cleanup_results( } } -fn start_terminate_child_tree(child: &mut Child) -> io::Result<()> { - start_terminate_child_tree_with(child, terminate_process_id) +fn start_terminate_child_tree_for_drop(child: &mut Child) -> io::Result<()> { + start_terminate_child_tree_with(child, signal_process_tree_for_drop) } fn start_terminate_child_tree_with( @@ -312,7 +405,7 @@ fn ignore_already_exited(error: io::Error) -> io::Result<()> { } } -fn terminate_process_id(pid: u32) -> io::Result<()> { +async fn terminate_process_id(pid: u32) -> io::Result<()> { if pid == 0 { return Ok(()); } @@ -326,7 +419,7 @@ fn terminate_process_id(pid: u32) -> io::Result<()> { } #[cfg(windows)] { - run_taskkill_with(pid, std::process::Command::status) + run_taskkill_with(pid, |mut command| async move { command.status().await }).await } #[cfg(not(any(unix, windows)))] { @@ -335,6 +428,25 @@ fn terminate_process_id(pid: u32) -> io::Result<()> { } } +fn signal_process_tree_for_drop(pid: u32) -> io::Result<()> { + if pid == 0 { + return Ok(()); + } + #[cfg(unix)] + { + terminate_unix_process_group_with( + pid, + |process_group, signal| unsafe { libc::kill(process_group, signal) }, + io::Error::last_os_error, + ) + } + #[cfg(not(unix))] + { + let _ = pid; + Ok(()) + } +} + #[cfg(unix)] fn terminate_unix_process_group_with( pid: u32, @@ -359,13 +471,14 @@ where } #[cfg(any(windows, test))] -fn run_taskkill_with(pid: u32, run: Run) -> io::Result<()> +async fn run_taskkill_with(pid: u32, run: Run) -> io::Result<()> where - Run: FnOnce(&mut std::process::Command) -> io::Result, + Run: FnOnce(Command) -> RunFuture, + RunFuture: Future>, { - let mut command = std::process::Command::new("taskkill"); + let mut command = Command::new("taskkill"); command.args(["/T", "/F", "/PID", &pid.to_string()]); - let status = run(&mut command)?; + let status = run(command).await?; if status.success() { Ok(()) } else { @@ -765,7 +878,6 @@ fn spawn_shell_command(shell_command: &str, mode: &str) -> VmResult { }; #[cfg(unix)] process.process_group(0); - process.kill_on_drop(true); match mode { "r" => { process.stdout(Stdio::piped()).stdin(Stdio::null()); @@ -780,7 +892,7 @@ fn spawn_shell_command(shell_command: &str, mode: &str) -> VmResult { .map_err(|error| VmError::HostError(format!("io_popen failed: {error}")))?; if mode == "r" { let Some(stdout) = child.stdout.take() else { - let cleanup = start_terminate_child_tree(&mut child); + let cleanup = start_terminate_child_tree_for_drop(&mut child); return Err(VmError::HostError(match cleanup { Ok(()) => "io_popen('r') did not provide stdout pipe".to_string(), Err(error) => format!( @@ -794,7 +906,7 @@ fn spawn_shell_command(shell_command: &str, mode: &str) -> VmResult { }) } else { let Some(stdin) = child.stdin.take() else { - let cleanup = start_terminate_child_tree(&mut child); + let cleanup = start_terminate_child_tree_for_drop(&mut child); return Err(VmError::HostError(match cleanup { Ok(()) => "io_popen('w') did not provide stdin pipe".to_string(), Err(error) => format!( @@ -816,6 +928,16 @@ mod tests { use super::*; + #[cfg(windows)] + static DROP_TREE_TERMINATION_CALLS: std::sync::atomic::AtomicUsize = + std::sync::atomic::AtomicUsize::new(0); + + #[cfg(windows)] + fn record_drop_tree_termination(_pid: u32) -> io::Result<()> { + DROP_TREE_TERMINATION_CALLS.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + fn file_resource() -> IoResource { let file = std::fs::File::open("Cargo.toml").expect("test fixture should exist"); IoResource::new(IoHandle::File(BufReader::new(File::from_std(file)))) @@ -831,6 +953,25 @@ mod tests { }) } + #[cfg(unix)] + fn start_process_kill_then_pending<'a>( + handle: &'a mut IoHandle, + started: Arc>, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + match handle { + IoHandle::PopenRead { child, .. } | IoHandle::PopenWrite { child, .. } => { + child.start_kill().map_err(|error| { + VmError::HostError(format!("test process kill failed: {error}")) + })?; + } + IoHandle::File(_) => panic!("expected process handle"), + } + *started.lock().expect("started lock") = true; + pending().await + }) + } + #[tokio::test] async fn cancelling_explicit_close_retains_the_handle_in_the_resource() { let resource = file_resource(); @@ -853,6 +994,63 @@ mod tests { ); } + #[cfg(unix)] + #[tokio::test(flavor = "current_thread")] + async fn cancelling_process_close_retains_child_and_allows_retry_to_reap() { + let resource = + IoResource::new(spawn_shell_command("sleep 30", "r").expect("process should spawn")); + let shared = Arc::clone(&resource.handle); + let pid = { + let slot = shared.lock().await; + match slot.as_ref().expect("resource handle") { + IoHandle::PopenRead { child, .. } => child.id().expect("child pid"), + other => panic!("expected popen read handle, got {other:?}"), + } + }; + let started = Arc::new(StdMutex::new(false)); + let close_started = Arc::clone(&started); + let mut cancelled = Box::pin(close_shared_io_handle_with( + Arc::clone(&shared), + move |handle| start_process_kill_then_pending(handle, close_started), + )); + let mut cx = Context::from_waker(Waker::noop()); + + assert!(matches!(cancelled.as_mut().poll(&mut cx), Poll::Pending)); + assert!(*started.lock().expect("started lock")); + drop(cancelled); + assert!( + shared.lock().await.is_some(), + "cancelled close must preserve the child and its pipe" + ); + + close_shared_io_handle_with(Arc::clone(&shared), close_io_handle_future) + .await + .expect("retry should terminate and reap the retained child"); + assert!(shared.lock().await.is_none()); + assert!( + !std::path::Path::new(&format!("/proc/{pid}")).exists(), + "retry must reap the direct child" + ); + } + + #[cfg(windows)] + #[tokio::test(flavor = "current_thread")] + async fn dropping_process_resource_does_not_run_synchronous_tree_termination() { + DROP_TREE_TERMINATION_CALLS.store(0, std::sync::atomic::Ordering::SeqCst); + let resource = IoResource::new_with_process_tree_terminator( + spawn_shell_command("ping -n 30 127.0.0.1 >NUL", "r").expect("process should spawn"), + record_drop_tree_termination, + ); + + drop(resource); + + assert_eq!( + DROP_TREE_TERMINATION_CALLS.load(std::sync::atomic::Ordering::SeqCst), + 0, + "Drop must use only the direct nonblocking child kill on Windows" + ); + } + #[tokio::test] async fn process_resource_close_polls_until_the_leader_is_reaped() { let mut resource = @@ -919,7 +1117,7 @@ mod tests { let leader_attempted = Arc::new(StdMutex::new(false)); let attempted = Arc::clone(&leader_attempted); let error = terminate_process_tree_and_leader( - || { + || async { Err(io::Error::new( io::ErrorKind::PermissionDenied, "tree denied", @@ -957,35 +1155,34 @@ mod tests { assert_eq!(error.raw_os_error(), Some(libc::EPERM)); } - #[test] - fn taskkill_launch_failure_is_an_error() { - let launch = run_taskkill_with(42, |_| { + #[tokio::test] + async fn taskkill_launch_failure_is_an_error() { + let launch = run_taskkill_with(42, |_| async { Err(io::Error::new(io::ErrorKind::NotFound, "taskkill missing")) }) + .await .expect_err("taskkill launch failure must propagate"); assert_eq!(launch.kind(), io::ErrorKind::NotFound); } #[cfg(unix)] - #[test] - fn unsuccessful_taskkill_status_is_an_error() { - let status = run_taskkill_with(42, |_| { - std::process::Command::new("sh") - .args(["-c", "exit 7"]) - .status() + #[tokio::test] + async fn unsuccessful_taskkill_status_is_an_error() { + let status = run_taskkill_with(42, |_| async { + Command::new("sh").args(["-c", "exit 7"]).status().await }) + .await .expect_err("unsuccessful taskkill status must propagate"); assert!(status.to_string().contains("status")); } #[cfg(windows)] - #[test] - fn unsuccessful_taskkill_status_is_an_error() { - let status = run_taskkill_with(42, |_| { - std::process::Command::new("cmd") - .args(["/C", "exit", "7"]) - .status() + #[tokio::test] + async fn unsuccessful_taskkill_status_is_an_error() { + let status = run_taskkill_with(42, |_| async { + Command::new("cmd").args(["/C", "exit", "7"]).status().await }) + .await .expect_err("unsuccessful taskkill status must propagate"); assert!(status.to_string().contains("status")); } diff --git a/src/vm/execution_scope.rs b/src/vm/execution_scope.rs index edd5340c..cacee2b9 100644 --- a/src/vm/execution_scope.rs +++ b/src/vm/execution_scope.rs @@ -14,8 +14,7 @@ //! resources own their close (see //! [`HostResource`](crate::vm::resource::HostResource)). -use std::sync::Arc; -use std::task::{Context, Poll, Wake, Waker}; +use std::task::{Context, Poll}; use super::operation::driver::{OperationOutcome, OperationSpec}; use super::operation::error::OperationError; @@ -112,6 +111,12 @@ impl std::fmt::Display for ExecutionScopeError { } impl ExecutionScopeError { + /// Whether this error reports a retained resource cleanup that may be + /// polled again after the caller restores the required runtime context. + pub fn is_retryable_cleanup(&self) -> bool { + matches!(self, Self::Resource(error) if error.is_retryable()) + } + /// Recovers the underlying `OperationError` when the failure is an /// operation-domain error; returns `None` for scope-state violations. pub fn into_operation_error(self) -> Option { @@ -612,6 +617,9 @@ impl ExecutionScope { .expect("finish_close set terminal"))) } Poll::Ready(Err(error)) => { + if error.is_retryable() { + return Poll::Ready(Err(ExecutionScopeError::Resource(error))); + } self.record_failure(ScopeCloseError::Resource(error)); self.finish_close(); Poll::Ready(Ok(self @@ -666,12 +674,6 @@ impl ExecutionScope { } } -struct ScopeDropWake; - -impl Wake for ScopeDropWake { - fn wake(self: Arc) {} -} - impl Drop for ExecutionScope { fn drop(&mut self) { if self.state == ScopeState::Active { @@ -682,15 +684,15 @@ impl Drop for ExecutionScope { if self.state != ScopeState::Closing { return; } - let waker = Waker::from(Arc::new(ScopeDropWake)); - let mut cx = Context::from_waker(&waker); - let _ = self.poll_close(&mut cx); - if self.state == ScopeState::Closing { - // A standalone scope drop cannot keep polling a Pending resource, - // but it must still launch every remaining ancestor close with the - // VmDrop reason before ResourceTable itself is dropped. - let _ = self.begin_drop_resource_close_nonblocking(); + if !self.operations_drained { + let reason = self.close_reason.unwrap_or(ResourceCloseReason::VmDrop); + let _ = self.operations.cancel_all(operation_reason(reason)); + self.operations_drained = true; } + // Abrupt drop only issues immediate cancellation/kill requests. It + // never polls an operation or resource future and cannot promise that + // process leaders will eventually be reaped. + let _ = self.begin_drop_resource_close_nonblocking(); } } diff --git a/src/vm/host.rs b/src/vm/host.rs index f5cbcf8e..1b0520ab 100644 --- a/src/vm/host.rs +++ b/src/vm/host.rs @@ -2723,7 +2723,22 @@ impl Vm { Ok(()) } + /// Removes the embedding async bridge after all bridge work and the + /// execution scope are quiescent. + /// + /// Live resources/operations and a pending reset retain the bridge because + /// process-backed async IO may still need the embedding Tokio runtime to + /// complete cleanup and reap its direct child. pub fn clear_async_bridge(&mut self) -> VmResult<()> { + if self.host.scope_reset_pending + || !self.host.execution_scope.resources().is_empty() + || !self.host.execution_scope.operations().is_empty() + { + return Err(VmError::HostError( + "cannot clear async bridge while the execution scope has live resources, operations, or a pending reset" + .to_string(), + )); + } if self.host.has_active_bridge_operations() || self .instance diff --git a/src/vm/host_runtime.rs b/src/vm/host_runtime.rs index dfdd2777..1c205c94 100644 --- a/src/vm/host_runtime.rs +++ b/src/vm/host_runtime.rs @@ -577,6 +577,9 @@ impl HostRuntime { match result { Poll::Pending => Poll::Pending, Poll::Ready(Err(error)) => { + if error.is_retryable_cleanup() { + return Poll::Ready(Err(VmError::ExecutionScope(error))); + } self.scope_reset_error = Some(error.clone()); Poll::Ready(Err(VmError::ExecutionScope(error))) } diff --git a/src/vm/mod.rs b/src/vm/mod.rs index c4d4cad2..751be9a2 100644 --- a/src/vm/mod.rs +++ b/src/vm/mod.rs @@ -1267,6 +1267,11 @@ impl Vm { /// A successful return only starts the reset; callers must poll /// `poll_reset_for_reuse` to obtain the deterministic completion result /// before observing an empty scope or reusing the VM. + /// + /// Process-backed async IO cleanup must be polled while a live Tokio + /// runtime is current. An off-runtime poll returns a retryable + /// `RuntimeRequired` resource error while retaining the old scope, + /// process/pipes, and pending reset for a later poll under that runtime. pub fn reset_for_reuse(&mut self) -> VmResult<()> { if let Err(error) = validate_frame_allocation_limits(&self.program) { self.host.mark_reset_failed(&error); @@ -1287,7 +1292,19 @@ impl Vm { return Err(error); } if let Err(error) = self.host.reset_execution_scope() { - self.instance.invalidate_callback_registries(); + if matches!( + &error, + VmError::ExecutionScope(scope_error) if scope_error.is_retryable_cleanup() + ) { + // Reset has been accepted and the old scope remains sealed. + // Clear guest-visible execution state now; a later successful + // cleanup poll only publishes the retained replacement scope. + self.run_ctx.reset_for_reuse(); + self.instance.reset(&self.program); + self.engine.reset_runtime_state(&self.program); + } else { + self.instance.invalidate_callback_registries(); + } return Err(error); } self.run_ctx.reset_for_reuse(); @@ -1679,21 +1696,14 @@ impl Vm { impl Drop for Vm { fn drop(&mut self) { - let _ = self.cancel_waiting_host_op_with_reason( - crate::vm::operation::OperationCancelReason::VmDrop, - ); - let _ = self.cancel_callable_stream_with_reason( - crate::vm::operation::OperationCancelReason::VmDrop, - ); - let _ = self.terminate_all_callable_streams_with_reason( - crate::vm::operation::OperationCancelReason::VmDrop, - ); + // Abrupt teardown is cancel-only. HostRuntime/ExecutionScope drop issue + // immediate cancellation and process kill requests without polling any + // future, entering a runtime, or waiting for worker/process quiescence. + // Embedders that require deterministic cleanup and child reaping must + // complete explicit close/reset before dropping the VM. self.host .cancel_submitted_host_ops(crate::vm::operation::OperationCancelReason::VmDrop); self.instance.drop_cleanup(); - // Live IO handles and in-flight IO operations are retired by the - // `ExecutionScope`'s own `Drop`, which runs as part of `HostRuntime`. - // (No custom close-all side channel is needed.) } } diff --git a/src/vm/operation/registry.rs b/src/vm/operation/registry.rs index 9c0947d1..85667fa5 100644 --- a/src/vm/operation/registry.rs +++ b/src/vm/operation/registry.rs @@ -792,7 +792,7 @@ impl Drop for OperationRegistry { // Best-effort teardown: cancel pending operations so the owning // drivers can release resources. The summary is intentionally ignored; // counting failures is irrelevant while the registry is being dropped. - let _ = self.cancel_all(OperationCancelReason::VmReset); + let _ = self.cancel_all(OperationCancelReason::VmDrop); } } diff --git a/src/vm/resource/close.rs b/src/vm/resource/close.rs index ad6f8cdd..95137ee8 100644 --- a/src/vm/resource/close.rs +++ b/src/vm/resource/close.rs @@ -33,6 +33,9 @@ pub enum CloseProgress { /// `begin_close` returns [`CloseProgress::Pending`]. /// - A concrete `Drop` remains the last-resort guard, but the VM may only reuse /// a resource and its slot once `poll_close` completes. +/// - Abrupt Drop must stay nonblocking. Resources that require asynchronous +/// cleanup may issue an immediate best-effort cancellation/kill request, but +/// must not poll, wait, or enter an executor from Drop. /// /// The `Any` supertrait lets the table reconnect each erased value to its /// concrete `TypeId` without ever naming a concrete class. diff --git a/src/vm/resource/error.rs b/src/vm/resource/error.rs index ed8b0421..a182d698 100644 --- a/src/vm/resource/error.rs +++ b/src/vm/resource/error.rs @@ -62,6 +62,10 @@ pub enum ResourceErrorCode { /// quiescence (at least one remains pending) and so must not claim /// success. ResourceClosePending, + /// Cleanup requires a live Tokio runtime in the caller's current context. + /// The resource remains owned and the close may be polled again after the + /// embedding re-enters and drives that runtime. + RuntimeRequired, } impl ResourceErrorCode { @@ -83,10 +87,23 @@ impl ResourceErrorCode { Self::ResourceNotClosing => "resource_not_closing", Self::ResourceCloseInProgress => "resource_close_in_progress", Self::ResourceClosePending => "resource_close_pending", + Self::RuntimeRequired => "runtime_required", } } } +/// Whether a resource-close failure consumes the resource or permits a later +/// poll to retry the same cleanup state. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum ResourceErrorDisposition { + /// The cleanup attempt is terminal; normal close sweeps reclaim the slot. + #[default] + Terminal, + /// The cleanup attempt could not run in the current context. The table + /// retains the resource in `Closing` so a later poll can retry it. + Retryable, +} + /// A structured, human- and machine-readable resource error. /// /// `code` is the stable machine category, `operation` is the VM scope name the @@ -99,6 +116,7 @@ pub struct ResourceError { message: String, limit: Option, value: Option, + disposition: ResourceErrorDisposition, } impl ResourceError { @@ -114,6 +132,7 @@ impl ResourceError { message: message.into(), limit: None, value: None, + disposition: ResourceErrorDisposition::Terminal, } } @@ -142,6 +161,16 @@ impl ResourceError { self.value } + /// How a resource table must treat this error during close. + pub fn disposition(&self) -> ResourceErrorDisposition { + self.disposition + } + + /// Whether cleanup can be retried without discarding the resource. + pub fn is_retryable(&self) -> bool { + self.disposition == ResourceErrorDisposition::Retryable + } + /// Attaches an optional capacity/limit payload. pub fn with_limit(mut self, limit: usize) -> Self { self.limit = Some(limit); @@ -153,6 +182,13 @@ impl ResourceError { self.value = Some(value); self } + + /// Marks this cleanup failure as retryable. Close machinery must retain + /// the concrete resource and all of its owned state when returning it. + pub fn with_retryable_disposition(mut self) -> Self { + self.disposition = ResourceErrorDisposition::Retryable; + self + } } impl fmt::Display for ResourceError { diff --git a/src/vm/resource/table.rs b/src/vm/resource/table.rs index 26bc170a..0f1ebb5d 100644 --- a/src/vm/resource/table.rs +++ b/src/vm/resource/table.rs @@ -356,8 +356,9 @@ impl ResourceTable { /// Polls one in-progress close to completion. /// /// Returns `Ready(Ok(()))` on a clean finish, `Ready(Err(_))` on a cleanup - /// failure (the slot is still reclaimed), or `Pending` while the resource - /// needs more time. + /// failure, or `Pending` while the resource needs more time. Terminal + /// failures reclaim the slot; a retryable failure retains the concrete + /// resource in `Closing` for a later poll. pub fn poll_close( &mut self, resource: Resource, @@ -370,6 +371,10 @@ impl ResourceTable { let state = self.replace_slot_state(slot_index, SlotState::Vacant); match state { SlotState::Closing(mut resource) => match resource.poll_close(cx) { + Poll::Ready(Err(error)) if error.is_retryable() => { + self.put_slot_state(slot_index, SlotState::Closing(resource)); + Poll::Ready(Err(error)) + } Poll::Ready(result) => { self.reclaim(slot_index); Poll::Ready(result) @@ -393,18 +398,20 @@ impl ResourceTable { /// /// This is the event-driven close-all: unlike a synchronous sweep it can /// wait on genuinely `Pending` resources using the caller's waker. A - /// cleanup failure does not stop the remaining best-effort closes: every - /// resource close is attempted and the first failure is retained until the - /// whole sweep finishes. + /// terminal cleanup failure does not stop the remaining best-effort closes: + /// every resource close is attempted and the first failure is retained + /// until the whole sweep finishes. A retryable cleanup error returns + /// immediately and retains that resource plus the in-flight sweep state. /// /// Contract: - /// - Returns [`Poll::Ready`] **only** once the table is quiescent - /// ([`len`](ResourceTable::len) `== 0`). `Ready(Ok(n))` reports the - /// cumulative number of resources closed across all polls; `Ready(Err)` - /// reports the first cleanup failure once every resource has finished. - /// - Returns [`Poll::Pending`] whenever any Open or Closing resource - /// remains. The cumulative closed count, the first cleanup error, and the - /// initial `reason` are persisted across Pending polls. + /// - Returns [`Poll::Ready`] once the table is quiescent, or immediately + /// for a retryable context error while retaining the affected resource. + /// `Ready(Ok(n))` reports the cumulative number of resources closed across + /// all polls; a terminal `Ready(Err)` reports the first cleanup failure + /// once every resource has finished. + /// - Returns [`Poll::Pending`] when resources remain and no retryable error + /// needs to be reported to the caller. The cumulative closed count, first + /// terminal cleanup error, and initial `reason` persist across polls. /// - The `reason` is bound on the first poll of a sweep. Supplying a /// conflicting reason is rejected deterministically with /// [`ResourceErrorCode::ResourceCloseInProgress`] and leaves the in-flight @@ -474,18 +481,35 @@ impl ResourceTable { progressed = false; let open_indices = self.open_indices()?; for slot_index in open_indices { - progressed |= self.try_begin_close( + match self.try_begin_close( slot_index, reason, &mut closed, &mut failed, &mut first_error, - ); + ) { + Ok(made_progress) => progressed |= made_progress, + Err(error) => { + self.persist_close_all_progress(closed, failed, first_error); + return Poll::Ready(Err(error)); + } + } } let closing_indices = self.closing_indices()?; for slot_index in closing_indices { - progressed |= - self.try_poll_close(slot_index, cx, &mut closed, &mut failed, &mut first_error); + match self.try_poll_close( + slot_index, + cx, + &mut closed, + &mut failed, + &mut first_error, + ) { + Ok(made_progress) => progressed |= made_progress, + Err(error) => { + self.persist_close_all_progress(closed, failed, first_error); + return Poll::Ready(Err(error)); + } + } } } @@ -721,29 +745,33 @@ impl ResourceTable { closed: &mut usize, failed: &mut usize, first_error: &mut Option, - ) -> bool { + ) -> ResourceResult { let state = self.replace_slot_state(slot_index, SlotState::Vacant); let SlotState::Open(mut resource) = state else { // Not open (e.g. already closing); restore and report no progress. self.put_slot_state(slot_index, state); - return false; + return Ok(false); }; match resource.begin_close(reason) { Ok(CloseProgress::Ready) => { self.reclaim(slot_index); *closed += 1; - true + Ok(true) } Ok(CloseProgress::Pending) => { self.put_slot_state(slot_index, SlotState::Closing(resource)); - true + Ok(true) + } + Err(error) if error.is_retryable() => { + self.put_slot_state(slot_index, SlotState::Open(resource)); + Err(error) } Err(error) => { self.reclaim(slot_index); *closed += 1; *failed += 1; first_error.get_or_insert(error); - true + Ok(true) } } } @@ -755,13 +783,17 @@ impl ResourceTable { closed: &mut usize, failed: &mut usize, first_error: &mut Option, - ) -> bool { + ) -> ResourceResult { let state = self.replace_slot_state(slot_index, SlotState::Vacant); let SlotState::Closing(mut resource) = state else { self.put_slot_state(slot_index, state); - return false; + return Ok(false); }; match resource.poll_close(cx) { + Poll::Ready(Err(error)) if error.is_retryable() => { + self.put_slot_state(slot_index, SlotState::Closing(resource)); + Err(error) + } Poll::Ready(result) => { self.reclaim(slot_index); *closed += 1; @@ -769,15 +801,30 @@ impl ResourceTable { *failed += 1; first_error.get_or_insert(error); } - true + Ok(true) } Poll::Pending => { self.put_slot_state(slot_index, SlotState::Closing(resource)); - false + Ok(false) } } } + fn persist_close_all_progress( + &mut self, + closed: usize, + failed: usize, + first_error: Option, + ) { + let state = self + .close_all + .as_mut() + .expect("close-all progress exists during a sweep"); + state.closed = closed; + state.failed = failed; + state.first_error = first_error; + } + fn reclaim(&mut self, slot_index: usize) { self.put_slot_state(slot_index, SlotState::Vacant); if u64::from(self.slots[slot_index].generation.get()) < MAX_HANDLE_GENERATION { @@ -971,14 +1018,10 @@ impl ResourceTable { impl Drop for ResourceTable { fn drop(&mut self) { - // Best-effort last-resort cleanup with a no-op waker. This performs at - // most one synchronous sweep; it explicitly does NOT claim quiescence. - // In the intended flow the owning scope drives poll-based close to - // quiescence via `poll_close_all` before dropping the table, so this - // path only catches resources whose close was never driven. Genuinely - // event-driven Pending resources may remain live here and are released - // by their own `Drop` guards. - let _ = self.close_all(ResourceCloseReason::VmReset); + // Abrupt table teardown only begins immediate resource cancellation. + // Pending cleanup futures are never polled from Drop, and no eventual + // quiescence or process reap is promised on this path. + let _ = self.begin_close_remaining_for_drop(ResourceCloseReason::VmDrop); } } diff --git a/tests/builtins/io_async_tests.rs b/tests/builtins/io_async_tests.rs index eba71879..3497dd73 100644 --- a/tests/builtins/io_async_tests.rs +++ b/tests/builtins/io_async_tests.rs @@ -6,8 +6,6 @@ use vm::{ Value, Vm, VmError, VmResult, VmStatus, compile_source, }; -use super::vm_reset::reset_for_reuse_to_ready; - fn run_source(source: &str) -> Result, VmError> { let compiled = compile_source(&format!("use io;\n{source}")).expect("async io source should compile"); @@ -304,13 +302,44 @@ fn async_io_reset_kills_and_reaps_the_entire_popen_process_group() { descendant: Some(descendant_pid), }; - tokio::runtime::Runtime::new() + let clear_error = vm + .clear_async_bridge() + .expect_err("a live process resource must retain its async bridge"); + assert!(clear_error.to_string().contains("execution scope")); + + let reset_error = vm + .reset_for_reuse() + .expect_err("off-runtime process reset must be retryable"); + let VmError::ExecutionScope(scope_error) = reset_error else { + panic!("expected a resource-domain reset error, got {reset_error:?}"); + }; + let resource_error = scope_error + .into_resource_error() + .expect("runtime requirement must be a resource error"); + assert_eq!( + resource_error.code(), + vm::resource::error::ResourceErrorCode::RuntimeRequired + ); + assert!(resource_error.is_retryable()); + assert!(vm.scope_reset_pending()); + assert!(!vm.is_reusable()); + assert_eq!(vm.execution_scope().resources().len(), 1); + assert!(vm.execution_scope().is_closing()); + vm.clear_async_bridge() + .expect_err("a pending reset must retain its async bridge"); + + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() .expect("reset runtime should build") - .block_on(async { - reset_for_reuse_to_ready(&mut vm).expect("reset should reach quiescence"); - }); + .block_on(std::future::poll_fn(|cx| vm.poll_reset_for_reuse(cx))) + .expect("retry under a live Tokio runtime should quiesce the reset"); assert!(vm.execution_scope().resources().is_empty()); assert!(vm.execution_scope().operations().is_empty()); + assert!(!vm.scope_reset_pending()); + assert!(vm.is_reusable()); + vm.clear_async_bridge() + .expect("a quiescent VM may release its async bridge"); assert!( !marker_path.exists(), "a killed process group must not run descendants" diff --git a/tests/builtins_tests.rs b/tests/builtins_tests.rs index 9e29fc98..6cb050b6 100644 --- a/tests/builtins_tests.rs +++ b/tests/builtins_tests.rs @@ -1,5 +1,6 @@ #![cfg(feature = "runtime")] +#[cfg(any(not(feature = "async"), feature = "sqlite"))] #[path = "support/vm_reset.rs"] mod vm_reset; diff --git a/tests/vm/execution_scope_tests.rs b/tests/vm/execution_scope_tests.rs index 858f3629..ad495dac 100644 --- a/tests/vm/execution_scope_tests.rs +++ b/tests/vm/execution_scope_tests.rs @@ -21,6 +21,7 @@ use vm::resource::ResourceCloseReason; use vm::resource::ResourceTable; use vm::resource::close::{CloseProgress, HostResource}; use vm::resource::error::{ResourceErrorCode, ResourceResult}; +use vm::{OpCode, Program, Vm}; // ---------------------------------------------------------------- helpers @@ -113,6 +114,45 @@ impl HostOperation for CancelAwareWorker { } } +struct DropPendingResource { + begins: Arc, + polls: Arc, +} + +impl HostResource for DropPendingResource { + fn begin_close(&mut self, _reason: ResourceCloseReason) -> ResourceResult { + self.begins.fetch_add(1, Ordering::SeqCst); + Ok(CloseProgress::Pending) + } + + fn poll_close(&mut self, _cx: &mut Context<'_>) -> Poll> { + self.polls.fetch_add(1, Ordering::SeqCst); + Poll::Pending + } +} + +struct DropPendingOperation { + cancels: Arc, + polls: Arc, +} + +impl HostOperation for DropPendingOperation { + fn poll(&mut self, _cx: &mut Context<'_>) -> Poll> { + self.polls.fetch_add(1, Ordering::SeqCst); + Poll::Pending + } + + fn cancel(&mut self, _reason: OperationCancelReason) -> OperationResult<()> { + self.cancels.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn poll_quiescent(&mut self, _cx: &mut Context<'_>) -> Poll<()> { + self.polls.fetch_add(1, Ordering::SeqCst); + Poll::Pending + } +} + // ------------------------------------------------------------------ scope #[test] @@ -195,6 +235,50 @@ fn empty_scope_quiesces_cleanly() { } } +#[test] +fn vm_drop_begins_resource_cleanup_without_polling_it() { + let begins = Arc::new(AtomicUsize::new(0)); + let polls = Arc::new(AtomicUsize::new(0)); + let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8])); + vm.execution_scope() + .push_resource(DropPendingResource { + begins: Arc::clone(&begins), + polls: Arc::clone(&polls), + }) + .expect("resource"); + + drop(vm); + + assert_eq!(begins.load(Ordering::SeqCst), 1); + assert_eq!( + polls.load(Ordering::SeqCst), + 0, + "VM Drop must not poll resource cleanup futures" + ); +} + +#[test] +fn scope_drop_cancels_operations_without_polling_quiescence() { + let cancels = Arc::new(AtomicUsize::new(0)); + let polls = Arc::new(AtomicUsize::new(0)); + let mut scope = ExecutionScope::new().expect("scope"); + scope + .start_operation(OperationSpec::new(DropPendingOperation { + cancels: Arc::clone(&cancels), + polls: Arc::clone(&polls), + })) + .expect("operation"); + + drop(scope); + + assert_eq!(cancels.load(Ordering::SeqCst), 1); + assert_eq!( + polls.load(Ordering::SeqCst), + 0, + "scope Drop must not poll operation futures or quiescence" + ); +} + #[test] fn poll_close_stays_pending_until_operation_worker_quiesces() { let mut scope = ExecutionScope::new().expect("scope"); From 5b01424505184e73d420ac25ccef70ce10b26dce Mon Sep 17 00:00:00 2001 From: fffonion Date: Sun, 20 Sep 2026 04:25:34 +0800 Subject: [PATCH 17/23] fix(vm): require async quiescence for bridge mutation --- src/vm/host.rs | 55 ++------ src/vm/host_runtime.rs | 20 +++ src/vm/tests.rs | 265 ++++++++++++++++++++++++++++++++++++- tests/vm/http_sse_tests.rs | 11 ++ 4 files changed, 307 insertions(+), 44 deletions(-) diff --git a/src/vm/host.rs b/src/vm/host.rs index 1b0520ab..0df2190a 100644 --- a/src/vm/host.rs +++ b/src/vm/host.rs @@ -2701,24 +2701,20 @@ impl Vm { .insert(builtin_call_index, host_slot); } - pub fn set_async_bridge(&mut self, bridge: Box) -> VmResult<()> { - if self.host.has_active_bridge_operations() - || self - .instance - .waiting_host_op - .as_ref() - .is_some_and(|waiting| { - matches!( - waiting.source, - crate::vm::host::WaitingHostOpSource::HostBridge - ) - }) + fn ensure_async_bridge_mutation_is_quiescent(&self, action: &str) -> VmResult<()> { + if self.instance.waiting_host_op.is_some() + || self.instance.host_stream.is_some() + || !self.host.async_work_is_quiescent() { - return Err(VmError::HostError( - "cannot replace async bridge while an active host operation is present".to_string(), - )); + return Err(VmError::HostError(format!( + "cannot {action} async bridge while VM async work, an active host operation, or execution scope lifecycle is not quiescent" + ))); } - self.cancel_waiting_host_op_with_reason(OperationCancelReason::Requested)?; + Ok(()) + } + + pub fn set_async_bridge(&mut self, bridge: Box) -> VmResult<()> { + self.ensure_async_bridge_mutation_is_quiescent("replace")?; self.host.async_bridge = Some(bridge); Ok(()) } @@ -2730,32 +2726,7 @@ impl Vm { /// process-backed async IO may still need the embedding Tokio runtime to /// complete cleanup and reap its direct child. pub fn clear_async_bridge(&mut self) -> VmResult<()> { - if self.host.scope_reset_pending - || !self.host.execution_scope.resources().is_empty() - || !self.host.execution_scope.operations().is_empty() - { - return Err(VmError::HostError( - "cannot clear async bridge while the execution scope has live resources, operations, or a pending reset" - .to_string(), - )); - } - if self.host.has_active_bridge_operations() - || self - .instance - .waiting_host_op - .as_ref() - .is_some_and(|waiting| { - matches!( - waiting.source, - crate::vm::host::WaitingHostOpSource::HostBridge - ) - }) - { - return Err(VmError::HostError( - "cannot clear async bridge while an active host operation is present".to_string(), - )); - } - self.cancel_waiting_host_op_with_reason(OperationCancelReason::Requested)?; + self.ensure_async_bridge_mutation_is_quiescent("clear")?; self.host.async_bridge = None; Ok(()) } diff --git a/src/vm/host_runtime.rs b/src/vm/host_runtime.rs index 1c205c94..b9e37364 100644 --- a/src/vm/host_runtime.rs +++ b/src/vm/host_runtime.rs @@ -241,6 +241,26 @@ impl HostRuntime { !self.bridge_operations.is_empty() } + /// Whether no host-owned state can still depend on the installed async + /// bridge or its runtime. + /// + /// Keep bridge replacement and removal on this one lifecycle predicate so + /// newly introduced operation/resource owners cannot drift between the two + /// public mutation paths. + pub(crate) fn async_work_is_quiescent(&self) -> bool { + !self.scope_reset_pending + && self.scope_reset_error.is_none() + && self.reset_error.is_none() + && self.replacement_execution_scope.is_none() + && self.execution_scope.resources().is_clean() + && self.execution_scope.operations().is_empty() + && self.submitted_host_ops.is_empty() + && self.bridge_operations.is_empty() + && self.scoped_operation_completions.is_empty() + && self.stream_drivers.is_empty() + && self.pending_stream_terminations.is_empty() + } + pub(crate) fn has_pending_bridge_cancellations(&self) -> bool { self.bridge_operations .values() diff --git a/src/vm/tests.rs b/src/vm/tests.rs index 87256fc1..14c0b27f 100644 --- a/src/vm/tests.rs +++ b/src/vm/tests.rs @@ -10,7 +10,7 @@ use crate::vm::operation::driver::{HostOperation, OperationSpec}; use crate::vm::operation::{OperationCancelReason, OperationResult}; use crate::{BytecodeBuilder, decode_program, encode_program}; use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; use std::task::{Context, Poll, Waker}; @@ -2708,6 +2708,148 @@ fn manual_external_pending_without_bridge_is_untracked_and_resets_cleanly() { assert!(vm.is_reusable()); } +struct DropTrackingBridge { + drops: Arc, +} + +impl Drop for DropTrackingBridge { + fn drop(&mut self) { + self.drops.fetch_add(1, Ordering::SeqCst); + } +} + +impl HostAsyncBridge for DropTrackingBridge { + fn submit_op(&mut self, _op_id: HostOpId, _future: HostFuture) -> VmResult<()> { + Ok(()) + } + + fn poll_op(&mut self, _op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { + Poll::Pending + } + + fn poll_submitted_op( + &mut self, + _op_id: HostOpId, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Pending + } + + fn request_cancel_op( + &mut self, + _op_id: HostOpId, + _reason: OperationCancelReason, + ) -> VmResult<()> { + Ok(()) + } + + fn poll_cancel_op(&mut self, _op_id: HostOpId, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn cleanup_op(&mut self, _op_id: HostOpId, _terminal: HostAsyncOpTerminal) -> VmResult<()> { + Ok(()) + } +} + +fn vm_with_drop_tracking_bridge() -> (Vm, Arc) { + let drops = Arc::new(AtomicUsize::new(0)); + let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8])); + vm.set_async_bridge(Box::new(DropTrackingBridge { + drops: Arc::clone(&drops), + })) + .expect("initial bridge should install"); + (vm, drops) +} + +fn assert_bridge_mutations_rejected(vm: &mut Vm, installed_drops: &AtomicUsize) { + let rejected_drops = Arc::new(AtomicUsize::new(0)); + vm.set_async_bridge(Box::new(DropTrackingBridge { + drops: Arc::clone(&rejected_drops), + })) + .expect_err("bridge replacement must wait for complete VM async quiescence"); + assert_eq!( + installed_drops.load(Ordering::SeqCst), + 0, + "a rejected replacement must retain the runtime-owning bridge" + ); + assert_eq!(rejected_drops.load(Ordering::SeqCst), 1); + vm.clear_async_bridge() + .expect_err("bridge clear must wait for complete VM async quiescence"); + assert_eq!( + installed_drops.load(Ordering::SeqCst), + 0, + "a rejected clear must retain the runtime-owning bridge" + ); +} + +fn assert_bridge_mutations_succeed(vm: &mut Vm, installed_drops: &AtomicUsize) { + let replacement_drops = Arc::new(AtomicUsize::new(0)); + vm.set_async_bridge(Box::new(DropTrackingBridge { + drops: Arc::clone(&replacement_drops), + })) + .expect("a quiescent VM may replace its async bridge"); + assert_eq!(installed_drops.load(Ordering::SeqCst), 1); + vm.clear_async_bridge() + .expect("a quiescent VM may clear its async bridge"); + assert_eq!(replacement_drops.load(Ordering::SeqCst), 1); +} + +#[test] +fn every_waiting_source_blocks_bridge_replacement_and_clear_until_released() { + for (index, source) in [ + WaitingHostOpSource::HostBridge, + WaitingHostOpSource::Manual, + WaitingHostOpSource::ScopedOperation, + WaitingHostOpSource::CallableStream, + WaitingHostOpSource::CallableStreamTermination, + ] + .into_iter() + .enumerate() + { + let (mut vm, installed_drops) = vm_with_drop_tracking_bridge(); + vm.instance.waiting_host_op = Some(crate::vm::host::WaitingHostOp { + op_id: index as HostOpId + 1, + source, + expected_return_type: None, + expected_return_schema: None, + }); + + assert_bridge_mutations_rejected(&mut vm, &installed_drops); + + vm.instance.waiting_host_op = None; + assert_bridge_mutations_succeed(&mut vm, &installed_drops); + } +} + +#[test] +fn fresh_and_halted_quiescent_vms_allow_bridge_mutation() { + let (mut fresh, fresh_bridge_drops) = vm_with_drop_tracking_bridge(); + assert_bridge_mutations_succeed(&mut fresh, &fresh_bridge_drops); + + let (mut halted, halted_bridge_drops) = vm_with_drop_tracking_bridge(); + assert_eq!( + halted.run().expect("empty program should run"), + VmStatus::Halted + ); + assert_bridge_mutations_succeed(&mut halted, &halted_bridge_drops); +} + +#[test] +fn active_bridge_operation_blocks_bridge_mutation_until_reset_quiesces() { + let (mut vm, installed_drops) = vm_with_drop_tracking_bridge(); + vm.submit_host_future(Box::pin(async { + Ok(HostFutureOutput::returning(CallReturn::none())) + })) + .expect("host future should enter bridge ownership"); + + assert_bridge_mutations_rejected(&mut vm, &installed_drops); + + vm.reset_for_reuse() + .expect("reset should quiesce the bridge operation"); + assert_bridge_mutations_succeed(&mut vm, &installed_drops); +} + struct ScopedPendingDriver { cancellations: Arc>>, } @@ -2730,6 +2872,100 @@ impl HostOperation for ScopedPendingDriver { } } +#[test] +fn live_scope_resource_and_operation_each_block_bridge_mutation() { + let (mut resource_vm, resource_bridge_drops) = vm_with_drop_tracking_bridge(); + let resource = resource_vm + .execution_scope() + .push_resource(QuiescentAdmissionResource) + .expect("resource should enter the execution scope"); + assert_bridge_mutations_rejected(&mut resource_vm, &resource_bridge_drops); + resource_vm + .execution_scope() + .take_resource::(resource.handle()) + .expect("resource should leave the execution scope"); + assert_bridge_mutations_succeed(&mut resource_vm, &resource_bridge_drops); + + let (mut operation_vm, operation_bridge_drops) = vm_with_drop_tracking_bridge(); + let operation = operation_vm + .execution_scope() + .start_operation(OperationSpec::new(ScopedPendingDriver { + cancellations: Arc::new(Mutex::new(Vec::new())), + })) + .expect("operation should enter the execution scope"); + assert_bridge_mutations_rejected(&mut operation_vm, &operation_bridge_drops); + operation_vm + .execution_scope() + .abort_operation(operation, OperationCancelReason::Requested) + .expect("operation should quiesce"); + assert_bridge_mutations_succeed(&mut operation_vm, &operation_bridge_drops); +} + +struct PendingTerminationDriver { + ready: Arc, +} + +impl crate::vm::async_host::HostStreamDriver for PendingTerminationDriver { + fn poll_next( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Pending + } + + fn apply_action( + &mut self, + _action: Value, + ) -> VmResult { + Ok(crate::vm::async_host::HostStreamAction::Continue) + } + + fn poll_termination( + &mut self, + _scope: &mut crate::vm::execution_scope::ExecutionScope, + _termination: crate::vm::async_host::HostStreamTermination, + _cx: &mut Context<'_>, + ) -> Poll> { + if self.ready.load(Ordering::SeqCst) { + Poll::Ready(Ok(())) + } else { + Poll::Pending + } + } +} + +#[test] +fn callable_stream_driver_and_pending_termination_block_bridge_mutation() { + let ready = Arc::new(AtomicBool::new(false)); + let (mut vm, installed_drops) = vm_with_drop_tracking_bridge(); + let op_id = vm.allocate_host_op_id(); + vm.host.stream_drivers.insert( + op_id, + Box::new(PendingTerminationDriver { + ready: Arc::clone(&ready), + }), + ); + + assert_bridge_mutations_rejected(&mut vm, &installed_drops); + vm.host + .begin_stream_termination( + op_id, + crate::vm::async_host::HostStreamTermination::Cancelled( + OperationCancelReason::Requested, + ), + ) + .expect("stream termination should start"); + assert_bridge_mutations_rejected(&mut vm, &installed_drops); + + ready.store(true, Ordering::SeqCst); + let mut cx = Context::from_waker(Waker::noop()); + assert!(matches!( + vm.host.poll_stream_terminations(&mut cx), + Poll::Ready(Ok(())) + )); + assert_bridge_mutations_succeed(&mut vm, &installed_drops); +} + #[test] fn manually_completing_scoped_op_retires_operation_and_completion() { let cancellations = Arc::new(Mutex::new(Vec::new())); @@ -2992,6 +3228,30 @@ fn reset_retains_one_replacement_scope_until_old_scope_quiesces() { assert_eq!(REGISTRY_SOURCE.load(Ordering::SeqCst), 3); } +#[test] +fn pending_scope_reset_blocks_bridge_mutation_until_quiescent() { + let ready = Arc::new(AtomicBool::new(false)); + let (mut vm, installed_drops) = vm_with_drop_tracking_bridge(); + vm.execution_scope() + .push_resource(PendingResetResource { + ready: Arc::clone(&ready), + }) + .expect("pending resource should enter the active scope"); + vm.reset_for_reuse() + .expect("reset should remain pending while the resource closes"); + assert!(vm.scope_reset_pending()); + + assert_bridge_mutations_rejected(&mut vm, &installed_drops); + + ready.store(true, Ordering::SeqCst); + let mut cx = Context::from_waker(Waker::noop()); + assert!(matches!( + vm.poll_reset_for_reuse(&mut cx), + Poll::Ready(Ok(())) + )); + assert_bridge_mutations_succeed(&mut vm, &installed_drops); +} + #[test] fn replacement_scope_allocation_failure_is_terminal_and_not_reusable() { static ARENA_SOURCE: AtomicU64 = AtomicU64::new(1); @@ -3000,7 +3260,7 @@ fn replacement_scope_allocation_failure_is_terminal_and_not_reusable() { Ordering::SeqCst, ); - let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8])); + let (mut vm, installed_drops) = vm_with_drop_tracking_bridge(); let _arena_source = crate::vm::resource::table::test_seam::ScopedArenaSource::install(&ARENA_SOURCE); @@ -3031,6 +3291,7 @@ fn replacement_scope_allocation_failure_is_terminal_and_not_reusable() { !vm.is_reusable(), "重复 poll 后 terminal reset error 仍应保持" ); + assert_bridge_mutations_rejected(&mut vm, &installed_drops); } #[test] diff --git a/tests/vm/http_sse_tests.rs b/tests/vm/http_sse_tests.rs index 42a79c1f..ced49f19 100644 --- a/tests/vm/http_sse_tests.rs +++ b/tests/vm/http_sse_tests.rs @@ -1108,9 +1108,20 @@ async fn sse_reset_while_callback_waits_retires_stream_to_quiescence() { assert!(matches!(vm.resume().unwrap(), VmStatus::Waiting(_))); assert_eq!(wait_calls.load(Ordering::SeqCst), 1); + vm.set_async_bridge(Box::::default()) + .expect_err("an active SSE callable stream must retain its async bridge"); + vm.clear_async_bridge() + .expect_err("an active SSE callable stream must retain its async bridge"); + reset_and_wait(&mut vm) .await .expect("reset must cancel callback and retire the stream"); + vm.set_async_bridge(Box::::default()) + .expect("a quiescent SSE VM may replace its async bridge"); + vm.clear_async_bridge() + .expect("a quiescent SSE VM may clear its async bridge"); + vm.set_async_bridge(Box::::default()) + .expect("the reused SSE VM needs an async bridge"); drive(&mut vm) .await .expect("the reused VM must reacquire the permit"); From 1c3ba703f03e7bd0d8de6cfcf9ef904f463a3c46 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sun, 20 Sep 2026 04:48:40 +0800 Subject: [PATCH 18/23] fix(io): keep blocking process drop nonblocking --- src/builtins/runtime/io/async_io.rs | 94 ++++-- src/builtins/runtime/io/blocking.rs | 348 +++++++++++++++++++-- src/vm/resource/close.rs | 16 + src/vm/resource/table.rs | 8 +- tests/builtins/io_async_tests.rs | 8 +- tests/builtins/io_scope_lifecycle_tests.rs | 45 ++- tests/vm/execution_scope_tests.rs | 14 +- 7 files changed, 467 insertions(+), 66 deletions(-) diff --git a/src/builtins/runtime/io/async_io.rs b/src/builtins/runtime/io/async_io.rs index fe114304..817e4a4e 100644 --- a/src/builtins/runtime/io/async_io.rs +++ b/src/builtins/runtime/io/async_io.rs @@ -47,12 +47,7 @@ pub(crate) enum IoHandle { impl Drop for IoHandle { fn drop(&mut self) { - match self { - Self::PopenRead { child, .. } | Self::PopenWrite { child, .. } => { - let _ = start_terminate_child_tree_for_drop(child); - } - Self::File(_) => {} - } + start_close_io_handle_for_drop(self); } } @@ -95,6 +90,7 @@ fn prepare_process_cleanup( struct IoResource { handle: Arc>>, process_tree_terminator: fn(u32) -> io::Result<()>, + drop_process_tree_signaler: fn(u32) -> io::Result<()>, process_tree_cleanup: Option, process_tree_cleanup_complete: bool, leader_kill_started: bool, @@ -106,6 +102,7 @@ impl IoResource { Self { handle: Arc::new(Mutex::new(Some(handle))), process_tree_terminator: signal_process_tree_for_drop, + drop_process_tree_signaler: signal_process_tree_for_drop, process_tree_cleanup: None, process_tree_cleanup_complete: false, leader_kill_started: false, @@ -113,7 +110,7 @@ impl IoResource { } } - #[cfg(test)] + #[cfg(all(test, unix))] fn new_with_process_tree_terminator( handle: IoHandle, process_tree_terminator: fn(u32) -> io::Result<()>, @@ -121,6 +118,24 @@ impl IoResource { Self { handle: Arc::new(Mutex::new(Some(handle))), process_tree_terminator, + drop_process_tree_signaler: signal_process_tree_for_drop, + process_tree_cleanup: None, + process_tree_cleanup_complete: false, + leader_kill_started: false, + deferred_process_cleanup_error: None, + } + } + + #[cfg(all(test, windows))] + fn new_with_process_terminators( + handle: IoHandle, + process_tree_terminator: fn(u32) -> io::Result<()>, + drop_process_tree_signaler: fn(u32) -> io::Result<()>, + ) -> Self { + Self { + handle: Arc::new(Mutex::new(Some(handle))), + process_tree_terminator, + drop_process_tree_signaler, process_tree_cleanup: None, process_tree_cleanup_complete: false, leader_kill_started: false, @@ -167,6 +182,21 @@ impl IoResource { Ok(progress) } + fn begin_close_for_drop_nonblocking(&mut self) -> CloseProgress { + let Some(slot) = Arc::get_mut(&mut self.handle).map(Mutex::get_mut) else { + return CloseProgress::Pending; + }; + let Some(handle) = slot.as_mut() else { + return CloseProgress::Ready; + }; + if matches!(handle, IoHandle::File(_)) { + slot.take(); + return CloseProgress::Ready; + } + let _ = start_close_io_handle_for_drop_with(handle, self.drop_process_tree_signaler); + CloseProgress::Pending + } + fn poll_process_close(&mut self, cx: &mut Context<'_>) -> Poll> { let Self { handle, @@ -234,11 +264,7 @@ impl IoResource { impl Drop for IoResource { fn drop(&mut self) { - if let Some(mutex) = Arc::get_mut(&mut self.handle) - && let Some(handle) = mutex.get_mut().as_mut() - { - start_close_io_handle_for_drop(handle); - } + let _ = self.begin_close_for_drop_nonblocking(); } } @@ -257,6 +283,13 @@ impl HostResource for IoResource { self.begin_close_after_operations_quiesce() } + fn begin_close_for_drop( + &mut self, + _reason: ResourceCloseReason, + ) -> ResourceResult { + Ok(self.begin_close_for_drop_nonblocking()) + } + fn poll_close(&mut self, cx: &mut Context<'_>) -> Poll> { self.poll_process_close(cx) } @@ -280,10 +313,17 @@ fn runtime_required_resource_error() -> ResourceError { } fn start_close_io_handle_for_drop(handle: &mut IoHandle) { + let _ = start_close_io_handle_for_drop_with(handle, signal_process_tree_for_drop); +} + +fn start_close_io_handle_for_drop_with( + handle: &mut IoHandle, + signal_tree: fn(u32) -> io::Result<()>, +) -> io::Result<()> { match handle { - IoHandle::File(_) => {} + IoHandle::File(_) => Ok(()), IoHandle::PopenRead { child, .. } | IoHandle::PopenWrite { child, .. } => { - let _ = start_terminate_child_tree_for_drop(child); + start_terminate_child_tree_with(child, signal_tree) } } } @@ -929,12 +969,12 @@ mod tests { use super::*; #[cfg(windows)] - static DROP_TREE_TERMINATION_CALLS: std::sync::atomic::AtomicUsize = + static DROP_TREE_SIGNAL_CALLS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); #[cfg(windows)] - fn record_drop_tree_termination(_pid: u32) -> io::Result<()> { - DROP_TREE_TERMINATION_CALLS.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + fn record_drop_tree_signal(_pid: u32) -> io::Result<()> { + DROP_TREE_SIGNAL_CALLS.fetch_add(1, std::sync::atomic::Ordering::SeqCst); Ok(()) } @@ -994,7 +1034,7 @@ mod tests { ); } - #[cfg(unix)] + #[cfg(target_os = "linux")] #[tokio::test(flavor = "current_thread")] async fn cancelling_process_close_retains_child_and_allows_retry_to_reap() { let resource = @@ -1035,22 +1075,24 @@ mod tests { #[cfg(windows)] #[tokio::test(flavor = "current_thread")] - async fn dropping_process_resource_does_not_run_synchronous_tree_termination() { - DROP_TREE_TERMINATION_CALLS.store(0, std::sync::atomic::Ordering::SeqCst); - let resource = IoResource::new_with_process_tree_terminator( + async fn dropping_process_resource_exercises_only_the_drop_signal_helper() { + DROP_TREE_SIGNAL_CALLS.store(0, std::sync::atomic::Ordering::SeqCst); + let resource = IoResource::new_with_process_terminators( spawn_shell_command("ping -n 30 127.0.0.1 >NUL", "r").expect("process should spawn"), - record_drop_tree_termination, + |_| panic!("Drop must not invoke taskkill or explicit-close waiting"), + record_drop_tree_signal, ); drop(resource); assert_eq!( - DROP_TREE_TERMINATION_CALLS.load(std::sync::atomic::Ordering::SeqCst), - 0, - "Drop must use only the direct nonblocking child kill on Windows" + DROP_TREE_SIGNAL_CALLS.load(std::sync::atomic::Ordering::SeqCst), + 1, + "Drop must invoke the immediate signal helper" ); } + #[cfg(target_os = "linux")] #[tokio::test] async fn process_resource_close_polls_until_the_leader_is_reaped() { let mut resource = @@ -1078,7 +1120,7 @@ mod tests { ); } - #[cfg(unix)] + #[cfg(target_os = "linux")] #[tokio::test] async fn resource_shutdown_reaps_leader_before_reporting_tree_failure() { let handle = spawn_shell_command("sleep 30", "r").expect("process should spawn"); diff --git a/src/builtins/runtime/io/blocking.rs b/src/builtins/runtime/io/blocking.rs index 49170dc8..d895d0d5 100644 --- a/src/builtins/runtime/io/blocking.rs +++ b/src/builtins/runtime/io/blocking.rs @@ -20,21 +20,69 @@ pub(super) enum IoHandle { /// The typed resource stored in the execution scope for one IO handle. struct IoResource { handle: Option, + explicit_cleanup: fn(&mut IoHandle) -> VmResult<()>, + drop_cleanup: fn(&mut IoHandle) -> VmResult<()>, + drop_cleanup_started: bool, } impl IoResource { fn new(handle: IoHandle) -> Self { Self { handle: Some(handle), + explicit_cleanup: close_io_handle, + drop_cleanup: start_close_io_handle_for_drop, + drop_cleanup_started: false, } } + + #[cfg(test)] + fn new_with_process_cleanup( + handle: IoHandle, + explicit_cleanup: fn(&mut IoHandle) -> VmResult<()>, + drop_cleanup: fn(&mut IoHandle) -> VmResult<()>, + ) -> Self { + Self { + handle: Some(handle), + explicit_cleanup, + drop_cleanup, + drop_cleanup_started: false, + } + } + + fn close_explicit(&mut self) -> VmResult<()> { + let Some(handle) = self.handle.as_mut() else { + return Ok(()); + }; + (self.explicit_cleanup)(handle)?; + self.handle.take(); + Ok(()) + } + + fn begin_close_for_drop_nonblocking(&mut self) -> ResourceResult { + let Some(handle) = self.handle.as_mut() else { + return Ok(CloseProgress::Ready); + }; + if matches!(handle, IoHandle::File(_)) { + self.handle.take(); + return Ok(CloseProgress::Ready); + } + if !self.drop_cleanup_started { + (self.drop_cleanup)(handle).map_err(|error| { + ResourceError::new( + ResourceErrorCode::ResourceCleanupFailed, + "io::resource", + error.to_string(), + ) + })?; + self.drop_cleanup_started = true; + } + Ok(CloseProgress::Pending) + } } impl Drop for IoResource { fn drop(&mut self) { - if let Some(handle) = self.handle.take() { - let _ = close_io_handle(handle); - } + let _ = self.begin_close_for_drop_nonblocking(); } } @@ -50,17 +98,22 @@ pub(crate) fn io_file_resource() -> crate::host_extension::HostResourceTypeMeta impl HostResource for IoResource { fn begin_close(&mut self, _reason: ResourceCloseReason) -> ResourceResult { - if let Some(handle) = self.handle.take() { - close_io_handle(handle).map_err(|error| { - ResourceError::new( - ResourceErrorCode::ResourceCleanupFailed, - "io::resource", - error.to_string(), - ) - })?; - } + self.close_explicit().map_err(|error| { + ResourceError::new( + ResourceErrorCode::ResourceCleanupFailed, + "io::resource", + error.to_string(), + ) + })?; Ok(CloseProgress::Ready) } + + fn begin_close_for_drop( + &mut self, + _reason: ResourceCloseReason, + ) -> ResourceResult { + self.begin_close_for_drop_nonblocking() + } } /// Opens a file handle for runtime I/O inline on non-async builds. @@ -275,18 +328,17 @@ pub(super) fn builtin_io_flush(vm: &mut Vm, handle_id: i64) -> VmResult { pub(super) fn builtin_io_close(vm: &mut Vm, handle_id: i64) -> VmResult { let token = io_resource_for_handle(vm, handle_id)?; let handle = token.handle(); - let owned = { + { let mut resource = vm .execution_scope() .resources_mut() .get_mut(&token) .map_err(|error| io_borrow_error(handle_id, error))?; - resource - .handle - .take() - .ok_or_else(|| VmError::HostError("io handle is closed".to_string()))? - }; - let close_result = close_io_handle(owned); + if resource.handle.is_none() { + return Err(VmError::HostError("io handle is closed".to_string())); + } + resource.close_explicit()?; + } let progress = vm .execution_scope() .close_resource::(handle, ResourceCloseReason::Requested) @@ -298,7 +350,6 @@ pub(super) fn builtin_io_close(vm: &mut Vm, handle_id: i64) -> VmResult { "io_close scope retirement is still pending".to_string(), )); } - close_result?; Ok(true) } @@ -434,8 +485,8 @@ fn spawn_shell_command(command: &str, mode: &str) -> VmResult { } } -fn close_io_handle(mut handle: IoHandle) -> VmResult<()> { - match &mut handle { +fn close_io_handle(handle: &mut IoHandle) -> VmResult<()> { + match handle { IoHandle::File(file) => file .flush() .map_err(|error| VmError::HostError(format!("io_close flush failed: {error}"))), @@ -447,6 +498,18 @@ fn close_io_handle(mut handle: IoHandle) -> VmResult<()> { } } +fn start_close_io_handle_for_drop(handle: &mut IoHandle) -> VmResult<()> { + match handle { + IoHandle::File(_) => Ok(()), + IoHandle::PopenRead { child } => start_terminate_child_tree_for_drop(child), + IoHandle::PopenWrite { child } => { + let _ = child.stdin.take(); + start_terminate_child_tree_for_drop(child) + } + } + .map_err(|error| VmError::HostError(format!("io drop popen terminate failed: {error}"))) +} + fn terminate_child_tree(child: &mut Child) -> VmResult<()> { let pid = child.id(); terminate_process_tree_and_leader( @@ -468,6 +531,21 @@ fn kill_and_reap_child(child: &mut Child) -> io::Result<()> { child.wait().map(|_| ()) } +fn start_terminate_child_tree_for_drop(child: &mut Child) -> io::Result<()> { + let pid = child.id(); + let tree_result = signal_process_tree_for_drop(pid); + let leader_result = child.kill().or_else(ignore_already_exited); + combine_process_cleanup_results(tree_result, leader_result) +} + +fn ignore_already_exited(error: io::Error) -> io::Result<()> { + if error.kind() == io::ErrorKind::InvalidInput { + Ok(()) + } else { + Err(error) + } +} + fn terminate_process_tree_and_leader( terminate_tree: Tree, terminate_leader: Leader, @@ -521,6 +599,25 @@ fn terminate_process_tree(pid: u32) -> io::Result<()> { } } +fn signal_process_tree_for_drop(pid: u32) -> io::Result<()> { + if pid == 0 { + return Ok(()); + } + #[cfg(unix)] + { + terminate_unix_process_group_with( + pid, + |process_group, signal| unsafe { libc::kill(process_group, signal) }, + io::Error::last_os_error, + ) + } + #[cfg(not(unix))] + { + let _ = pid; + Ok(()) + } +} + #[cfg(unix)] fn terminate_unix_process_group_with( pid: u32, @@ -583,9 +680,216 @@ fn read_line_from_reader(reader: &mut impl Read) -> VmResult { mod tests { use std::cell::Cell; use std::io; + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; use super::*; + static DROP_CLEANUP_CALLS: AtomicUsize = AtomicUsize::new(0); + static EXPLICIT_CLEANUP_CALLS: AtomicUsize = AtomicUsize::new(0); + static PROCESS_CLEANUP_TEST_LOCK: Mutex<()> = Mutex::new(()); + #[cfg(windows)] + static WINDOWS_TASKKILL_CALLBACK_CALLS: AtomicUsize = AtomicUsize::new(0); + #[cfg(windows)] + static WINDOWS_WAIT_CALLBACK_CALLS: AtomicUsize = AtomicUsize::new(0); + + fn record_drop_cleanup(_handle: &mut IoHandle) -> VmResult<()> { + DROP_CLEANUP_CALLS.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn record_explicit_cleanup(handle: &mut IoHandle) -> VmResult<()> { + EXPLICIT_CLEANUP_CALLS.fetch_add(1, Ordering::SeqCst); + match handle { + IoHandle::PopenRead { child } | IoHandle::PopenWrite { child } => child + .wait() + .map(|_| ()) + .map_err(|error| VmError::HostError(format!("test child wait failed: {error}"))), + IoHandle::File(_) => panic!("expected process handle"), + } + } + + fn fail_explicit_cleanup(_handle: &mut IoHandle) -> VmResult<()> { + Err(VmError::HostError( + "injected explicit cleanup failure".to_string(), + )) + } + + #[cfg(windows)] + fn record_windows_drop_cleanup(handle: &mut IoHandle) -> VmResult<()> { + DROP_CLEANUP_CALLS.fetch_add(1, Ordering::SeqCst); + start_close_io_handle_for_drop(handle) + } + + #[cfg(windows)] + fn record_windows_explicit_cleanup(handle: &mut IoHandle) -> VmResult<()> { + use std::os::windows::process::ExitStatusExt as _; + + let child = match handle { + IoHandle::PopenRead { child } | IoHandle::PopenWrite { child } => child, + IoHandle::File(_) => panic!("expected process handle"), + }; + run_taskkill_with(child.id(), |_| { + WINDOWS_TASKKILL_CALLBACK_CALLS.fetch_add(1, Ordering::SeqCst); + Ok(std::process::ExitStatus::from_raw(0)) + }) + .map_err(|error| VmError::HostError(format!("test taskkill failed: {error}")))?; + WINDOWS_WAIT_CALLBACK_CALLS.fetch_add(1, Ordering::SeqCst); + child + .wait() + .map(|_| ()) + .map_err(|error| VmError::HostError(format!("test child wait failed: {error}"))) + } + + fn exited_process_handle() -> IoHandle { + let command = if cfg!(windows) { "exit /B 0" } else { "exit 0" }; + spawn_shell_command(command, "r").expect("test process should spawn") + } + + #[test] + fn process_resource_drop_uses_only_nonblocking_cleanup_path() { + let _guard = PROCESS_CLEANUP_TEST_LOCK.lock().expect("test lock"); + DROP_CLEANUP_CALLS.store(0, Ordering::SeqCst); + EXPLICIT_CLEANUP_CALLS.store(0, Ordering::SeqCst); + let resource = IoResource::new_with_process_cleanup( + exited_process_handle(), + record_explicit_cleanup, + record_drop_cleanup, + ); + + drop(resource); + + assert_eq!(DROP_CLEANUP_CALLS.load(Ordering::SeqCst), 1); + assert_eq!(EXPLICIT_CLEANUP_CALLS.load(Ordering::SeqCst), 0); + } + + #[test] + fn successful_explicit_process_cleanup_retires_ready() { + let _guard = PROCESS_CLEANUP_TEST_LOCK.lock().expect("test lock"); + DROP_CLEANUP_CALLS.store(0, Ordering::SeqCst); + EXPLICIT_CLEANUP_CALLS.store(0, Ordering::SeqCst); + let mut resource = IoResource::new_with_process_cleanup( + exited_process_handle(), + record_explicit_cleanup, + record_drop_cleanup, + ); + + resource + .close_explicit() + .expect("explicit cleanup should complete"); + assert_eq!( + resource + .begin_close(ResourceCloseReason::Requested) + .expect("retirement should complete"), + CloseProgress::Ready + ); + drop(resource); + + assert_eq!(EXPLICIT_CLEANUP_CALLS.load(Ordering::SeqCst), 1); + assert_eq!(DROP_CLEANUP_CALLS.load(Ordering::SeqCst), 0); + } + + #[test] + fn failed_builtin_close_preserves_process_resource_for_retry() { + let _guard = PROCESS_CLEANUP_TEST_LOCK.lock().expect("test lock"); + DROP_CLEANUP_CALLS.store(0, Ordering::SeqCst); + EXPLICIT_CLEANUP_CALLS.store(0, Ordering::SeqCst); + let mut vm = Vm::new(crate::vm::Program::new( + Vec::new(), + vec![crate::vm::OpCode::Ret as u8], + )); + let token = vm + .execution_scope() + .push_resource(IoResource::new_with_process_cleanup( + exited_process_handle(), + fail_explicit_cleanup, + record_drop_cleanup, + )) + .expect("resource insert"); + let handle = token.handle(); + let raw = handle.raw() as i64; + + let args = [crate::vm::Value::Int(raw)]; + let error = builtin_io_close(&mut vm, &args).expect_err("cleanup failure must propagate"); + assert!( + error + .to_string() + .contains("injected explicit cleanup failure") + ); + let recovered = vm + .execution_scope() + .resources() + .typed::(handle) + .expect("failed close must leave the resource live"); + assert!( + vm.execution_scope() + .resources() + .get(&recovered) + .expect("resource borrow") + .handle + .is_some() + ); + vm.execution_scope() + .resources_mut() + .get_mut(&recovered) + .expect("resource borrow") + .explicit_cleanup = record_explicit_cleanup; + + assert!(builtin_io_close(&mut vm, &args).expect("retry should close the resource")); + assert!( + vm.execution_scope() + .resources() + .typed::(handle) + .is_err(), + "successful retry must retire the resource" + ); + assert_eq!(EXPLICIT_CLEANUP_CALLS.load(Ordering::SeqCst), 1); + assert_eq!(DROP_CLEANUP_CALLS.load(Ordering::SeqCst), 0); + } + + #[cfg(windows)] + #[test] + fn windows_drop_path_skips_taskkill_and_wait_cleanup() { + let _guard = PROCESS_CLEANUP_TEST_LOCK.lock().expect("test lock"); + DROP_CLEANUP_CALLS.store(0, Ordering::SeqCst); + WINDOWS_TASKKILL_CALLBACK_CALLS.store(0, Ordering::SeqCst); + WINDOWS_WAIT_CALLBACK_CALLS.store(0, Ordering::SeqCst); + let resource = IoResource::new_with_process_cleanup( + exited_process_handle(), + record_windows_explicit_cleanup, + record_windows_drop_cleanup, + ); + + drop(resource); + + assert_eq!(DROP_CLEANUP_CALLS.load(Ordering::SeqCst), 1); + assert_eq!(WINDOWS_TASKKILL_CALLBACK_CALLS.load(Ordering::SeqCst), 0); + assert_eq!(WINDOWS_WAIT_CALLBACK_CALLS.load(Ordering::SeqCst), 0); + } + + #[cfg(windows)] + #[test] + fn windows_explicit_close_can_run_taskkill_and_wait_callbacks() { + let _guard = PROCESS_CLEANUP_TEST_LOCK.lock().expect("test lock"); + DROP_CLEANUP_CALLS.store(0, Ordering::SeqCst); + WINDOWS_TASKKILL_CALLBACK_CALLS.store(0, Ordering::SeqCst); + WINDOWS_WAIT_CALLBACK_CALLS.store(0, Ordering::SeqCst); + let mut resource = IoResource::new_with_process_cleanup( + exited_process_handle(), + record_windows_explicit_cleanup, + record_windows_drop_cleanup, + ); + + resource + .close_explicit() + .expect("explicit cleanup should complete"); + drop(resource); + + assert_eq!(WINDOWS_TASKKILL_CALLBACK_CALLS.load(Ordering::SeqCst), 1); + assert_eq!(WINDOWS_WAIT_CALLBACK_CALLS.load(Ordering::SeqCst), 1); + assert_eq!(DROP_CLEANUP_CALLS.load(Ordering::SeqCst), 0); + } + #[test] fn process_tree_failure_still_attempts_direct_blocking_leader_cleanup() { let leader_attempted = Cell::new(false); diff --git a/src/vm/resource/close.rs b/src/vm/resource/close.rs index 95137ee8..a69c706b 100644 --- a/src/vm/resource/close.rs +++ b/src/vm/resource/close.rs @@ -29,6 +29,9 @@ pub enum CloseProgress { /// Contract: /// - [`begin_close`](HostResource::begin_close) must be idempotent and must /// synchronously issue any cancel/close request. +/// - [`begin_close_for_drop`](HostResource::begin_close_for_drop) is the abrupt +/// teardown counterpart. It must only issue immediate, nonblocking requests: +/// no waiting, polling, executor entry, or background work launch. /// - [`poll_close`](HostResource::poll_close) is called only after /// `begin_close` returns [`CloseProgress::Pending`]. /// - A concrete `Drop` remains the last-resort guard, but the VM may only reuse @@ -61,6 +64,19 @@ pub trait HostResource: Any + Send + 'static { Ok(CloseProgress::Ready) } + /// Begins best-effort cleanup from an abrupt owner `Drop`. + /// + /// The default preserves existing behavior for resources whose + /// `begin_close` already obeys the nonblocking contract. Resources with a + /// synchronous explicit-close path must override this method and report + /// `Pending` whenever cleanup has only been launched. + fn begin_close_for_drop( + &mut self, + reason: ResourceCloseReason, + ) -> ResourceResult { + self.begin_close(reason) + } + /// Polls an in-progress close to completion. /// /// Only invoked after `begin_close` returned [`CloseProgress::Pending`]. diff --git a/src/vm/resource/table.rs b/src/vm/resource/table.rs index 0f1ebb5d..1cbc8862 100644 --- a/src/vm/resource/table.rs +++ b/src/vm/resource/table.rs @@ -541,9 +541,9 @@ impl ResourceTable { /// /// Unlike the reusable close/reset sweep, this phase does not wait for a /// pending resource to become quiescent before continuing. It invokes - /// `begin_close` once for each still-open slot, retains closing slots in - /// `Closing`, and never reports table quiescence. Already-closing slots are - /// left untouched, preserving exactly-once begin semantics. + /// `begin_close_for_drop` once for each still-open slot, retains closing + /// slots in `Closing`, and never reports table quiescence. Already-closing + /// slots are left untouched, preserving exactly-once begin semantics. pub(crate) fn begin_close_remaining_for_drop( &mut self, reason: ResourceCloseReason, @@ -557,7 +557,7 @@ impl ResourceTable { self.put_slot_state(slot_index, state); continue; }; - match resource.begin_close(reason) { + match resource.begin_close_for_drop(reason) { Ok(CloseProgress::Ready) => self.reclaim(slot_index), Ok(CloseProgress::Pending) => { self.put_slot_state(slot_index, SlotState::Closing(resource)); diff --git a/tests/builtins/io_async_tests.rs b/tests/builtins/io_async_tests.rs index 3497dd73..baff78c2 100644 --- a/tests/builtins/io_async_tests.rs +++ b/tests/builtins/io_async_tests.rs @@ -208,7 +208,7 @@ fn wait_for_file(path: &std::path::Path) -> String { panic!("timed out waiting for {}", path.display()); } -#[cfg(unix)] +#[cfg(target_os = "linux")] fn pid_is_running(pid: u32) -> bool { let Ok(pid) = libc::pid_t::try_from(pid) else { return false; @@ -227,7 +227,7 @@ fn pid_is_running(pid: u32) -> bool { result == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) } -#[cfg(unix)] +#[cfg(target_os = "linux")] fn wait_for_pid_exit(pid: u32) -> bool { for _ in 0..200 { if !pid_is_running(pid) { @@ -258,7 +258,7 @@ fn guest_popen_program(command: &str, expression: &str) -> String { format!(r#"let h = io::popen("{command}", "r"); {expression}"#) } -#[cfg(unix)] +#[cfg(target_os = "linux")] #[test] fn async_io_reset_kills_and_reaps_the_entire_popen_process_group() { let nonce = SystemTime::now() @@ -358,7 +358,7 @@ fn async_io_reset_kills_and_reaps_the_entire_popen_process_group() { let _ = std::fs::remove_file(marker_path); } -#[cfg(unix)] +#[cfg(target_os = "linux")] #[test] fn async_io_vm_drop_terminates_live_popen_process_tree() { let nonce = SystemTime::now() diff --git a/tests/builtins/io_scope_lifecycle_tests.rs b/tests/builtins/io_scope_lifecycle_tests.rs index 69c8cebf..74571206 100644 --- a/tests/builtins/io_scope_lifecycle_tests.rs +++ b/tests/builtins/io_scope_lifecycle_tests.rs @@ -163,14 +163,14 @@ fn drop_retires_io_resources_through_scope() { drop(vm); } -#[cfg(unix)] +#[cfg(target_os = "linux")] struct ProcessTreeCleanup { leader: i32, descendant: i32, marker: std::path::PathBuf, } -#[cfg(unix)] +#[cfg(target_os = "linux")] impl Drop for ProcessTreeCleanup { fn drop(&mut self) { unsafe { @@ -181,7 +181,7 @@ impl Drop for ProcessTreeCleanup { } } -#[cfg(unix)] +#[cfg(target_os = "linux")] fn process_is_running(pid: i32) -> bool { let path = format!("/proc/{pid}/stat"); let Ok(stat) = std::fs::read_to_string(path) else { @@ -193,7 +193,7 @@ fn process_is_running(pid: i32) -> bool { !state.starts_with('Z') } -#[cfg(unix)] +#[cfg(target_os = "linux")] fn wait_for_process_exit(pid: i32) { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); while process_is_running(pid) { @@ -205,7 +205,7 @@ fn wait_for_process_exit(pid: i32) { } } -#[cfg(unix)] +#[cfg(target_os = "linux")] fn read_process_marker(path: &std::path::Path) -> (i32, i32) { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); loop { @@ -227,7 +227,7 @@ fn read_process_marker(path: &std::path::Path) -> (i32, i32) { } } -#[cfg(unix)] +#[cfg(target_os = "linux")] #[test] fn reset_for_reuse_terminates_live_popen_process_tree() { let marker = std::env::temp_dir().join(format!( @@ -253,10 +253,37 @@ fn reset_for_reuse_terminates_live_popen_process_tree() { let _ = std::fs::remove_file(marker); } -#[cfg(unix)] +#[cfg(target_os = "linux")] +#[test] +fn vm_drop_signals_live_popen_process_tree_without_driving_close() { + let marker = std::env::temp_dir().join(format!( + "pd-vm-blocking-io-drop-{}-{}.marker", + std::process::id(), + SystemTimeNonce::new() + )); + let command = format!( + "parent=$$; sleep 30 & child=$!; printf '%s %s' $parent $child > {}; wait $child", + marker.display() + ); + let vm = vm_for(&format!("let h = io::popen(\"{command}\", \"r\"); h;")); + let (leader, descendant) = read_process_marker(&marker); + let _cleanup = ProcessTreeCleanup { + leader, + descendant, + marker: marker.clone(), + }; + + drop(vm); + + wait_for_process_exit(leader); + wait_for_process_exit(descendant); + let _ = std::fs::remove_file(marker); +} + +#[cfg(target_os = "linux")] struct SystemTimeNonce(u128); -#[cfg(unix)] +#[cfg(target_os = "linux")] impl SystemTimeNonce { fn new() -> Self { Self( @@ -268,7 +295,7 @@ impl SystemTimeNonce { } } -#[cfg(unix)] +#[cfg(target_os = "linux")] impl std::fmt::Display for SystemTimeNonce { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.0.fmt(formatter) diff --git a/tests/vm/execution_scope_tests.rs b/tests/vm/execution_scope_tests.rs index ad495dac..473fd0bb 100644 --- a/tests/vm/execution_scope_tests.rs +++ b/tests/vm/execution_scope_tests.rs @@ -116,6 +116,7 @@ impl HostOperation for CancelAwareWorker { struct DropPendingResource { begins: Arc, + drop_begins: Arc, polls: Arc, } @@ -125,6 +126,14 @@ impl HostResource for DropPendingResource { Ok(CloseProgress::Pending) } + fn begin_close_for_drop( + &mut self, + _reason: ResourceCloseReason, + ) -> ResourceResult { + self.drop_begins.fetch_add(1, Ordering::SeqCst); + Ok(CloseProgress::Pending) + } + fn poll_close(&mut self, _cx: &mut Context<'_>) -> Poll> { self.polls.fetch_add(1, Ordering::SeqCst); Poll::Pending @@ -238,18 +247,21 @@ fn empty_scope_quiesces_cleanly() { #[test] fn vm_drop_begins_resource_cleanup_without_polling_it() { let begins = Arc::new(AtomicUsize::new(0)); + let drop_begins = Arc::new(AtomicUsize::new(0)); let polls = Arc::new(AtomicUsize::new(0)); let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8])); vm.execution_scope() .push_resource(DropPendingResource { begins: Arc::clone(&begins), + drop_begins: Arc::clone(&drop_begins), polls: Arc::clone(&polls), }) .expect("resource"); drop(vm); - assert_eq!(begins.load(Ordering::SeqCst), 1); + assert_eq!(begins.load(Ordering::SeqCst), 0); + assert_eq!(drop_begins.load(Ordering::SeqCst), 1); assert_eq!( polls.load(Ordering::SeqCst), 0, From 0dddf7d94773d16cef5a20f5b2f17c86b12013ff Mon Sep 17 00:00:00 2001 From: fffonion Date: Sat, 19 Sep 2026 22:24:47 +0800 Subject: [PATCH 19/23] fix(sqlite): enforce transaction deadline at commit --- src/builtins/runtime/sqlite.rs | 173 ++++++++++++++++++++++++++++++--- 1 file changed, 160 insertions(+), 13 deletions(-) diff --git a/src/builtins/runtime/sqlite.rs b/src/builtins/runtime/sqlite.rs index e980dd6f..7e96311f 100644 --- a/src/builtins/runtime/sqlite.rs +++ b/src/builtins/runtime/sqlite.rs @@ -1536,17 +1536,45 @@ fn parse_transaction_statements( .collect() } +fn transaction_deadline_error(max_transaction_ms: u64) -> VmError { + VmError::HostError(format!( + "SQLite transaction exceeded the configured {max_transaction_ms} ms deadline" + )) +} + +fn set_transaction_busy_timeout( + connection: &Connection, + deadline: Instant, + max_transaction_ms: u64, + busy_timeout_ms: u64, +) -> VmResult<()> { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(transaction_deadline_error(max_transaction_ms)); + } + connection + .busy_timeout(remaining.min(Duration::from_millis(busy_timeout_ms))) + .map_err(sqlite_error) +} + fn transaction_with_connection( connection: &mut Connection, statements: Vec, max_transaction_ms: u64, + busy_timeout_ms: u64, ) -> VmResult> { let deadline = Instant::now() .checked_add(Duration::from_millis(max_transaction_ms)) .ok_or_else(|| { VmError::HostError("SQLite transaction deadline is out of range".to_string()) })?; - transaction_with_connection_until(connection, statements, deadline, max_transaction_ms) + transaction_with_connection_until( + connection, + statements, + deadline, + max_transaction_ms, + busy_timeout_ms, + ) } fn transaction_with_connection_until( @@ -1554,6 +1582,7 @@ fn transaction_with_connection_until( statements: Vec, deadline: Instant, max_transaction_ms: u64, + busy_timeout_ms: u64, ) -> VmResult> { connection .progress_handler( @@ -1561,17 +1590,28 @@ fn transaction_with_connection_until( Some(move || Instant::now() >= deadline), ) .map_err(sqlite_error)?; + if let Err(error) = connection.commit_hook(Some(move || Instant::now() >= deadline)) { + connection + .progress_handler(0, None:: bool>) + .map_err(sqlite_error)?; + return Err(sqlite_error(error)); + } let result = (|| { + set_transaction_busy_timeout(connection, deadline, max_transaction_ms, busy_timeout_ms)?; let transaction = connection .transaction_with_behavior(TransactionBehavior::Immediate) .map_err(sqlite_error)?; let mut results = Vec::with_capacity(statements.len()); for statement in statements { if Instant::now() >= deadline { - return Err(VmError::HostError(format!( - "SQLite transaction exceeded the configured {max_transaction_ms} ms deadline" - ))); + return Err(transaction_deadline_error(max_transaction_ms)); } + set_transaction_busy_timeout( + &transaction, + deadline, + max_transaction_ms, + busy_timeout_ms, + )?; let result = if statement.query { let value = query_with_connection( &transaction, @@ -1594,20 +1634,30 @@ fn transaction_with_connection_until( } } if Instant::now() >= deadline { - return Err(VmError::HostError(format!( - "SQLite transaction exceeded the configured {max_transaction_ms} ms deadline" - ))); + return Err(transaction_deadline_error(max_transaction_ms)); } + set_transaction_busy_timeout(&transaction, deadline, max_transaction_ms, busy_timeout_ms)?; + // SQLite invokes the commit hook after it has obtained the + // rollback-journal EXCLUSIVE lock and before commit phase one. The + // hook therefore closes the race where the bounded busy handler wakes + // after the deadline and would otherwise complete a late commit. transaction.commit().map_err(sqlite_error)?; Ok(results) })(); - connection + let commit_hook_cleanup = connection + .commit_hook(None:: bool>) + .map_err(sqlite_error); + let progress_cleanup = connection .progress_handler(0, None:: bool>) - .map_err(sqlite_error)?; + .map_err(sqlite_error); + let busy_timeout_cleanup = connection + .busy_timeout(Duration::from_millis(busy_timeout_ms)) + .map_err(sqlite_error); + commit_hook_cleanup?; + progress_cleanup?; + busy_timeout_cleanup?; if Instant::now() >= deadline && result.is_err() { - return Err(VmError::HostError(format!( - "SQLite transaction exceeded the configured {max_transaction_ms} ms deadline" - ))); + return Err(transaction_deadline_error(max_transaction_ms)); } result } @@ -1626,6 +1676,7 @@ pub(super) async fn builtin_sqlite_transaction_impl( context.allow_unsafe_sql, )?; let max_transaction_ms = context.limits.max_transaction_ms; + let busy_timeout_ms = context.limits.busy_timeout_ms; let closed = Arc::clone(&context.closed); let value = context .connection @@ -1636,7 +1687,7 @@ pub(super) async fn builtin_sqlite_transaction_impl( "SQLite database is already closed".to_string(), )); } - transaction_with_connection(connection, statements, max_transaction_ms) + transaction_with_connection(connection, statements, max_transaction_ms, busy_timeout_ms) }) .await .map_err(adapter_call_error)?; @@ -1826,6 +1877,7 @@ mod tests { statements, Instant::now() + Duration::from_millis(500), 500, + limits.busy_timeout_ms, ) }); @@ -1851,6 +1903,101 @@ mod tests { fs::remove_dir_all(root).expect("temporary SQLite root should be removed"); } + #[test] + fn transaction_deadline_rolls_back_when_commit_waits_for_reader() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock should be after the Unix epoch") + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "rustscript-sqlite-commit-deadline-{}-{nonce}", + std::process::id() + )); + fs::create_dir_all(&root).expect("temporary SQLite root should be created"); + let database_path = root.join("state.db"); + Connection::open(&database_path) + .expect("setup connection should open") + .execute_batch("PRAGMA journal_mode = DELETE; CREATE TABLE items (value INTEGER)") + .expect("setup table should be created in rollback-journal mode"); + + let reader = Connection::open(&database_path).expect("reader connection should open"); + reader + .execute_batch("BEGIN") + .expect("reader transaction should begin"); + let visible_count: i64 = reader + .query_row("SELECT count(*) FROM items", [], |row| row.get(0)) + .expect("reader should acquire a shared lock"); + assert_eq!( + visible_count, 0, + "the reader should see the initial database state" + ); + + let (write_observed_tx, write_observed_rx) = mpsc::channel(); + let worker_path = database_path.clone(); + let transaction = std::thread::spawn(move || { + let mut connection = + Connection::open(worker_path).expect("transaction connection should open"); + connection + .busy_timeout(Duration::from_secs(2)) + .expect("commit should wait for the reader past the transaction deadline"); + let limits = SqliteLimits::default(); + let statements = vec![TransactionStatement { + sql: "INSERT INTO items (value) VALUES (1)".to_string(), + params: Vec::new(), + query: false, + limits, + after_execute: Some(Box::new(move || { + write_observed_tx + .send(()) + .expect("write observation receiver should remain open"); + })), + }]; + let result = transaction_with_connection_until( + &mut connection, + statements, + Instant::now() + Duration::from_millis(100), + 100, + 2_000, + ); + let restored_busy_timeout: i64 = connection + .query_row("PRAGMA busy_timeout", [], |row| row.get(0)) + .expect("configured busy timeout should remain queryable"); + (result, connection.is_autocommit(), restored_busy_timeout) + }); + + write_observed_rx + .recv_timeout(Duration::from_secs(2)) + .expect("the INSERT must execute while the reader holds its shared lock"); + std::thread::sleep(Duration::from_millis(250)); + reader + .execute_batch("ROLLBACK") + .expect("reader should release its shared lock"); + drop(reader); + + let (result, is_autocommit, restored_busy_timeout) = transaction + .join() + .expect("transaction worker should not panic"); + let error = result.expect_err("a commit unblocked after the deadline must roll back"); + assert!( + error.to_string().contains("100 ms deadline"), + "commit deadline must surface explicitly, got: {error}" + ); + assert!(is_autocommit, "the timed-out transaction must be closed"); + assert_eq!( + restored_busy_timeout, 2_000, + "the connection busy timeout must be restored after rollback" + ); + + let verifier = Connection::open(&database_path) + .expect("verification connection should reopen the database"); + let count: i64 = verifier + .query_row("SELECT count(*) FROM items", [], |row| row.get(0)) + .expect("verification query should succeed"); + assert_eq!(count, 0, "a late commit must not persist its write"); + drop(verifier); + fs::remove_dir_all(root).expect("temporary SQLite root should be removed"); + } + #[tokio::test] async fn retryable_close_failure_retains_resource_and_connection_permit() { let limits = SqliteLimits::default(); From 7796a826b0bc10bdbfbe86a0d6362a1407192b93 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sat, 19 Sep 2026 22:58:45 +0800 Subject: [PATCH 20/23] test(sqlite): cover commit-hook deadline veto --- src/builtins/runtime/sqlite.rs | 186 ++++++++++++++++++++++++++++++++- 1 file changed, 185 insertions(+), 1 deletion(-) diff --git a/src/builtins/runtime/sqlite.rs b/src/builtins/runtime/sqlite.rs index 7e96311f..5c9aa147 100644 --- a/src/builtins/runtime/sqlite.rs +++ b/src/builtins/runtime/sqlite.rs @@ -14,11 +14,15 @@ //! retains its operation lease in the adapter closure until that work finishes or //! is discarded. +#[cfg(test)] +use std::cell::RefCell; use std::fs; use std::future::Future; use std::path::{Component, Path, PathBuf}; use std::pin::Pin; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +#[cfg(test)] +use std::sync::mpsc::{Receiver, Sender}; use std::sync::{Arc, Mutex}; use std::task::{Context, Poll}; use std::time::{Duration, Instant}; @@ -42,6 +46,69 @@ use crate::vm::{ /// SQLite `progress_handler` step cadence used to enforce transaction deadlines. const SQLITE_PROGRESS_STEPS: i32 = 1_000; +#[cfg(test)] +struct TransactionCommitDeadlineGate { + reached: Sender<()>, + release: Receiver<()>, + commit_hook_vetoed: Arc, +} + +#[cfg(test)] +thread_local! { + static TRANSACTION_COMMIT_DEADLINE_GATE: + RefCell> = const { RefCell::new(None) }; +} + +#[cfg(test)] +struct TransactionCommitDeadlineGateGuard { + previous: Option, +} + +#[cfg(test)] +impl Drop for TransactionCommitDeadlineGateGuard { + fn drop(&mut self) { + TRANSACTION_COMMIT_DEADLINE_GATE.with(|slot| { + slot.replace(self.previous.take()); + }); + } +} + +#[cfg(test)] +fn install_transaction_commit_deadline_gate( + gate: TransactionCommitDeadlineGate, +) -> TransactionCommitDeadlineGateGuard { + let previous = TRANSACTION_COMMIT_DEADLINE_GATE.with(|slot| slot.replace(Some(gate))); + TransactionCommitDeadlineGateGuard { previous } +} + +#[cfg(test)] +fn wait_at_transaction_commit_deadline_gate() -> bool { + TRANSACTION_COMMIT_DEADLINE_GATE.with(|slot| { + let slot = slot.borrow(); + let Some(gate) = slot.as_ref() else { + return false; + }; + gate.reached + .send(()) + .expect("deadline gate controller should remain available"); + gate.release + .recv() + .expect("deadline gate controller should release the transaction"); + true + }) +} + +#[cfg(test)] +fn record_transaction_commit_hook(vetoed: bool) { + if vetoed { + TRANSACTION_COMMIT_DEADLINE_GATE.with(|slot| { + if let Some(gate) = slot.borrow().as_ref() { + gate.commit_hook_vetoed.store(true, Ordering::Release); + } + }); + } +} + /// Maximum adapter close attempts, including the initial request. const SQLITE_CLOSE_MAX_ATTEMPTS: usize = 3; @@ -1590,7 +1657,12 @@ fn transaction_with_connection_until( Some(move || Instant::now() >= deadline), ) .map_err(sqlite_error)?; - if let Err(error) = connection.commit_hook(Some(move || Instant::now() >= deadline)) { + if let Err(error) = connection.commit_hook(Some(move || { + let vetoed = Instant::now() >= deadline; + #[cfg(test)] + record_transaction_commit_hook(vetoed); + vetoed + })) { connection .progress_handler(0, None:: bool>) .map_err(sqlite_error)?; @@ -1636,7 +1708,21 @@ fn transaction_with_connection_until( if Instant::now() >= deadline { return Err(transaction_deadline_error(max_transaction_ms)); } + #[cfg(test)] + // A gated test must proceed directly to `commit` after crossing the + // deadline so the installed commit hook owns the veto. + let deadline_gate_enabled = wait_at_transaction_commit_deadline_gate(); + #[cfg(not(test))] set_transaction_busy_timeout(&transaction, deadline, max_transaction_ms, busy_timeout_ms)?; + #[cfg(test)] + if !deadline_gate_enabled { + set_transaction_busy_timeout( + &transaction, + deadline, + max_transaction_ms, + busy_timeout_ms, + )?; + } // SQLite invokes the commit hook after it has obtained the // rollback-journal EXCLUSIVE lock and before commit phase one. The // hook therefore closes the race where the bounded busy handler wakes @@ -1903,6 +1989,104 @@ mod tests { fs::remove_dir_all(root).expect("temporary SQLite root should be removed"); } + #[test] + fn transaction_commit_hook_vetoes_expired_deadline_and_hooks_are_restored() { + let mut connection = Connection::open_in_memory().expect("test connection should open"); + connection + .execute_batch("CREATE TABLE items (value INTEGER)") + .expect("test table should be created"); + let limits = SqliteLimits::default(); + let original_busy_timeout_ms = 1_337; + connection + .busy_timeout(Duration::from_millis(original_busy_timeout_ms)) + .expect("original busy timeout should be configured"); + + let (gate_reached_tx, gate_reached_rx) = mpsc::channel(); + let (gate_release_tx, gate_release_rx) = mpsc::channel(); + let commit_hook_vetoed = Arc::new(AtomicBool::new(false)); + let deadline = Instant::now() + Duration::from_secs(1); + let gate_controller = std::thread::spawn(move || { + let reached = gate_reached_rx.recv_timeout(Duration::from_secs(5)).is_ok(); + if reached { + while Instant::now() < deadline { + std::thread::sleep(deadline.saturating_duration_since(Instant::now())); + } + } + gate_release_tx + .send(()) + .expect("transaction should remain at the deadline gate"); + reached + }); + let _gate_guard = install_transaction_commit_deadline_gate(TransactionCommitDeadlineGate { + reached: gate_reached_tx, + release: gate_release_rx, + commit_hook_vetoed: Arc::clone(&commit_hook_vetoed), + }); + let statements = vec![TransactionStatement { + sql: "INSERT INTO items (value) VALUES (1)".to_string(), + params: Vec::new(), + query: false, + limits, + after_execute: None, + }]; + + let result = transaction_with_connection_until( + &mut connection, + statements, + deadline, + 1_000, + original_busy_timeout_ms, + ); + assert!( + gate_controller + .join() + .expect("gate controller should not panic"), + "transaction must reach the pre-commit deadline gate" + ); + let error = result.expect_err("the expired commit must be vetoed"); + assert!( + error.to_string().contains("1000 ms deadline"), + "commit deadline must surface explicitly, got: {error}" + ); + assert!( + commit_hook_vetoed.load(Ordering::Acquire), + "the installed commit hook must veto the expired commit" + ); + assert!( + connection.is_autocommit(), + "the vetoed transaction must restore autocommit" + ); + let count_after_veto: i64 = connection + .query_row("SELECT count(*) FROM items", [], |row| row.get(0)) + .expect("row count should remain queryable after the veto"); + assert_eq!(count_after_veto, 0, "the vetoed write must not persist"); + let restored_busy_timeout: i64 = connection + .query_row("PRAGMA busy_timeout", [], |row| row.get(0)) + .expect("restored busy timeout should remain queryable"); + assert_eq!( + restored_busy_timeout, original_busy_timeout_ms as i64, + "the original busy timeout must be restored" + ); + + connection + .execute_batch( + "WITH RECURSIVE numbers(value) AS (\ + VALUES(1) \ + UNION ALL \ + SELECT value + 1 FROM numbers WHERE value < 5000\ + ) \ + INSERT INTO items SELECT value FROM numbers", + ) + .expect("a later write on the same connection should succeed"); + let count_after_reuse: i64 = connection + .query_row("SELECT count(*) FROM items", [], |row| row.get(0)) + .expect("row count should remain queryable after connection reuse"); + assert_eq!( + count_after_reuse, 5_000, + "progress and commit hooks must not leak into later writes" + ); + } + #[test] fn transaction_deadline_rolls_back_when_commit_waits_for_reader() { let nonce = SystemTime::now() From 332ab2ce2af9a19574f5ce4f3680395fd73b2ebd Mon Sep 17 00:00:00 2001 From: fffonion Date: Sun, 20 Sep 2026 12:35:00 +0800 Subject: [PATCH 21/23] docs: correct HTTP and async host contracts --- docs/host-sdk-descriptors.md | 114 ++++++++++++++++++++--------------- docs/http-client.md | 15 +++-- 2 files changed, 76 insertions(+), 53 deletions(-) diff --git a/docs/host-sdk-descriptors.md b/docs/host-sdk-descriptors.md index ac5b3c5d..a84ffa37 100644 --- a/docs/host-sdk-descriptors.md +++ b/docs/host-sdk-descriptors.md @@ -151,59 +151,79 @@ request/result/event values as `Map(unknown)` or `unknown`; the `tests/typed_host_no_dynamic_contract_tests.rs` guard fails the build if a public standard function regresses to a dynamic schema. -## 3. Raw handles and declared contracts +## 3. Async functions and declared contracts -Some hosting surfaces already expose a raw `i64` scope token (I/O handles and -SQLite connections in the standard library) and cannot change the Rust signature -without touching their worker/operation internals. Those functions declare their -guest contract explicitly, **on the same function**, instead of duplicating a -catalog entry: +Standard I/O and SQLite use ordinary macro-generated async host functions. A +function may keep an internal raw `i64` execution-scope token while its explicit +contract exposes a typed guest resource. Put that contract on the same function +instead of duplicating a catalog entry. + +Every value that crosses an `.await` must be owned. Implement +`CaptureAsyncHostContext` to copy or clone the required VM state before the VM +borrow ends, and use owned parameters such as `String`, `VmMap`, or +`VmArrayHandle`. The current `io::open` shape is representative (body details +are abbreviated): ```rust -/// Opens an I/O handle. -#[pd_host_function( - name = "io::open", - contract = super::io_open_contract, // declared next to the function -)] -pub(super) fn builtin_io_open(vm: &mut Vm, path: &str, mode: &str) -> VmResult> { - /* unchanged runtime path; the returned id is a driver-scheduled pending op */ +#[derive(Clone)] +pub(crate) struct IoPolicyContext { + policy: Option, } -fn io_open_contract() -> vm::HostFunctionSchema { - vm::HostFunctionSchema::with_return( - "io::open", - vec![ - vm::HostParamSchema::value("path", vm::HostTypeSchema::String), - vm::HostParamSchema::value("mode", vm::HostTypeSchema::String), - ], - vm::HostTypeSchema::Resource(io_file_key()), // typed `io.file` resource - ) +impl vm::CaptureAsyncHostContext for IoPolicyContext { + fn capture(vm: &mut Vm) -> VmResult { + Ok(Self { policy: io_policy(vm) }) + } } -``` - -Modules that can await library futures directly should declare ordinary async -host functions. The macro captures any `#[pd_host_context]` value before the VM -borrow ends, submits the future through the embedding's async bridge, and maps -the resolved value through the declared contract: -```rust -/// Performs one bounded request through a shared library client. -#[pd_host_function(name = "http::client::request", contract = http_request_contract)] -pub(super) async fn builtin_http_client_request( - #[pd_host_context] context: HttpRequestContext, - request: VmMapHandle, -) -> VmResult { - context.request(request).await +#[pd_host_function(name = "io::open", contract = super::io_open_contract)] +pub(crate) async fn builtin_io_open( + #[pd_host_context] context: IoPolicyContext, + path: String, + mode: String, +) -> VmResult> { + // Mode validation and `OpenOptions` setup are omitted here. + let path = authorize_io_path(context.policy.as_ref(), &path, writes).await?; + let file = options + .open(path) + .await + .map_err(|error| VmError::HostError(format!("io_open failed: {error}")))?; + let handle = IoHandle::File(BufReader::new(file)); + Ok(vm::HostFutureOutput::complete(move |vm| { + let token = vm + .execution_scope() + .push_resource(IoResource::new(handle)) + .map_err(|error| VmError::HostError(format!( + "io resource insert failed: {error}" + )))?; + Ok(token.into_handle().raw() as i64) + })) } ``` -The context and parameter types of an async host function must own every value -that crosses the suspension boundary. Owned callable schemas, including bare -function types such as `VmCallable Action>`, are accepted. When an -async opening phase must hand control to generic VM continuation machinery, -return `HostFutureOutput` and use `HostFutureOutput::continue_with`; value -mapping preserves that continuation. The HTTP SSE builtin uses this only to -transfer an opened Hyper response into the generic callable-stream driver. +When context depends on a resource-token argument, implement +`capture_with_args(vm, args)` to validate the token and clone the owned resource +state before submission. `IoHandleContext`, `SqliteConnectionContext`, and +`SqliteCloseContext` use that form; the raw token parameter remains in the Rust +signature for contract binding but is not borrowed across suspension. + +The generated wrapper captures `#[pd_host_context]` and converts ordinary +parameters before creating the future. It then calls `Vm::submit_host_future`, +which assigns an operation id and submits the boxed future to the embedding's +generic `HostAsyncBridge`. The bridge owns polling on its executor; the VM tracks +that submitted operation and resolves the result through the declared contract. +The embedding must keep the bridge's executor and runtime alive and driven until +its submitted futures finish. Process-backed I/O also requires a live Tokio +runtime while close or reset cleanup is polled, as detailed below. + +Owned callable schemas, including bare function types such as +`VmCallable Action>`, are accepted. Use +`HostFutureOutput::complete` when the resolved future must briefly re-enter the +VM to insert or retire a resource. When an async opening phase must hand control +to generic VM continuation machinery, return `HostFutureOutput` and use +`HostFutureOutput::continue_with`; value mapping preserves that continuation. +The HTTP SSE builtin uses this only to transfer an opened Hyper response into +the generic callable-stream driver. ### Tokio runtime lease for process-backed IO @@ -233,11 +253,11 @@ lease is valid. What the contract does and does not change: -- The contract **replaces only the guest schema**. The adapter, binding class, and - host-state effects still come from the one macro expansion — there is no second - registration path to keep in sync. +- The contract **replaces only the guest schema**. The adapter, binding class, + async submission, and host-state effects still come from the one macro + expansion — there is no second registration path to keep in sync. - Guest resource effects are **derived from the contract schema**, so the raw - handle signature cannot drift from what the guest sees. + token signature cannot drift from what the guest sees. - The contract's declared name is validated against the function name at construction; a renamed function cannot silently keep a stale contract. - Resource *declarations* stay with the module: implement `HostResourceType` once diff --git a/docs/http-client.md b/docs/http-client.md index ca0db620..b86206b0 100644 --- a/docs/http-client.md +++ b/docs/http-client.md @@ -264,22 +264,25 @@ The shared in-flight HTTP call default is 64. Zero values for streaming byte lim ## Destination policy and protocol transports -Every protocol uses the same admission, address-pinning, and security policy: +Every protocol uses the same admission and connection-time address-validation +policy: - URLs require a host and reject userinfo; - both the protocol's scheme family and the configured scheme allowlist must admit the URL; - host and effective port must match their configured allowlists; -- every DNS result is validated, and the selected validated address is pinned for the connection; +- admission resolves the target and validates every returned address; when Hyper opens a connection, its connector resolves the hostname again and validates every address from that lookup before allowing a connect; - when private addresses are disabled, private, loopback, link-local, multicast, unspecified, documentation, transition, reserved, and other special-use IPv4/IPv6 ranges are rejected; IPv4-mapped IPv6 addresses receive the IPv4 checks; -- the original validated hostname remains the TLS SNI name and HTTP `Host` authority when connecting to a pinned address; +- the admitted hostname remains the TLS SNI name and HTTP `Host` authority; the address selected during admission is not pinned to the connection; - buffered HTTP and SSE revalidate every redirect and remove `Authorization` and `Cookie` on a cross-origin redirect; - ambient proxy settings are ignored. There is no implicit cookie jar, authentication source, or global proxy state. The policy snapshot taken at call admission applies for the complete operation. Every request and redirect target receives an admission-time DNS/private-address -check. Hyper's connector repeats the address check on the DNS results used when -it opens a new connection, retaining the original hostname for HTTP authority -and TLS SNI. +check. Hyper's connector performs its own lookup whenever it opens a new +connection and rejects the lookup if any returned address is disallowed. This +prevents a target from reaching a disallowed private address through DNS +rebinding without claiming that the admission-selected address is the one used +for the socket. Buffered HTTP and SSE share one cloneable Hyper client stored in per-VM HTTP module state. Hyper owns HTTP/1 transport setup, connection pooling, idle From fc4b95ee9cba28d1c37683f4713f2bbbdb5bedd9 Mon Sep 17 00:00:00 2001 From: fffonion Date: Sun, 20 Sep 2026 12:48:49 +0800 Subject: [PATCH 22/23] docs(host): remove historical SDK guidance --- docs/host-sdk-descriptors.md | 88 +----------------------------------- 1 file changed, 2 insertions(+), 86 deletions(-) diff --git a/docs/host-sdk-descriptors.md b/docs/host-sdk-descriptors.md index a84ffa37..dc8aec87 100644 --- a/docs/host-sdk-descriptors.md +++ b/docs/host-sdk-descriptors.md @@ -1,20 +1,11 @@ # Host SDK: descriptors, resources, and effects -This guide is for anyone authoring a host module or migrating an existing host -extension onto the current SDK. It describes the preferred authoring surface: -a `#[pd_host_function]` declaration is the single source of a function's guest +This guide describes the current Host SDK authoring surface, where a +`#[pd_host_function]` declaration is the single source of a function's guest schema, runtime binding, typed resource requirements, named-struct contract, and hidden host-state effects. A module then lists its functions explicitly; the guest catalog is derived from those descriptors. -The legacy `HostApiBuilder::{resource,named_struct,function}` and -`HostFunctionRegistry::register_*` APIs remain public and fully supported during -the compatibility window (see [Compatibility window](#compatibility-window)). -New modules should not need them. - -Organization freeze outcomes: -[host-descriptor-migration-report.md](host-descriptor-migration-report.md). - ## 1. The model One host function declaration produces, in a single macro expansion: @@ -356,78 +347,3 @@ catalog preserves the expected module and aggregate fingerprints. It also checks that resource declarations and named structs follow the selected modules, public returns stay typed, and native-only HTTP and SQLite surfaces do not appear when their build gates are off. - -## 7. Compatibility window - -Still public and supported, for downstream migration: - -- `HostApiBuilder::{resource, named_struct, function}`. -- `HostFunctionRegistry::register`, `register_static`, `register_stack`, - `register_static_stack`, `register_args`, `register_static_args`, - `register_static_non_yielding_args`, and the catalog/exact-family - registrations, including `register_exact_owned`. -- The per-module `register_*_builtin_module{,_from_catalog}` entry points. - -Preferred for new code: - -- `HostModuleDescriptor` + `install` / `install_from_catalog`, - `HostFunctionDescriptor`, `HostResourceType`, `HostOwnedAdapterFactory`. - -Removal threshold: the legacy builder and low-level registry APIs are removed -only after every repository in the organization migration matrix has passed its -gate against a frozen core SHA. Until then, legacy catalogs compose with -descriptor modules through `install_from_catalog`. - -## 8. Before and after - -**Before** (one function, four places to keep in sync): - -```rust -// 1. the runtime adapter -#[pd_host_function(name = "demo::read_counter")] -fn read_counter(vm: &mut Vm, handle: i64) -> VmResult { /* ... */ } - -// 2. a hand-written resource entry -builder.resource(ResourceTypeSchema::new(counter_key(), "A monotonic counter")); - -// 3. a hand-written function schema -builder.function(HostFunctionSchema::with_return( - "demo::read_counter", - vec![HostParamSchema::with_passing( - "counter", - HostTypeSchema::Resource(counter_key()), - HostParamPassing::Borrow, - )], - HostTypeSchema::Int, -)); - -// 4. an adapter table plus an exact registration -const ADAPTER_CONTRACTS: &[AdapterContract] = &[AdapterContract { - name: "demo::read_counter", - arity: 1, - adapter: read_counter_adapter, -}]; -registry.transactionally(|staged| { /* validate, register, authorize */ }) -``` - -**After** (one function, one declaration, one module list): - -```rust -#[pd_host_function(name = "demo::read_counter")] -fn read_counter(counter: ResourceRef<'_, Counter>) -> VmResult { - Ok(*counter as u64 as i64) -} - -pub fn demo_module() -> HostModuleDescriptor { - HostModuleDescriptor { - name: "demo", - functions: &[read_counter_descriptor], - resources: &[], - } -} - -demo_module().install(registry)?; // validates and installs transactionally -``` - -The wrapper, the schema, the resource declaration, the binding class, the -effects, and the exact registry entry all come from the single declaration. From 87d8c5d5cb42b3381aecadb383c0b7bf34ff204f Mon Sep 17 00:00:00 2001 From: fffonion Date: Sun, 20 Sep 2026 13:15:13 +0800 Subject: [PATCH 23/23] docs: correct host SDK examples and HTTP defaults --- docs/host-sdk-descriptors.md | 69 ++++++++++++++++++++++++++++-------- docs/http-client.md | 2 ++ 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/docs/host-sdk-descriptors.md b/docs/host-sdk-descriptors.md index dc8aec87..c3266a92 100644 --- a/docs/host-sdk-descriptors.md +++ b/docs/host-sdk-descriptors.md @@ -44,7 +44,7 @@ explicit and reproducible. ```rust use pd_host_function::pd_host_function; -use vm::{ResourceRef, ResourceOwned, VmResult, resource}; +use vm::{resource, ResourceOwned, ResourceRef, Vm, VmError, VmResult}; /// A typed host resource with a canonical declaration. pub struct Counter(u64); @@ -56,16 +56,36 @@ impl vm::HostResourceType for Counter { const DESCRIPTION: &'static str = "A monotonic counter"; } +fn counter_resource() -> vm::HostResourceTypeMeta { + vm::HostResourceTypeMeta::of::() +} + +fn counter_key() -> vm::ResourceTypeKey { + counter_resource().schema.key +} + +fn make_counter_contract() -> vm::HostFunctionSchema { + vm::HostFunctionSchema::with_return( + "demo::make_counter", + vec![vm::HostParamSchema::value("seed", vm::HostTypeSchema::Int)], + vm::HostTypeSchema::Resource(counter_key()), + ) +} + /// Creates a counter and returns its handle. -#[pd_host_function(name = "demo::make_counter")] -pub fn make_counter(seed: i64) -> VmResult> { - Ok(resource::Resource::new(Counter(seed as u64))) +#[pd_host_function(name = "demo::make_counter", contract = make_counter_contract)] +pub fn make_counter(vm: &mut Vm, seed: i64) -> VmResult { + let token = vm + .host_context() + .push_resource(Counter(seed as u64)) + .map_err(|error| VmError::HostError(error.to_string()))?; + Ok(token.handle().raw() as i64) } /// Reads the current count. #[pd_host_function(name = "demo::read_counter")] pub fn read_counter(counter: ResourceRef<'_, Counter>) -> VmResult { - Ok(*counter as u64 as i64) + Ok(counter.get().0 as i64) } /// Consumes the counter. @@ -80,9 +100,9 @@ Rules that follow from the table above: - **Typed resource wrappers carry the effect.** `ResourceRef<'_, T>` is a `Borrow` effect, `ResourceMut<'_, T>` is `BorrowMut`, `ResourceOwned` - is `TakeOwned`, and returning `Resource` is `Create`. The guest - parameter/return schemas and the resource declarations both come from those - wrappers; nothing needs to be repeated in a catalog. + is `TakeOwned`. A creation path inserts `T` through `HostContext`, returns + the raw `i64` token expected by the current adapter, and declares a typed + resource return contract; that contract supplies the `Create` effect. - **One declaration per resource type.** Implement `HostResourceType` once. Every function that mentions `T` contributes that declaration, identical duplicates dedupe, and the same key claimed by a different Rust type fails before any @@ -119,25 +139,45 @@ boundary — resolve it in a separate host call. ### Named structs Rust signatures cannot spell field names. For a fixed-shape value, implement -`HostNamedStruct` and mark the declaration: +`HostNamedStruct` and mark the declaration. `HostNamedStruct` supplies only the +schema; the returned Rust value must also use the adapter's runtime conversion: ```rust -pub struct JitConfig; +pub struct JitConfig { + enabled: bool, +} impl vm::HostNamedStruct for JitConfig { const NAME: &'static str = "JitConfig"; + fn host_struct_fields() -> Vec { vec![vm::HostStructField::new("enabled", vm::HostTypeSchema::Bool)] } } +impl vm::IntoHostCallOutcome for JitConfig { + fn into_host_call_outcome(self) -> vm::CallOutcome { + vm::CallOutcome::Return(vm::return_one(vm::Value::map(vec![( + vm::Value::string("enabled"), + vm::Value::Bool(self.enabled), + )]))) + } +} + +/// Returns the current JIT configuration. #[pd_host_function(name = "jit::get_config")] #[pd_host_named_struct] -pub fn get_config(vm: &mut Vm) -> VmResult { /* ... */ } +pub fn get_config() -> VmResult { + Ok(JitConfig { enabled: true }) +} ``` The generated descriptor emits the full `Named { name, fields }` schema and the -module catalog derives the named struct from it. Do **not** model public host +module catalog derives the named struct from it. The adapter has a blanket +`IntoHostCallOutcome` implementation for built-in scalar and container outputs +that implement `IntoVmValue`. The public SDK re-exports +`IntoHostCallOutcome`, so a custom named Rust type implements that trait and +returns its map-shaped `Value` explicitly, as shown. Do **not** model public host request/result/event values as `Map(unknown)` or `unknown`; the `tests/typed_host_no_dynamic_contract_tests.rs` guard fails the build if a public standard function regresses to a dynamic schema. @@ -254,8 +294,9 @@ What the contract does and does not change: - Resource *declarations* stay with the module: implement `HostResourceType` once for the concrete type and list it in `HostModuleDescriptor::resources`. Then the resource key, its description, and its Rust type identity have exactly one - source. Deriving the contract's key from that declaration (as above) keeps the - two from drifting. + source. The current I/O module follows this rule with `io_file_key()`, which + returns `io_file_resource().schema.key`; contract code should likewise obtain + the key from its canonical resource factory instead of repeating a key string. ## 4. Installing a module diff --git a/docs/http-client.md b/docs/http-client.md index b86206b0..dd9cee39 100644 --- a/docs/http-client.md +++ b/docs/http-client.md @@ -251,6 +251,8 @@ The network future never owns or re-enters the VM. Callback error, protocol comp | `allow_private_ips` | `false` | Reject private and other special-use addresses | | `max_redirects` | 5 | Buffered/SSE redirect bound | | `max_request_body_bytes` | 1 MiB | Request body bound | +| `max_request_header_count` | 100 | Caller-supplied request header field-count bound | +| `max_request_header_bytes` | 64 KiB | Serialized caller-supplied request header block bound | | `max_response_body_bytes` | 8 MiB | Buffered response body bound | | `connect_timeout` | 10 s | DNS/connect/TLS phase bound | | `request_timeout` | 30 s | Buffered request total duration |