From 28945a3ac7ce9faa039a1096ea166a5bd4417324 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 04:46:41 +0000 Subject: [PATCH 01/19] test(client): cover surplus decisions endpoint and model validation Add tests for the Surplus provider covering the live response shape, posting to the v1 decisions endpoint with a namespaced JEV model id, rejecting responses from a different JEV release, and confirming Surplus is the only provider without native box question support. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/client/mod_tests.rs | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/crates/tinyinference-decisions/src/client/mod_tests.rs b/crates/tinyinference-decisions/src/client/mod_tests.rs index 2d984e82..f7b12379 100644 --- a/crates/tinyinference-decisions/src/client/mod_tests.rs +++ b/crates/tinyinference-decisions/src/client/mod_tests.rs @@ -825,3 +825,89 @@ async fn self_hosted_preserves_all_decision_types_and_rejects_invalid_answers() assert_eq!(failure.attempts, 0); assert!(requests.lock().await.is_empty()); } + +fn surplus_request() -> EvaluationRequest { + let mut request = request(); + request.model = "jev-1.14".into(); + request +} + +/// The live Surplus response shape (2026-11-15): an OpenRouter-style resolved +/// model id plus generation id, provider, and cost fields the crate ignores. +fn surplus_success(model: &str) -> String { + json!({ + "model": model, + "answers": { + "route": { + "type": "choice", + "choice": "bob", + "probabilities": {"alice": 0.2, "bob": 0.8}, + "confidence": 0.6 + } + }, + "usage": { + "input_tokens": 292, + "output_tokens": 20, + "cost": 0.000_012_2, + "cost_details": {"upstream_inference_cost": 0.000_012_2}, + "is_byok": false + }, + "id": "gen-dec-123", + "provider": "TypeSafe" + }) + .to_string() +} + +#[tokio::test] +async fn surplus_posts_to_v1_decisions_and_accepts_a_namespaced_jev_release() { + let mut config = ClientConfig::surplus("secret-test-key"); + config.retry.max_retries = 0; + let (client, requests) = mock_client( + config, + vec![response( + 200, + &surplus_success("typesafe/jev-1.14-20261014"), + "", + )], + ); + + let result = client.evaluate(&surplus_request()).await.unwrap(); + + assert_eq!(result.response.model, "typesafe/jev-1.14-20261014"); + assert_eq!(result.response.usage.input_tokens, Some(292)); + assert_eq!(result.response.usage.cost_usd_nanos, Some(12_200)); + let requests = requests.lock().await; + let url = reqwest::Url::parse(&requests[0].url).unwrap(); + assert_eq!(url.host_str(), Some("api.surplusintelligence.ai")); + assert_eq!(url.path(), "/v1/decisions"); + assert_eq!( + requests[0].headers.get("authorization").unwrap(), + "Bearer secret-test-key" + ); + assert!(requests[0].body.contains("\"model\":\"jev-1.14\"")); +} + +#[tokio::test] +async fn surplus_rejects_a_response_from_another_jev_release() { + let mut config = ClientConfig::surplus("secret-test-key"); + config.retry.max_retries = 0; + let (client, _) = mock_client( + config, + vec![response(200, &surplus_success("typesafe/jev-1.13"), "")], + ); + let failure = evaluation_failure(client.evaluate(&surplus_request()).await); + assert!(matches!(*failure.error, Error::InvalidResponse { .. })); +} + +#[test] +fn surplus_is_the_only_provider_without_native_box_questions() { + assert!(!Provider::Surplus.answers_box()); + for provider in [ + Provider::TypeSafe, + Provider::OpenRouter, + Provider::OpenJev, + Provider::SelfHosted, + ] { + assert!(provider.answers_box(), "{provider:?}"); + } +} From 28de1be889bcdab12c11919e56081a6a16432088 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 04:46:55 +0000 Subject: [PATCH 02/19] feat(client): add Surplus Intelligence provider Adds a Surplus provider with a `surplus` config constructor pointing at the Surplus decisions API, and routes its responses through the OpenRouter validation path since it names releases in the `typesafe/` namespace. Surplus rejects Box questions, so `answers_box` reports that and the client emulates them with Noul questions over image crops. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../tinyinference-decisions/src/client/mod.rs | 2 +- .../src/client/types.rs | 40 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/crates/tinyinference-decisions/src/client/mod.rs b/crates/tinyinference-decisions/src/client/mod.rs index 67a17142..faf08de2 100644 --- a/crates/tinyinference-decisions/src/client/mod.rs +++ b/crates/tinyinference-decisions/src/client/mod.rs @@ -170,7 +170,7 @@ impl Client { Provider::TypeSafe | Provider::OpenJev | Provider::SelfHosted => { response.validate_for(request) } - Provider::OpenRouter => response.validate_for_openrouter(request), + Provider::OpenRouter | Provider::Surplus => response.validate_for_openrouter(request), } } } diff --git a/crates/tinyinference-decisions/src/client/types.rs b/crates/tinyinference-decisions/src/client/types.rs index b17b47ae..e4984c1e 100644 --- a/crates/tinyinference-decisions/src/client/types.rs +++ b/crates/tinyinference-decisions/src/client/types.rs @@ -7,6 +7,8 @@ use crate::{Error, EvaluationResponse}; const DEFAULT_SYSTEM_ONE_PATH: &str = "v1/systemone"; const TINYHUMANS_SYSTEM_ONE_PATH: &str = "agent-integrations/openrouter/systemone"; const OPENJEV_BASE_URL: &str = "https://api.openjev.sh"; +const SURPLUS_BASE_URL: &str = "https://api.surplusintelligence.ai"; +const SURPLUS_DECISIONS_PATH: &str = "v1/decisions"; /// Hosted or self-hosted System One API provider. #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] @@ -26,6 +28,24 @@ pub enum Provider { /// key may be empty for an unauthenticated local server, in which case no /// `Authorization` header is sent. SelfHosted, + /// Surplus Intelligence's decisions API, which serves `TypeSafe`'s Jev at + /// `/v1/decisions`. + /// + /// Responses name the resolved release in the `typesafe/` namespace, as + /// with `OpenRouter`. Surplus rejects Box questions, so the client + /// emulates them (see [`BoxSupport`]). + Surplus, +} + +impl Provider { + /// Whether this provider answers [`Question::Box`](crate::Question::Box) + /// natively. A provider that does not gets Box questions emulated with + /// Noul questions over image crops unless the caller overrides it with + /// [`ClientConfig::with_box_support`]. + #[must_use] + pub fn answers_box(self) -> bool { + !matches!(self, Self::Surplus) + } } /// Async Jev-compatible System One client. @@ -110,6 +130,26 @@ impl ClientConfig { } } + /// Create configuration for Surplus Intelligence's decisions API at + /// `https://api.surplusintelligence.ai/v1/decisions`. + /// + /// Surplus serves Jev under release ids such as `jev-1.14`; set + /// [`EvaluationRequest::model`](crate::EvaluationRequest::model) + /// accordingly. Box questions are emulated because Surplus rejects them. + #[must_use] + pub fn surplus(api_key: impl Into) -> Self { + Self { + api_key: ApiKey(api_key.into()), + base_url: SURPLUS_BASE_URL.to_owned(), + system_one_path: SURPLUS_DECISIONS_PATH, + endpoint_url: None, + provider: Provider::Surplus, + timeout: Duration::from_secs(30), + retry: RetryPolicy::default(), + sdk_name: None, + } + } + /// Create configuration for a self-hosted, Jev-compatible System One /// endpoint. /// From 835d12cad4eb47a670b4ac809e56751ee1d35720 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 04:48:45 +0000 Subject: [PATCH 03/19] feat(emulation): add crop and geometry helpers for image emulation Introduce an emulation module with crop and geometry utilities backed by the image crate, so decisions can reason about image regions and coordinates. The new code is covered by dedicated unit tests. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- Cargo.lock | 147 +++++++++++++- crates/tinyinference-decisions/Cargo.toml | 3 + .../src/emulation/crop.rs | 54 ++++++ .../src/emulation/crop_tests.rs | 95 +++++++++ .../src/emulation/geometry.rs | 181 ++++++++++++++++++ .../src/emulation/geometry_tests.rs | 123 ++++++++++++ .../src/emulation/mod.rs | 5 + crates/tinyinference-decisions/src/lib.rs | 1 + 8 files changed, 605 insertions(+), 4 deletions(-) create mode 100644 crates/tinyinference-decisions/src/emulation/crop.rs create mode 100644 crates/tinyinference-decisions/src/emulation/crop_tests.rs create mode 100644 crates/tinyinference-decisions/src/emulation/geometry.rs create mode 100644 crates/tinyinference-decisions/src/emulation/geometry_tests.rs create mode 100644 crates/tinyinference-decisions/src/emulation/mod.rs diff --git a/Cargo.lock b/Cargo.lock index e8be6430..5b6f7bd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.5" @@ -173,6 +179,18 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.12.1" @@ -229,6 +247,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b929dd2a135771f40afabd1073f85eed59c800b7c61901e5c254270497dad64" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-common" version = "0.2.2" @@ -299,7 +326,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -320,12 +347,32 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "find-msvc-tools" version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" +[[package]] +name = "flate2" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4" +dependencies = [ + "crc32fast", + "miniz_oxide 0.9.1", + "zlib-rs", +] + [[package]] name = "foldhash" version = "0.2.0" @@ -701,6 +748,21 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "moxcms", + "num-traits", + "png", + "zune-core", + "zune-jpeg", +] + [[package]] name = "ipnet" version = "2.12.2" @@ -824,6 +886,26 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.4" @@ -835,6 +917,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "nom" version = "7.1.3" @@ -954,6 +1046,19 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.9", +] + [[package]] name = "potential_utf" version = "0.1.6" @@ -978,6 +1083,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + [[package]] name = "quinn" version = "0.11.12" @@ -1031,7 +1142,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1253,7 +1364,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1456,6 +1567,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "slab" version = "0.4.12" @@ -1565,7 +1682,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1646,6 +1763,7 @@ dependencies = [ "axum", "base64 0.23.1", "httpdate", + "image", "reqwest", "serde", "serde_json", @@ -2542,6 +2660,12 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" @@ -2575,3 +2699,18 @@ dependencies = [ "cc", "pkg-config", ] + +[[package]] +name = "zune-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/crates/tinyinference-decisions/Cargo.toml b/crates/tinyinference-decisions/Cargo.toml index dc37d71b..05b2c946 100644 --- a/crates/tinyinference-decisions/Cargo.toml +++ b/crates/tinyinference-decisions/Cargo.toml @@ -16,6 +16,9 @@ async-trait = { workspace = true } # Encodes attached images for the System One `assets` map. base64 = "0.23" httpdate = { workspace = true } +# Decodes, crops, and re-encodes images to emulate Box questions on providers +# that only answer Noul; PNG and JPEG are the formats the provider accepts. +image = { version = "0.25.10", default-features = false, features = ["jpeg", "png"] } reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/tinyinference-decisions/src/emulation/crop.rs b/crates/tinyinference-decisions/src/emulation/crop.rs new file mode 100644 index 00000000..bbb7672a --- /dev/null +++ b/crates/tinyinference-decisions/src/emulation/crop.rs @@ -0,0 +1,54 @@ +//! Decoding a source image and encoding upscaled crops of it. + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; +use image::{DynamicImage, ImageFormat as Codec, imageops::FilterType}; + +use super::geometry::Rect; +use crate::request::image::{MAX_SIDE, MIN_SIDE}; +use crate::{Error, ImageAsset, ImageFormat, Result}; + +/// Decode the image attached under `key`. +pub(crate) fn decode(key: &str, asset: &ImageAsset) -> Result { + let bytes = STANDARD + .decode(&asset.data_base64) + .map_err(|_| Error::invalid_request(format!("image `{key}` must be padded base64")))?; + let codec = match asset.format { + ImageFormat::Png => Codec::Png, + ImageFormat::Jpg => Codec::Jpeg, + }; + image::load_from_memory_with_format(&bytes, codec).map_err(|_| { + Error::invalid_request(format!( + "image `{key}` could not be decoded to emulate a Box question" + )) + }) +} + +/// Cut `rect` out of `image` as a PNG asset the provider accepts, returning it +/// with its file size. +/// +/// A crop whose shorter side is under the provider's minimum is upscaled +/// (bilinear) until it reaches it, keeping its aspect ratio unless that would +/// break the maximum side, which then clamps the longer side. +pub(crate) fn crop(image: &DynamicImage, rect: Rect) -> Result<(ImageAsset, usize)> { + let (width, height) = (rect.width(), rect.height()); + let mut crop = image.crop_imm(rect.left, rect.top, width, height); + let shorter = width.min(height); + if shorter < MIN_SIDE { + let scale = |side: u32| { + u32::try_from((u64::from(side) * u64::from(MIN_SIDE)).div_ceil(u64::from(shorter))) + .unwrap_or(MAX_SIDE) + .clamp(MIN_SIDE, MAX_SIDE) + }; + crop = crop.resize_exact(scale(width), scale(height), FilterType::Triangle); + } + let mut bytes = std::io::Cursor::new(Vec::new()); + crop.write_to(&mut bytes, Codec::Png) + .map_err(|_| Error::invalid_request("an image crop could not be encoded as PNG"))?; + let bytes = bytes.into_inner(); + Ok((ImageAsset::png(&bytes), bytes.len())) +} + +#[cfg(test)] +#[path = "crop_tests.rs"] +pub(crate) mod tests; diff --git a/crates/tinyinference-decisions/src/emulation/crop_tests.rs b/crates/tinyinference-decisions/src/emulation/crop_tests.rs new file mode 100644 index 00000000..ed503560 --- /dev/null +++ b/crates/tinyinference-decisions/src/emulation/crop_tests.rs @@ -0,0 +1,95 @@ +//! Decoding source images and encoding upscaled crops. + +use image::{ImageFormat as Codec, Rgb, RgbImage}; + +use super::*; +use crate::request::image::{MAX_SIDE, MIN_SIDE}; + +/// A white image with a red square covering `red`. +pub(crate) fn picture(width: u32, height: u32, red: Rect) -> RgbImage { + RgbImage::from_fn(width, height, |x, y| { + if (red.left..red.right).contains(&x) && (red.top..red.bottom).contains(&y) { + Rgb([255, 0, 0]) + } else { + Rgb([255, 255, 255]) + } + }) +} + +pub(crate) fn encode(image: &RgbImage, codec: Codec) -> Vec { + let mut bytes = std::io::Cursor::new(Vec::new()); + image.write_to(&mut bytes, codec).unwrap(); + bytes.into_inner() +} + +pub(crate) fn is_red(pixel: &image::Rgba) -> bool { + pixel[0] > 200 && pixel[1] < 80 && pixel[2] < 80 +} + +fn rect(left: u32, top: u32, right: u32, bottom: u32) -> Rect { + Rect { + left, + top, + right, + bottom, + } +} + +fn decoded(asset: &ImageAsset) -> image::DynamicImage { + assert_eq!(asset.format, ImageFormat::Png); + decode("crop", asset).unwrap() +} + +#[test] +fn a_small_crop_is_upscaled_to_the_minimum_side_and_keeps_its_content() { + let source = picture(400, 300, rect(120, 80, 140, 100)); + let image = decode("page", &ImageAsset::png(&encode(&source, Codec::Png))).unwrap(); + let (asset, bytes) = crop(&image, rect(100, 75, 200, 150)).unwrap(); + let crop = decoded(&asset); + assert_eq!((crop.width(), crop.height()), (374, MIN_SIDE)); + assert_eq!(bytes, asset.checked_len().unwrap()); + assert!(crop.to_rgba8().pixels().any(is_red)); + + let (elsewhere, _) = crop_of(&image, rect(200, 150, 300, 225)); + assert!(!elsewhere.to_rgba8().pixels().any(is_red)); +} + +fn crop_of(image: &image::DynamicImage, rect: Rect) -> (image::DynamicImage, usize) { + let (asset, bytes) = crop(image, rect).unwrap(); + (decoded(&asset), bytes) +} + +#[test] +fn a_crop_already_large_enough_is_not_resized() { + let source = picture(1000, 800, rect(0, 0, 1, 1)); + let image = decode("page", &ImageAsset::png(&encode(&source, Codec::Png))).unwrap(); + let (crop, _) = crop_of(&image, rect(100, 100, 400, 390)); + assert_eq!((crop.width(), crop.height()), (300, 290)); +} + +#[test] +fn an_extreme_aspect_crop_is_clamped_to_the_maximum_side() { + let source = picture(1000, 800, rect(0, 0, 1, 1)); + let image = decode("page", &ImageAsset::png(&encode(&source, Codec::Png))).unwrap(); + let (crop, _) = crop_of(&image, rect(0, 0, 400, 50)); + assert_eq!((crop.width(), crop.height()), (MAX_SIDE, MIN_SIDE)); +} + +#[test] +fn jpeg_sources_are_decoded_and_cropped_to_png() { + let source = picture(600, 400, rect(0, 0, 300, 200)); + let image = decode("page", &ImageAsset::jpeg(&encode(&source, Codec::Jpeg))).unwrap(); + assert_eq!((image.width(), image.height()), (600, 400)); + let (crop, _) = crop_of(&image, rect(0, 0, 300, 200)); + assert!(crop.to_rgba8().pixels().any(is_red)); +} + +#[test] +fn an_undecodable_image_names_its_asset() { + let asset = ImageAsset::png(&crate::request::image::tests::png_header(400, 300)); + let error = decode("page", &asset).unwrap_err(); + assert!( + matches!(&error, Error::InvalidRequest { reason } if reason.contains("`page`")), + "{error:?}" + ); +} diff --git a/crates/tinyinference-decisions/src/emulation/geometry.rs b/crates/tinyinference-decisions/src/emulation/geometry.rs new file mode 100644 index 00000000..41e1c593 --- /dev/null +++ b/crates/tinyinference-decisions/src/emulation/geometry.rs @@ -0,0 +1,181 @@ +//! Pixel rectangles, grids over them, and selecting a box from cell scores. + +use crate::BoxEdges; + +/// A cell counts as containing the target from this probability up. +pub(crate) const POSITIVE: f64 = 0.5; +/// Refinement cells must be at least this many pixels on their shorter side; +/// smaller crops carry too little of the original image to be worth a round. +const MIN_REFINE_CELL_PX: u32 = 32; +/// A refinement region is widened or heightened until neither side exceeds +/// this multiple of the other, so its cells can be upscaled to the provider's +/// minimum side without breaking its maximum. +const MAX_REFINE_ASPECT: u32 = 4; + +/// A pixel rectangle; `right` and `bottom` are exclusive. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct Rect { + pub(crate) left: u32, + pub(crate) top: u32, + pub(crate) right: u32, + pub(crate) bottom: u32, +} + +impl Rect { + /// The whole of a `width` by `height` image. + pub(crate) fn image(width: u32, height: u32) -> Self { + Self { + left: 0, + top: 0, + right: width, + bottom: height, + } + } + + pub(crate) fn width(self) -> u32 { + self.right - self.left + } + + pub(crate) fn height(self) -> u32 { + self.bottom - self.top + } + + /// This rectangle as fractions of a `width` by `height` image. + pub(crate) fn edges(self, width: u32, height: u32) -> BoxEdges { + let (width, height) = (f64::from(width), f64::from(height)); + BoxEdges { + top: f64::from(self.top) / height, + bottom: f64::from(self.bottom) / height, + left: f64::from(self.left) / width, + right: f64::from(self.right) / width, + } + } + + fn union(self, other: Self) -> Self { + Self { + left: self.left.min(other.left), + top: self.top.min(other.top), + right: self.right.max(other.right), + bottom: self.bottom.max(other.bottom), + } + } +} + +/// The region selected from a grid of cell probabilities. +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) struct Pick { + /// Bounding rectangle of the selected cells. + pub(crate) rect: Rect, + /// Highest cell probability. + pub(crate) confidence: f64, + /// Whether any cell reached [`POSITIVE`]. + pub(crate) found: bool, +} + +/// An `n` by `n` grid over `region`, row-major. Cell boundaries are floored, +/// so the cells tile the region exactly. +pub(crate) fn grid(region: Rect, n: u32) -> Vec { + let split = |start: u32, length: u32, index: u32| { + let offset = u64::from(length) * u64::from(index) / u64::from(n); + start + u32::try_from(offset).unwrap_or(length) + }; + let (width, height) = (region.width(), region.height()); + (0..n) + .flat_map(|row| (0..n).map(move |column| (row, column))) + .map(|(row, column)| Rect { + left: split(region.left, width, column), + top: split(region.top, height, row), + right: split(region.left, width, column + 1), + bottom: split(region.top, height, row + 1), + }) + .collect() +} + +/// Select a box from the row-major `n` by `n` `cells` and their +/// `probabilities`. +/// +/// The best cell (the first, on ties) anchors the box. When it reaches +/// [`POSITIVE`], the box is the bounding rectangle of every positive cell +/// connected to it, diagonals included; other positive cells are taken to be +/// another instance. Otherwise the box is the best cell alone. +pub(crate) fn pick(cells: &[Rect], probabilities: &[f64], n: u32) -> Pick { + let n = usize::try_from(n).unwrap_or(usize::MAX); + let best = probabilities + .iter() + .enumerate() + .fold(0, |best, (index, probability)| { + if *probability > probabilities[best] { + index + } else { + best + } + }); + let confidence = probabilities[best]; + let found = confidence >= POSITIVE; + let mut rect = cells[best]; + if found { + let mut seen = vec![false; cells.len()]; + let mut pending = vec![best]; + seen[best] = true; + while let Some(index) = pending.pop() { + rect = rect.union(cells[index]); + let (row, column) = (index / n, index % n); + for neighbour_row in row.saturating_sub(1)..=(row + 1).min(n - 1) { + for neighbour_column in column.saturating_sub(1)..=(column + 1).min(n - 1) { + let neighbour = neighbour_row * n + neighbour_column; + if !seen[neighbour] && probabilities[neighbour] >= POSITIVE { + seen[neighbour] = true; + pending.push(neighbour); + } + } + } + } + } + Pick { + rect, + confidence, + found, + } +} + +/// The region a refinement round grids, or `None` when refining `chosen` +/// would only repeat the first round (it is the whole `image`) or cut cells +/// smaller than [`MIN_REFINE_CELL_PX`]. +/// +/// A region more than [`MAX_REFINE_ASPECT`] times longer than it is wide (or +/// the reverse) is grown about its centre, shifted to stay inside the image. +pub(crate) fn refinement_region(chosen: Rect, image: Rect, n: u32) -> Option { + let mut region = chosen; + let (width, height) = (region.width(), region.height()); + if width > height.saturating_mul(MAX_REFINE_ASPECT) { + (region.top, region.bottom) = grow( + region.top, + height, + width.div_ceil(MAX_REFINE_ASPECT), + image.bottom, + ); + } else if height > width.saturating_mul(MAX_REFINE_ASPECT) { + (region.left, region.right) = grow( + region.left, + width, + height.div_ceil(MAX_REFINE_ASPECT), + image.right, + ); + } + let too_small = region.width().min(region.height()) < MIN_REFINE_CELL_PX.saturating_mul(n); + (region != image && !too_small).then_some(region) +} + +/// Grow the span `start..start + length` to `target` (at most `limit`) about +/// its centre, keeping it inside `0..limit`. +fn grow(start: u32, length: u32, target: u32, limit: u32) -> (u32, u32) { + let target = target.min(limit); + let start = start + .saturating_sub((target - length) / 2) + .min(limit - target); + (start, start + target) +} + +#[cfg(test)] +#[path = "geometry_tests.rs"] +mod tests; diff --git a/crates/tinyinference-decisions/src/emulation/geometry_tests.rs b/crates/tinyinference-decisions/src/emulation/geometry_tests.rs new file mode 100644 index 00000000..f623d2cf --- /dev/null +++ b/crates/tinyinference-decisions/src/emulation/geometry_tests.rs @@ -0,0 +1,123 @@ +//! Grid cells, cell selection, and refinement regions. + +use super::*; + +fn rect(left: u32, top: u32, right: u32, bottom: u32) -> Rect { + Rect { + left, + top, + right, + bottom, + } +} + +#[test] +fn grid_covers_the_region_row_major_without_gaps() { + let cells = grid(rect(0, 0, 10, 7), 3); + assert_eq!(cells.len(), 9); + assert_eq!(cells[0], rect(0, 0, 3, 2)); + assert_eq!(cells[1], rect(3, 0, 6, 2)); + assert_eq!(cells[2], rect(6, 0, 10, 2)); + assert_eq!(cells[3], rect(0, 2, 3, 4)); + assert_eq!(cells[8], rect(6, 4, 10, 7)); + let area: u32 = cells.iter().map(|cell| cell.width() * cell.height()).sum(); + assert_eq!(area, 70); +} + +#[test] +fn grid_is_offset_by_the_region_origin() { + let cells = grid(rect(100, 50, 300, 250), 2); + assert_eq!( + cells, + vec![ + rect(100, 50, 200, 150), + rect(200, 50, 300, 150), + rect(100, 150, 200, 250), + rect(200, 150, 300, 250), + ] + ); +} + +#[test] +fn edges_are_fractions_of_the_whole_image() { + let edges = rect(100, 50, 300, 250).edges(400, 500); + assert_eq!(edges.left, 0.25); + assert_eq!(edges.right, 0.75); + assert_eq!(edges.top, 0.1); + assert_eq!(edges.bottom, 0.5); +} + +#[test] +fn pick_unions_the_positive_component_around_the_best_cell() { + let cells = grid(rect(0, 0, 400, 400), 4); + #[rustfmt::skip] + let probabilities = [ + 0.1, 0.6, 0.0, 0.0, + 0.0, 0.9, 0.7, 0.0, + 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.8, + ]; + let pick = pick(&cells, &probabilities, 4); + // The 0.8 corner is positive but not connected to the 0.9 cell. + assert_eq!(pick.rect, rect(100, 0, 300, 200)); + assert_eq!(pick.confidence, 0.9); + assert!(pick.found); +} + +#[test] +fn pick_joins_diagonal_neighbours() { + let cells = grid(rect(0, 0, 400, 400), 4); + #[rustfmt::skip] + let probabilities = [ + 0.9, 0.0, 0.0, 0.0, + 0.0, 0.5, 0.0, 0.0, + 0.0, 0.0, 0.49, 0.0, + 0.0, 0.0, 0.0, 0.0, + ]; + let pick = pick(&cells, &probabilities, 4); + assert_eq!(pick.rect, rect(0, 0, 200, 200)); +} + +#[test] +fn pick_falls_back_to_the_first_best_cell_when_none_is_positive() { + let cells = grid(rect(0, 0, 400, 400), 4); + let mut probabilities = [0.1; 16]; + probabilities[6] = 0.3; + probabilities[9] = 0.3; + let pick = pick(&cells, &probabilities, 4); + assert_eq!(pick.rect, cells[6]); + assert_eq!(pick.confidence, 0.3); + assert!(!pick.found); +} + +#[test] +fn refinement_keeps_a_squarish_region_and_skips_the_whole_image() { + let image = rect(0, 0, 800, 600); + assert_eq!( + refinement_region(rect(200, 150, 400, 300), image, 4), + Some(rect(200, 150, 400, 300)) + ); + assert_eq!(refinement_region(image, image, 4), None); +} + +#[test] +fn refinement_widens_a_thin_region_inside_the_image() { + let image = rect(0, 0, 1200, 800); + // 50 wide, 400 tall: widened to 100 about its centre. + assert_eq!( + refinement_region(rect(600, 0, 650, 400), image, 2), + Some(rect(575, 0, 675, 400)) + ); + // Against the left edge the widening shifts right instead. + assert_eq!( + refinement_region(rect(0, 0, 50, 400), image, 2), + Some(rect(0, 0, 100, 400)) + ); +} + +#[test] +fn refinement_is_skipped_when_cells_would_be_too_small() { + let image = rect(0, 0, 1000, 1000); + assert_eq!(refinement_region(rect(0, 0, 100, 100), image, 4), None); + assert!(refinement_region(rect(0, 0, 128, 128), image, 4).is_some()); +} diff --git a/crates/tinyinference-decisions/src/emulation/mod.rs b/crates/tinyinference-decisions/src/emulation/mod.rs new file mode 100644 index 00000000..1132f904 --- /dev/null +++ b/crates/tinyinference-decisions/src/emulation/mod.rs @@ -0,0 +1,5 @@ +//! Box questions emulated with Noul questions over image crops, for providers +//! that do not answer Box natively. + +mod crop; +mod geometry; diff --git a/crates/tinyinference-decisions/src/lib.rs b/crates/tinyinference-decisions/src/lib.rs index 939c6a93..a83039aa 100644 --- a/crates/tinyinference-decisions/src/lib.rs +++ b/crates/tinyinference-decisions/src/lib.rs @@ -45,6 +45,7 @@ //! decision probability. mod client; +mod emulation; mod error; mod request; mod response; From 10f40763adee6d8f7a6772ae6ce18d44f7c01de8 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 04:51:22 +0000 Subject: [PATCH 04/19] feat(emulation): scaffold box emulation pipeline Add the skeleton of the box emulation module, including the emulator, probe, and response types plus the round, batching, and refinement entry points, all still unimplemented. Expose the request helper that detects whether a JSON value mentions a string so the emulation code can reuse it. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/emulation/mod.rs | 71 ++++ .../src/emulation/mod_tests.rs | 393 ++++++++++++++++++ .../src/request/mod.rs | 2 +- 3 files changed, 465 insertions(+), 1 deletion(-) create mode 100644 crates/tinyinference-decisions/src/emulation/mod_tests.rs diff --git a/crates/tinyinference-decisions/src/emulation/mod.rs b/crates/tinyinference-decisions/src/emulation/mod.rs index 1132f904..429f5618 100644 --- a/crates/tinyinference-decisions/src/emulation/mod.rs +++ b/crates/tinyinference-decisions/src/emulation/mod.rs @@ -3,3 +3,74 @@ mod crop; mod geometry; + +use std::collections::BTreeMap; + +use geometry::{Pick, Rect}; + +use crate::{ + Answer, Error, EvaluationRequest, EvaluationResponse, ImageAsset, ImageFormat, Question, + Result, Usage, +}; + +pub(crate) struct BoxEmulator<'a> { + request: &'a EvaluationRequest, +} + +pub(crate) struct Probe { + pub(crate) target: usize, + pub(crate) cell: Rect, + pub(crate) id: String, + pub(crate) question: Question, + pub(crate) asset: ImageAsset, + pub(crate) bytes: usize, +} + +#[derive(Default)] +pub(crate) struct Responses { + answers: BTreeMap, +} + +impl Responses { + pub(crate) fn add(&mut self, _response: EvaluationResponse, _request_id: Option) { + todo!() + } +} + +impl<'a> BoxEmulator<'a> { + pub(crate) fn new(_request: &'a EvaluationRequest, _grid: u32) -> Result { + todo!() + } + pub(crate) fn first_round(&self) -> Result> { + todo!() + } + pub(crate) fn refinement_round(&self, _picks: &[Pick]) -> Result> { + todo!() + } + pub(crate) fn batches( + &self, + _probes: &[Probe], + _with_base: bool, + _limit: usize, + ) -> Result> { + todo!() + } + pub(crate) fn picks(&self, _probes: &[Probe], _responses: &Responses) -> Result>> { + todo!() + } + pub(crate) fn response(&self, _responses: Responses, _picks: &[Pick]) -> (EvaluationResponse, Option) { + todo!() + } +} + +pub(crate) fn refine(_first: Vec, _second: Vec>) -> Vec { + todo!() +} + +fn source_asset(_request: &EvaluationRequest, _id: &str, _instructions: &serde_json::Value) -> Result { + todo!() +} + +#[cfg(test)] +#[path = "mod_tests.rs"] +mod tests; diff --git a/crates/tinyinference-decisions/src/emulation/mod_tests.rs b/crates/tinyinference-decisions/src/emulation/mod_tests.rs new file mode 100644 index 00000000..2ef5a901 --- /dev/null +++ b/crates/tinyinference-decisions/src/emulation/mod_tests.rs @@ -0,0 +1,393 @@ +//! Source selection, probe generation, batching, and answer synthesis. + +use std::collections::BTreeMap; + +use image::ImageFormat as Codec; +use serde_json::json; + +use super::crop::tests::{encode, picture}; +use super::*; +use crate::{BoxQuestion, Noul, NoulAnswer}; + +fn rect(left: u32, top: u32, right: u32, bottom: u32) -> Rect { + Rect { + left, + top, + right, + bottom, + } +} + +fn png(width: u32, height: u32) -> ImageAsset { + ImageAsset::png(&encode( + &picture(width, height, rect(0, 0, 1, 1)), + Codec::Png, + )) +} + +fn boxed(instructions: serde_json::Value) -> Question { + Question::Box(BoxQuestion { instructions }) +} + +fn noul(instructions: &str) -> Question { + Question::Noul(Noul { + instructions: json!(instructions), + criteria: None, + }) +} + +fn request(state: serde_json::Value, questions: Vec<(&str, Question)>) -> EvaluationRequest { + EvaluationRequest::new( + "jev-1.14", + state, + questions + .into_iter() + .map(|(id, question)| (id.to_owned(), question)) + .collect(), + ) +} + +fn source_of(request: &EvaluationRequest) -> Result { + let (id, question) = request.questions.iter().next().unwrap(); + let Question::Box(question) = question else { + panic!("first question must be a Box"); + }; + source_asset(request, id, &question.instructions) +} + +#[test] +fn the_source_is_an_asset_key_named_in_the_instructions() { + let request = request( + json!({"a": "shot-a", "b": "shot-b"}), + vec![("button", boxed(json!("the Verify button in shot-b")))], + ) + .with_image("shot-a", png(300, 300)) + .with_image("shot-b", png(300, 300)); + assert_eq!(source_of(&request).unwrap(), "shot-b"); +} + +#[test] +fn a_key_contained_in_a_longer_named_key_is_not_ambiguous() { + let request = request( + json!({"a": "img-1", "b": "img-10"}), + vec![("button", boxed(json!("the Verify button in img-10")))], + ) + .with_image("img-1", png(300, 300)) + .with_image("img-10", png(300, 300)); + assert_eq!(source_of(&request).unwrap(), "img-10"); +} + +#[test] +fn the_source_is_the_asset_held_by_a_backticked_state_field() { + let request = request( + json!({"page": "shot-a", "logo": "shot-b"}), + vec![("button", boxed(json!({"find": "the Verify button in `page`"})))], + ) + .with_image("shot-a", png(300, 300)) + .with_image("shot-b", png(300, 300)); + assert_eq!(source_of(&request).unwrap(), "shot-a"); +} + +#[test] +fn the_source_is_the_sole_image_otherwise() { + let request = request( + json!({"page": "shot-a"}), + vec![("button", boxed(json!("the Verify button")))], + ) + .with_image("shot-a", png(300, 300)); + assert_eq!(source_of(&request).unwrap(), "shot-a"); +} + +#[test] +fn an_unnamed_or_ambiguous_source_is_a_request_error() { + let two = |instructions: &str| { + request( + json!({"a": "shot-a", "b": "shot-b"}), + vec![("button", boxed(json!(instructions)))], + ) + .with_image("shot-a", png(300, 300)) + .with_image("shot-b", png(300, 300)) + }; + for instructions in ["the Verify button", "the button in shot-a or shot-b", "`a` or `b`"] { + let error = source_of(&two(instructions)).unwrap_err(); + assert!( + matches!(&error, Error::InvalidRequest { reason } if reason.contains("`button`")), + "{instructions}: {error:?}" + ); + } + let none = request(json!("no image"), vec![("button", boxed(json!("x")))]); + assert!(matches!( + source_of(&none), + Err(Error::InvalidRequest { .. }) + )); +} + +#[test] +fn reserved_probe_ids_and_keys_are_rejected() { + let clash = request( + json!({"page": "shot"}), + vec![ + ("button", boxed(json!("the button"))), + ("__tinyinference_box_mine", noul("anything")), + ], + ) + .with_image("shot", png(300, 300)); + assert!(matches!( + BoxEmulator::new(&clash, 4), + Err(Error::InvalidRequest { .. }) + )); +} + +fn screenshot_request() -> EvaluationRequest { + request( + json!({"page": "shot"}), + vec![ + ("button", boxed(json!("the Verify button in `page`"))), + ("present", noul("Is there a Verify button in `page`?")), + ("polite", noul("Is the page polite?")), + ], + ) + .with_image("shot", png(800, 400)) +} + +#[test] +fn the_first_round_asks_one_noul_per_grid_cell_with_its_own_crop() { + let request = screenshot_request(); + let emulator = BoxEmulator::new(&request, 4).unwrap(); + let probes = emulator.first_round().unwrap(); + assert_eq!(probes.len(), 16); + assert_eq!(probes[0].cell, rect(0, 0, 200, 100)); + assert_eq!(probes[15].cell, rect(600, 300, 800, 400)); + assert_eq!(probes[5].id, "__tinyinference_box0_round0_r1c1"); + let Question::Noul(question) = &probes[5].question else { + panic!("probes must be Noul questions"); + }; + let text = question.instructions.as_str().unwrap(); + assert!(text.contains("`__tinyinference_box0_round0_r1c1`"), "{text}"); + assert!(text.contains("the Verify button in `page`"), "{text}"); + assert_eq!(probes[5].asset.format, ImageFormat::Png); + assert_eq!(probes[5].bytes, probes[5].asset.checked_len().unwrap()); +} + +#[test] +fn probes_join_a_base_request_that_does_not_carry_their_source() { + let request = request( + json!("a text-only state"), + vec![ + ("button", boxed(json!("the Verify button in shot"))), + ("polite", noul("Is the page polite?")), + ], + ) + .with_image("shot", png(800, 400)); + let emulator = BoxEmulator::new(&request, 2).unwrap(); + let probes = emulator.first_round().unwrap(); + let batches = emulator.batches(&probes, true, 32 * 1024 * 1024).unwrap(); + assert_eq!(batches.len(), 1); + request.validate().unwrap(); + let batch = &batches[0]; + batch.validate().unwrap(); + assert!(batch.needs_image_opt_in()); + assert_eq!(batch.state, request.state); + assert_eq!(batch.model, "jev-1.14"); + assert_eq!(batch.questions.len(), 5); + assert!(batch.questions.contains_key("polite")); + assert!(!batch.questions.contains_key("button")); + assert!(!batch.assets.contains_key("shot")); + assert!( + !batch + .questions + .values() + .any(|question| matches!(question, Question::Box(_))) + ); +} + +#[test] +fn a_base_request_carrying_the_source_image_is_sent_apart_from_its_crops() { + let request = screenshot_request(); + let emulator = BoxEmulator::new(&request, 2).unwrap(); + let probes = emulator.first_round().unwrap(); + let batches = emulator.batches(&probes, true, 32 * 1024 * 1024).unwrap(); + assert_eq!(batches.len(), 2); + let ids: Vec<&String> = batches[0].questions.keys().collect(); + assert_eq!(ids, ["polite", "present"]); + assert_eq!(batches[0].assets.keys().collect::>(), ["shot"]); + assert_eq!(batches[1].questions.len(), 4); + assert!(!batches[1].assets.contains_key("shot")); + for batch in &batches { + batch.validate().unwrap(); + } +} + +#[test] +fn a_request_of_only_box_questions_sends_only_probe_batches() { + let request = request( + json!({"page": "shot"}), + vec![("button", boxed(json!("the Verify button in `page`")))], + ) + .with_image("shot", png(800, 400)); + let emulator = BoxEmulator::new(&request, 2).unwrap(); + let probes = emulator.first_round().unwrap(); + let batches = emulator.batches(&probes, true, 32 * 1024 * 1024).unwrap(); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].questions.len(), 4); +} + +#[test] +fn batches_split_to_respect_the_image_byte_limit() { + let request = request( + json!({"page": "shot"}), + vec![("button", boxed(json!("the Verify button in `page`")))], + ) + .with_image("shot", png(800, 400)); + let emulator = BoxEmulator::new(&request, 4).unwrap(); + let probes = emulator.first_round().unwrap(); + let largest = probes.iter().map(|probe| probe.bytes).max().unwrap(); + let limit = largest * 3; + let batches = emulator.batches(&probes, true, limit).unwrap(); + assert!(batches.len() > 1, "{} batches", batches.len()); + let mut asked = 0; + for batch in &batches { + let total: usize = batch + .assets + .values() + .map(|crate::Asset::Image(image)| image.checked_len().unwrap()) + .sum(); + assert!(total <= limit); + asked += batch.questions.len(); + batch.validate().unwrap(); + } + assert_eq!(asked, 16); + + let error = emulator.batches(&probes, true, largest - 1).unwrap_err(); + assert!(matches!(error, Error::InvalidRequest { .. })); +} + +fn answers_for(probes: &[Probe], positive: &[&str]) -> Responses { + let mut responses = Responses::default(); + let answers = probes + .iter() + .map(|probe| { + let noul = if positive.iter().any(|id| probe.id.ends_with(id)) { + 0.9 + } else { + 0.1 + }; + (probe.id.clone(), Answer::Noul(NoulAnswer { noul })) + }) + .collect(); + responses.add( + EvaluationResponse { + model: "typesafe/jev-1.14-20261014".to_owned(), + answers, + usage: Usage { + input_tokens: Some(100), + output_tokens: Some(10), + cached_input_tokens: None, + cost_usd_nanos: Some(1_000), + }, + }, + Some("first".to_owned()), + ); + responses +} + +#[test] +fn box_answers_are_synthesised_and_probes_stripped() { + let request = screenshot_request(); + let emulator = BoxEmulator::new(&request, 4).unwrap(); + let probes = emulator.first_round().unwrap(); + let mut responses = answers_for(&probes, &["r1c1", "r1c2"]); + responses.add( + EvaluationResponse { + model: "typesafe/jev-1.14-20261014".to_owned(), + answers: BTreeMap::from([ + ("present".to_owned(), Answer::Noul(NoulAnswer { noul: 0.8 })), + ("polite".to_owned(), Answer::Noul(NoulAnswer { noul: 0.4 })), + ]), + usage: Usage { + input_tokens: Some(50), + output_tokens: None, + cached_input_tokens: None, + cost_usd_nanos: Some(500), + }, + }, + Some("second".to_owned()), + ); + let picks: Vec = emulator + .picks(&probes, &responses) + .unwrap() + .into_iter() + .map(Option::unwrap) + .collect(); + let (response, request_id) = emulator.response(responses, &picks); + assert_eq!(request_id.as_deref(), Some("first")); + assert_eq!(response.model, "typesafe/jev-1.14-20261014"); + assert_eq!( + response.answers.keys().collect::>(), + ["button", "polite", "present"] + ); + let Answer::Box(answer) = response.answers["button"] else { + panic!("button must be answered with a Box"); + }; + assert_eq!(answer.confidence, 0.9); + assert_eq!(answer.edges.left, 0.25); + assert_eq!(answer.edges.right, 0.75); + assert_eq!(answer.edges.top, 0.25); + assert_eq!(answer.edges.bottom, 0.5); + assert_eq!(response.usage.input_tokens, Some(150)); + assert_eq!(response.usage.output_tokens, Some(10)); + assert_eq!(response.usage.cost_usd_nanos, Some(1_500)); + response.validate_for_openrouter(&request).unwrap(); +} + +#[test] +fn a_missing_probe_answer_is_a_response_error() { + let request = screenshot_request(); + let emulator = BoxEmulator::new(&request, 2).unwrap(); + let probes = emulator.first_round().unwrap(); + let responses = answers_for(&probes[..3], &[]); + assert!(matches!( + emulator.picks(&probes, &responses), + Err(Error::InvalidResponse { .. }) + )); +} + +#[test] +fn refinement_grids_the_chosen_region_and_replaces_only_a_found_box() { + let request = request( + json!({"page": "shot"}), + vec![("button", boxed(json!("the Verify button in `page`")))], + ) + .with_image("shot", png(800, 800)); + let emulator = BoxEmulator::new(&request, 4).unwrap(); + let first = Pick { + rect: rect(200, 200, 400, 400), + confidence: 0.9, + found: true, + }; + let probes = emulator.refinement_round(&[first]).unwrap(); + assert_eq!(probes.len(), 16); + assert_eq!(probes[0].cell, rect(200, 200, 250, 250)); + assert_eq!(probes[0].id, "__tinyinference_box0_round1_r0c0"); + + let found = answers_for(&probes, &["r0c0"]); + let refined = refine(vec![first], emulator.picks(&probes, &found).unwrap()); + assert_eq!(refined[0].rect, rect(200, 200, 250, 250)); + assert_eq!(refined[0].confidence, 0.9); + + let missed = answers_for(&probes, &[]); + let kept = refine(vec![first], emulator.picks(&probes, &missed).unwrap()); + assert_eq!(kept[0], first); +} + +#[test] +fn refinement_skips_a_box_that_already_covers_the_image() { + let request = screenshot_request(); + let emulator = BoxEmulator::new(&request, 4).unwrap(); + let whole = Pick { + rect: Rect::image(800, 400), + confidence: 0.9, + found: true, + }; + assert!(emulator.refinement_round(&[whole]).unwrap().is_empty()); +} diff --git a/crates/tinyinference-decisions/src/request/mod.rs b/crates/tinyinference-decisions/src/request/mod.rs index 4d42fc80..d918e973 100644 --- a/crates/tinyinference-decisions/src/request/mod.rs +++ b/crates/tinyinference-decisions/src/request/mod.rs @@ -86,7 +86,7 @@ impl EvaluationRequest { } /// Whether any decoded string value or object key inside `value` contains `needle`. -fn value_mentions(value: &serde_json::Value, needle: &str) -> bool { +pub(crate) fn value_mentions(value: &serde_json::Value, needle: &str) -> bool { match value { serde_json::Value::String(text) => text.contains(needle), serde_json::Value::Array(items) => items.iter().any(|item| value_mentions(item, needle)), From b940d8cb8df9c48620efa412e4055b3a2a94241e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 04:52:09 +0000 Subject: [PATCH 05/19] feat(emulation): implement box question emulation via noul probes Implement the box emulator that cuts each box question's image into a grid, asks noul probes about every cell, and reconstructs the box from the positive cells connected to the best one, with an optional refinement round. Probe ids and assets are namespaced under a reserved prefix so they never reach the caller, and sub-response usage and metadata are accumulated across rounds. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/emulation/mod.rs | 460 +++++++++++++++++- 1 file changed, 436 insertions(+), 24 deletions(-) diff --git a/crates/tinyinference-decisions/src/emulation/mod.rs b/crates/tinyinference-decisions/src/emulation/mod.rs index 429f5618..02029ba0 100644 --- a/crates/tinyinference-decisions/src/emulation/mod.rs +++ b/crates/tinyinference-decisions/src/emulation/mod.rs @@ -1,74 +1,486 @@ //! Box questions emulated with Noul questions over image crops, for providers //! that do not answer Box natively. +//! +//! Each Box question's image is cut into an `n` by `n` grid. Every cell is +//! upscaled to the provider's minimum side, encoded as PNG, and asked about as +//! a Noul probe ("does this tile show the target?"). The box is the bounding +//! rectangle of the positive cells connected to the best one, and an optional +//! refinement round grids that box again. Probes never reach the caller: the +//! response carries exactly the caller's question ids. mod crop; mod geometry; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; -use geometry::{Pick, Rect}; +use image::DynamicImage; +use serde_json::{Value, json}; +pub(crate) use geometry::Pick; +use geometry::Rect; + +use crate::request::image::MAX_IMAGE_BYTES; +use crate::request::value_mentions; use crate::{ - Answer, Error, EvaluationRequest, EvaluationResponse, ImageAsset, ImageFormat, Question, - Result, Usage, + Answer, Asset, BoxAnswer, Error, EvaluationRequest, EvaluationResponse, ImageAsset, Noul, + NoulCriteria, Question, Result, Usage, }; +/// Probe question ids and asset keys start with this; callers may not use it. +pub(crate) const PROBE_PREFIX: &str = "__tinyinference_box"; + +/// One Box question being emulated. +struct Target { + /// The caller's question id. + id: String, + /// The question's instructions, as text for the probe questions. + instructions: String, + /// Asset key of the image the box is drawn on. + source: String, + width: u32, + height: u32, +} + +/// Plans probe rounds for a request's Box questions and turns their answers +/// back into Box answers. pub(crate) struct BoxEmulator<'a> { request: &'a EvaluationRequest, + grid: u32, + targets: Vec, + images: BTreeMap, } +/// One Noul question about one grid cell of a target's image. pub(crate) struct Probe { + /// Index of the target this probe belongs to. pub(crate) target: usize, + /// The cell, in pixels of the target's whole image. pub(crate) cell: Rect, + /// Question id, also the key of the probe's own crop asset. pub(crate) id: String, pub(crate) question: Question, pub(crate) asset: ImageAsset, + /// Size of the crop file. pub(crate) bytes: usize, } -#[derive(Default)] +/// Answers, usage, and metadata accumulated across sub-requests. +#[derive(Debug, Default)] pub(crate) struct Responses { + model: Option, + request_id: Option, answers: BTreeMap, + usage: Usage, } impl Responses { - pub(crate) fn add(&mut self, _response: EvaluationResponse, _request_id: Option) { - todo!() + /// Record one validated sub-response. The first model and the first + /// request id seen are kept; usage is summed. + pub(crate) fn add(&mut self, response: EvaluationResponse, request_id: Option) { + self.model.get_or_insert(response.model); + if self.request_id.is_none() { + self.request_id = request_id; + } + self.answers.extend(response.answers); + self.usage = add_usage(self.usage, response.usage); } } impl<'a> BoxEmulator<'a> { - pub(crate) fn new(_request: &'a EvaluationRequest, _grid: u32) -> Result { - todo!() + /// Resolve and decode the image of every Box question in `request`. + /// + /// # Errors + /// + /// Returns [`Error::InvalidRequest`] when a caller question id or asset + /// key uses [`PROBE_PREFIX`], a Box question does not identify exactly one + /// image, or an image cannot be decoded. + pub(crate) fn new(request: &'a EvaluationRequest, grid: u32) -> Result { + if request + .questions + .keys() + .chain(request.assets.keys()) + .any(|key| key.starts_with(PROBE_PREFIX)) + { + return Err(Error::invalid_request(format!( + "question ids and asset keys starting with `{PROBE_PREFIX}` are reserved for \ + emulated Box questions" + ))); + } + let mut images = BTreeMap::new(); + let mut targets = Vec::new(); + for (id, question) in &request.questions { + let Question::Box(question) = question else { + continue; + }; + let source = source_asset(request, id, &question.instructions)?; + if !images.contains_key(&source) { + let Asset::Image(asset) = &request.assets[&source]; + images.insert(source.clone(), crop::decode(&source, asset)?); + } + let image = &images[&source]; + targets.push(Target { + id: id.clone(), + instructions: as_text(&question.instructions), + width: image.width(), + height: image.height(), + source, + }); + } + Ok(Self { + request, + grid, + targets, + images, + }) } + + /// Probes over the whole image of every target. pub(crate) fn first_round(&self) -> Result> { - todo!() + let regions: Vec> = self + .targets + .iter() + .map(|target| Some(Rect::image(target.width, target.height))) + .collect(); + self.round(0, ®ions) } - pub(crate) fn refinement_round(&self, _picks: &[Pick]) -> Result> { - todo!() + + /// Probes over each first-round box worth refining; empty when none is. + pub(crate) fn refinement_round(&self, picks: &[Pick]) -> Result> { + let regions: Vec> = self + .targets + .iter() + .zip(picks) + .map(|(target, pick)| { + geometry::refinement_region( + pick.rect, + Rect::image(target.width, target.height), + self.grid, + ) + }) + .collect(); + self.round(1, ®ions) + } + + fn round(&self, round: u32, regions: &[Option]) -> Result> { + let mut probes = Vec::new(); + for (index, (target, region)) in self.targets.iter().zip(regions).enumerate() { + let Some(region) = region else { + continue; + }; + let image = &self.images[&target.source]; + for (cell_index, cell) in geometry::grid(*region, self.grid).into_iter().enumerate() { + let cell_index = u32::try_from(cell_index).unwrap_or(u32::MAX); + let id = format!( + "{PROBE_PREFIX}{index}_round{round}_r{}c{}", + cell_index / self.grid, + cell_index % self.grid + ); + let (asset, bytes) = crop::crop(image, cell)?; + probes.push(Probe { + target: index, + cell, + question: probe_question(&id, target), + id, + asset, + bytes, + }); + } + } + Ok(probes) } + + /// Requests that ask every probe, each carrying at most `limit` bytes of + /// images. + /// + /// With `with_base`, the caller's other questions and the assets they or + /// the state reference are sent too. Probes join that request unless it + /// carries one of their source images, so a tile is never judged next to + /// the whole picture it was cut from. Other requests carry the caller's + /// state and model, the probes, and their crops only. + /// + /// # Errors + /// + /// Returns [`Error::InvalidRequest`] when one crop alone exceeds `limit` + /// or the provider's per-image limit. pub(crate) fn batches( &self, - _probes: &[Probe], - _with_base: bool, - _limit: usize, + probes: &[Probe], + with_base: bool, + limit: usize, ) -> Result> { - todo!() + let empty = || EvaluationRequest { + state: self.request.state.clone(), + model: self.request.model.clone(), + questions: BTreeMap::new(), + assets: BTreeMap::new(), + }; + let mut batches = Vec::new(); + let mut current = empty(); + let mut total = 0_usize; + if with_base { + let base = self.base()?; + if !base.questions.is_empty() { + let carries_source = probes + .iter() + .any(|probe| base.assets.contains_key(&self.targets[probe.target].source)); + if carries_source { + batches.push(base); + } else { + total = image_bytes(&base)?; + current = base; + } + } + } + for probe in probes { + if probe.bytes > limit.min(MAX_IMAGE_BYTES) { + return Err(Error::invalid_request( + "an emulated Box image crop exceeds the provider's image size limit", + )); + } + if total.saturating_add(probe.bytes) > limit && !current.questions.is_empty() { + batches.push(std::mem::replace(&mut current, empty())); + total = 0; + } + current + .questions + .insert(probe.id.clone(), probe.question.clone()); + current + .assets + .insert(probe.id.clone(), Asset::Image(probe.asset.clone())); + total = total.saturating_add(probe.bytes); + } + if !current.questions.is_empty() { + batches.push(current); + } + Ok(batches) + } + + /// The caller's request without its Box questions or the images only they + /// reference. + fn base(&self) -> Result { + let questions: BTreeMap = self + .request + .questions + .iter() + .filter(|(_, question)| !matches!(question, Question::Box(_))) + .map(|(id, question)| (id.clone(), question.clone())) + .collect(); + let readable = serde_json::to_value((&self.request.state, &questions)) + .map_err(|_| Error::invalid_request("state and questions must serialise"))?; + let assets = self + .request + .assets + .iter() + .filter(|(key, _)| value_mentions(&readable, key)) + .map(|(key, asset)| (key.clone(), asset.clone())) + .collect(); + Ok(EvaluationRequest { + state: self.request.state.clone(), + model: self.request.model.clone(), + questions, + assets, + }) } - pub(crate) fn picks(&self, _probes: &[Probe], _responses: &Responses) -> Result>> { - todo!() + + /// The box each target's probes select, or `None` for a target this round + /// did not probe. + /// + /// # Errors + /// + /// Returns [`Error::InvalidResponse`] when a probe was not answered with a + /// Noul. + pub(crate) fn picks(&self, probes: &[Probe], responses: &Responses) -> Result>> { + let mut cells: Vec<(Vec, Vec)> = vec![(Vec::new(), Vec::new()); self.targets.len()]; + for probe in probes { + let Some(Answer::Noul(answer)) = responses.answers.get(&probe.id) else { + return Err(Error::invalid_response(format!( + "emulated Box probe `{}` was not answered with a noul", + probe.id + ))); + }; + let (rects, probabilities) = &mut cells[probe.target]; + rects.push(probe.cell); + probabilities.push(answer.noul); + } + Ok(cells + .into_iter() + .map(|(rects, probabilities)| { + (!rects.is_empty()).then(|| geometry::pick(&rects, &probabilities, self.grid)) + }) + .collect()) } - pub(crate) fn response(&self, _responses: Responses, _picks: &[Pick]) -> (EvaluationResponse, Option) { - todo!() + + /// The caller-facing response: the caller's non-Box answers plus one Box + /// answer per target, with every probe answer dropped. Also returns the + /// first sub-request id. + pub(crate) fn response( + &self, + responses: Responses, + picks: &[Pick], + ) -> (EvaluationResponse, Option) { + let mut answers: BTreeMap = responses + .answers + .into_iter() + .filter(|(id, _)| self.request.questions.contains_key(id)) + .collect(); + for (target, pick) in self.targets.iter().zip(picks) { + answers.insert( + target.id.clone(), + Answer::Box(BoxAnswer { + edges: pick.rect.edges(target.width, target.height), + confidence: pick.confidence, + }), + ); + } + let response = EvaluationResponse { + model: responses.model.unwrap_or_default(), + answers, + usage: responses.usage, + }; + (response, responses.request_id) } } -pub(crate) fn refine(_first: Vec, _second: Vec>) -> Vec { - todo!() +/// First-round picks with each one replaced by its refinement when that found +/// the target; a refinement with no positive cell keeps the first-round box. +pub(crate) fn refine(first: Vec, second: Vec>) -> Vec { + first + .into_iter() + .zip(second) + .map(|(first, second)| match second { + Some(second) if second.found => second, + _ => first, + }) + .collect() +} + +/// The asset key of the image a Box question is about. +/// +/// In order: the one asset key its instructions mention (a key contained in a +/// longer mentioned key does not count); else the one asset held by a +/// top-level state field its instructions name in backticks; else the +/// request's only image. +fn source_asset(request: &EvaluationRequest, id: &str, instructions: &Value) -> Result { + let ambiguous = || { + Error::invalid_request(format!( + "box question `{id}` names more than one image; emulating Box needs exactly one" + )) + }; + let mentioned: Vec<&String> = request + .assets + .keys() + .filter(|key| value_mentions(instructions, key)) + .collect(); + let mentioned: Vec<&String> = mentioned + .iter() + .filter(|key| { + !mentioned + .iter() + .any(|other| other != *key && other.contains(key.as_str())) + }) + .copied() + .collect(); + match mentioned.as_slice() { + [key] => return Ok((*key).clone()), + [] => {} + _ => return Err(ambiguous()), + } + let mut fields = Vec::new(); + backticked(instructions, &mut fields); + let held: BTreeSet<&String> = fields + .iter() + .filter_map(|field| request.state.get(field)?.as_str()) + .filter_map(|value| request.assets.get_key_value(value).map(|(key, _)| key)) + .collect(); + match held.len() { + 1 => return Ok(held.into_iter().next().cloned().unwrap_or_default()), + 0 => {} + _ => return Err(ambiguous()), + } + if let [(key, _)] = request.assets.iter().collect::>().as_slice() { + return Ok((*key).clone()); + } + Err(Error::invalid_request(format!( + "box question `{id}` must identify one image to emulate Box: mention its asset key, or \ + name a state field holding the key in backticks" + ))) } -fn source_asset(_request: &EvaluationRequest, _id: &str, _instructions: &serde_json::Value) -> Result { - todo!() +/// Every span between backticks in the string values of `value`. +fn backticked(value: &Value, spans: &mut Vec) { + match value { + Value::String(text) => spans.extend( + text.split('`') + .skip(1) + .step_by(2) + .filter(|span| !span.is_empty()) + .map(str::to_owned), + ), + Value::Array(items) => items.iter().for_each(|item| backticked(item, spans)), + Value::Object(map) => map.values().for_each(|item| backticked(item, spans)), + _ => {} + } +} + +fn as_text(value: &Value) -> String { + match value { + Value::String(text) => text.clone(), + other => other.to_string(), + } +} + +fn probe_question(id: &str, target: &Target) -> Question { + Question::Noul(Noul { + instructions: Value::String(format!( + "`{id}` is a tile cut from the image `{}`. Judge only the tile `{id}`. Does it show \ + all or part of the following? {}", + target.source, target.instructions + )), + criteria: Some(NoulCriteria { + r#true: json!("The tile shows all or part of it."), + r#false: json!("The tile shows none of it."), + }), + }) +} + +fn image_bytes(request: &EvaluationRequest) -> Result { + request + .assets + .values() + .try_fold(0_usize, |total, Asset::Image(image)| { + Ok(total.saturating_add(image.checked_len()?)) + }) +} + +fn add_usage(left: Usage, right: Usage) -> Usage { + fn sum(left: Option, right: Option) -> Option { + match (left, right) { + (Some(left), Some(right)) => Some(left.saturating(right)), + (value, None) | (None, value) => value, + } + } + Usage { + input_tokens: sum(left.input_tokens, right.input_tokens), + output_tokens: sum(left.output_tokens, right.output_tokens), + cached_input_tokens: sum(left.cached_input_tokens, right.cached_input_tokens), + cost_usd_nanos: sum(left.cost_usd_nanos, right.cost_usd_nanos), + } +} + +trait SaturatingAdd { + fn saturating(self, other: Self) -> Self; +} + +impl SaturatingAdd for u64 { + fn saturating(self, other: Self) -> Self { + self.saturating_add(other) + } +} + +impl SaturatingAdd for i64 { + fn saturating(self, other: Self) -> Self { + self.saturating_add(other) + } } #[cfg(test)] From 8f5e41d37655ad0c2495647b04301472e24a520e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 04:52:21 +0000 Subject: [PATCH 06/19] test(emulation): import ImageFormat in emulation tests Add the ImageFormat import to the emulation test module so the tests can reference the image format type. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyinference-decisions/src/emulation/mod_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinyinference-decisions/src/emulation/mod_tests.rs b/crates/tinyinference-decisions/src/emulation/mod_tests.rs index 2ef5a901..327cda7c 100644 --- a/crates/tinyinference-decisions/src/emulation/mod_tests.rs +++ b/crates/tinyinference-decisions/src/emulation/mod_tests.rs @@ -7,7 +7,7 @@ use serde_json::json; use super::crop::tests::{encode, picture}; use super::*; -use crate::{BoxQuestion, Noul, NoulAnswer}; +use crate::{BoxQuestion, ImageFormat, Noul, NoulAnswer}; fn rect(left: u32, top: u32, right: u32, bottom: u32) -> Rect { Rect { From bf7c88e356cd0c416828c25bdd6eead120487942 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 04:53:32 +0000 Subject: [PATCH 07/19] feat(client): emulate box questions for providers without native support Providers that cannot answer box questions natively now have each box question split into a grid of crop questions whose answers are combined back into a box answer, so callers keep sending box questions and receiving box answers regardless of provider. The grid size is validated at config time, and the retry loop was factored out so emulated and native paths share the same attempt accounting. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/client/emulated.rs | 19 ++ .../src/client/emulated_tests.rs | 284 ++++++++++++++++++ .../tinyinference-decisions/src/client/mod.rs | 68 +++-- .../src/client/types.rs | 75 +++++ .../src/emulation/mod.rs | 4 +- 5 files changed, 431 insertions(+), 19 deletions(-) create mode 100644 crates/tinyinference-decisions/src/client/emulated.rs create mode 100644 crates/tinyinference-decisions/src/client/emulated_tests.rs diff --git a/crates/tinyinference-decisions/src/client/emulated.rs b/crates/tinyinference-decisions/src/client/emulated.rs new file mode 100644 index 00000000..cfe3e71c --- /dev/null +++ b/crates/tinyinference-decisions/src/client/emulated.rs @@ -0,0 +1,19 @@ +//! Evaluating a request whose Box questions the provider cannot answer. + +use super::{BoxEmulation, Client}; +use crate::{EvaluationRequest, EvaluationResponse, Result}; + +impl Client { + pub(super) async fn evaluate_emulated( + &self, + _request: &EvaluationRequest, + _settings: BoxEmulation, + _attempts: &mut u32, + ) -> Result<(EvaluationResponse, Option)> { + todo!() + } +} + +#[cfg(test)] +#[path = "emulated_tests.rs"] +mod tests; diff --git a/crates/tinyinference-decisions/src/client/emulated_tests.rs b/crates/tinyinference-decisions/src/client/emulated_tests.rs new file mode 100644 index 00000000..152853e4 --- /dev/null +++ b/crates/tinyinference-decisions/src/client/emulated_tests.rs @@ -0,0 +1,284 @@ +//! Box emulation through the client: request rewriting, synthesis, metering. + +use std::sync::{Arc, Mutex}; + +use base64::Engine as _; +use reqwest::StatusCode; +use serde_json::{Value, json}; + +use super::*; +use crate::emulation::crop::tests::{encode, is_red, picture}; +use crate::emulation::geometry::Rect; +use crate::{Answer, BoxQuestion, ImageAsset, Noul, Question}; + +/// Answers every question from the request itself: a probe is positive when +/// its crop shows red, a Box is answered with a fixed box, a Noul with 0.7. +struct EchoTransport { + model: String, + bodies: Mutex>, +} + +#[async_trait::async_trait] +impl HttpTransport for EchoTransport { + async fn send( + &self, + request: reqwest::Request, + _response_limit: usize, + ) -> std::result::Result { + let body: Value = + serde_json::from_slice(request.body().and_then(reqwest::Body::as_bytes).unwrap()) + .unwrap(); + let opted_in = request.headers().get(IMAGE_OPT_IN_HEADER).is_some(); + let mut answers = serde_json::Map::new(); + for (id, question) in body["questions"].as_object().unwrap() { + let answer = match question["type"].as_str().unwrap() { + "box" => json!({ + "type": "box", + "box": {"top": 0.1, "bottom": 0.2, "left": 0.3, "right": 0.4}, + "confidence": 0.8 + }), + _ if id.starts_with(crate::emulation::PROBE_PREFIX) => { + let data = body["assets"][id]["data_base64"].as_str().unwrap(); + let bytes = base64::engine::general_purpose::STANDARD + .decode(data) + .unwrap(); + let crop = image::load_from_memory(&bytes).unwrap().to_rgba8(); + let noul = if crop.pixels().any(is_red) { 0.95 } else { 0.05 }; + json!({"type": "noul", "noul": noul}) + } + _ => json!({"type": "noul", "noul": 0.7}), + }; + answers.insert(id.clone(), answer); + } + let mut bodies = self.bodies.lock().unwrap(); + bodies.push((body, opted_in, request.url().path().to_owned())); + let response = json!({ + "model": self.model, + "answers": answers, + "usage": {"input_tokens": 10, "output_tokens": 1, "cost": 0.000_001}, + "id": "gen-dec-1", + "provider": "TypeSafe" + }); + Ok(TransportResponse { + status: StatusCode::OK, + request_id: Some(format!("req-{}", bodies.len())), + retry_after: None, + body: response.to_string().into_bytes(), + }) + } +} + +fn client(config: ClientConfig, model: &str) -> (Client, Arc) { + let transport = Arc::new(EchoTransport { + model: model.to_owned(), + bodies: Mutex::new(Vec::new()), + }); + let client = Client::with_transport(config, Arc::clone(&transport) as _).unwrap(); + (client, transport) +} + +fn surplus() -> (Client, Arc) { + client(ClientConfig::surplus("key"), "typesafe/jev-1.14-20261014") +} + +/// An 800 by 800 page with a red square at x 420..470, y 220..260. +fn page_request() -> EvaluationRequest { + let red = Rect { + left: 420, + top: 220, + right: 470, + bottom: 260, + }; + let png = encode(&picture(800, 800, red), image::ImageFormat::Png); + EvaluationRequest::new( + "jev-1.14", + json!({"page": "shot"}), + [ + ( + "square".to_owned(), + Question::Box(BoxQuestion { + instructions: json!("the red square in `page`"), + }), + ), + ( + "present".to_owned(), + Question::Noul(Noul { + instructions: json!("Is there a red square in `page`?"), + criteria: None, + }), + ), + ] + .into(), + ) + .with_image("shot", ImageAsset::png(&png)) +} + +fn sent_box(transport: &EchoTransport) -> bool { + transport.bodies.lock().unwrap().iter().any(|(body, _, _)| { + body["questions"] + .as_object() + .unwrap() + .values() + .any(|question| question["type"] == "box") + }) +} + +#[tokio::test] +async fn surplus_answers_box_questions_from_refined_crop_probes() { + let (client, transport) = surplus(); + let result = client.evaluate(&page_request()).await.unwrap(); + + assert!(!sent_box(&transport)); + let bodies = transport.bodies.lock().unwrap(); + // The page and its Noul, the first-round crops, the refinement crops. + assert_eq!(bodies.len(), 3); + for (_, opted_in, path) in bodies.iter() { + assert!(opted_in); + assert_eq!(path, "/v1/decisions"); + } + assert!(bodies[0].0["assets"].get("shot").is_some()); + assert_eq!(bodies[0].0["questions"].as_object().unwrap().len(), 1); + assert_eq!(bodies[1].0["questions"].as_object().unwrap().len(), 16); + assert_eq!(bodies[2].0["questions"].as_object().unwrap().len(), 16); + drop(bodies); + + let response = &result.response; + assert_eq!(response.model, "typesafe/jev-1.14-20261014"); + assert_eq!( + response.answers.keys().collect::>(), + ["present", "square"] + ); + let Answer::Box(answer) = response.answers["square"] else { + panic!("square must be answered with a Box"); + }; + assert_eq!(answer.confidence, 0.95); + assert_eq!( + ( + answer.edges.left, + answer.edges.right, + answer.edges.top, + answer.edges.bottom + ), + (0.5, 0.625, 0.25, 0.375) + ); + assert_eq!(response.usage.input_tokens, Some(30)); + assert_eq!(response.usage.cost_usd_nanos, Some(3_000)); + assert_eq!(result.attempts, 3); + assert_eq!(result.request_id.as_deref(), Some("req-1")); +} + +#[tokio::test] +async fn without_refinement_the_box_is_the_first_round_cell() { + let config = ClientConfig::surplus("key").with_box_support(BoxSupport::Emulated( + BoxEmulation { + grid: 4, + refine: false, + }, + )); + let (client, transport) = client(config, "typesafe/jev-1.14"); + let result = client.evaluate(&page_request()).await.unwrap(); + assert_eq!(transport.bodies.lock().unwrap().len(), 2); + let Answer::Box(answer) = result.response.answers["square"] else { + panic!("square must be answered with a Box"); + }; + assert_eq!( + ( + answer.edges.left, + answer.edges.right, + answer.edges.top, + answer.edges.bottom + ), + (0.5, 0.75, 0.25, 0.5) + ); +} + +#[tokio::test] +async fn native_providers_send_box_questions_unchanged() { + let (client, transport) = client(ClientConfig::new("key"), "jev-1.14"); + let result = client.evaluate(&page_request()).await.unwrap(); + assert!(sent_box(&transport)); + assert_eq!(transport.bodies.lock().unwrap().len(), 1); + assert!(matches!(result.response.answers["square"], Answer::Box(_))); +} + +#[tokio::test] +async fn box_support_can_be_overridden_either_way() { + let native = ClientConfig::surplus("key").with_box_support(BoxSupport::Native); + assert_eq!(native.box_support(), BoxSupport::Native); + let (client, transport) = client(native, "typesafe/jev-1.14"); + client.evaluate(&page_request()).await.unwrap(); + assert!(sent_box(&transport)); + + let emulated = + ClientConfig::new("key").with_box_support(BoxSupport::Emulated(BoxEmulation::default())); + let (client, transport) = client(emulated, "jev-1.14"); + client.evaluate(&page_request()).await.unwrap(); + assert!(!sent_box(&transport)); +} + +#[test] +fn box_support_defaults_follow_the_provider() { + assert_eq!( + ClientConfig::surplus("key").box_support(), + BoxSupport::Emulated(BoxEmulation { + grid: 4, + refine: true + }) + ); + assert_eq!(ClientConfig::new("key").box_support(), BoxSupport::Native); + let mut retargeted = ClientConfig::new("key"); + retargeted.provider = Provider::Surplus; + assert!(matches!( + retargeted.box_support(), + BoxSupport::Emulated(_) + )); +} + +#[test] +fn the_emulation_grid_is_bounded() { + for grid in [0, 1, 9] { + let config = ClientConfig::surplus("key").with_box_support(BoxSupport::Emulated( + BoxEmulation { + grid, + refine: true, + }, + )); + assert!( + matches!(Client::new(config), Err(Error::InvalidConfig { .. })), + "grid {grid}" + ); + } +} + +#[tokio::test] +async fn a_box_question_without_an_image_fails_before_any_request() { + let (client, transport) = surplus(); + let request = EvaluationRequest::new( + "jev-1.14", + json!("no image here"), + [( + "square".to_owned(), + Question::Box(BoxQuestion { + instructions: json!("the red square"), + }), + )] + .into(), + ); + let Err(Error::EvaluationFailure(failure)) = client.evaluate(&request).await else { + panic!("expected an evaluation failure"); + }; + assert!(matches!(*failure.error, Error::InvalidRequest { .. })); + assert_eq!(failure.attempts, 0); + assert!(transport.bodies.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn a_sub_response_from_another_model_fails_with_its_attempts() { + let (client, transport) = client(ClientConfig::surplus("key"), "typesafe/jev-1.13"); + let Err(Error::EvaluationFailure(failure)) = client.evaluate(&page_request()).await else { + panic!("expected an evaluation failure"); + }; + assert!(matches!(*failure.error, Error::InvalidResponse { .. })); + assert_eq!(failure.attempts, 1); + assert_eq!(transport.bodies.lock().unwrap().len(), 1); +} diff --git a/crates/tinyinference-decisions/src/client/mod.rs b/crates/tinyinference-decisions/src/client/mod.rs index faf08de2..7d8dffc9 100644 --- a/crates/tinyinference-decisions/src/client/mod.rs +++ b/crates/tinyinference-decisions/src/client/mod.rs @@ -4,15 +4,19 @@ #[path = "mod_tests.rs"] mod test; +mod emulated; mod types; -pub use types::{Client, ClientConfig, EvaluationFailure, EvaluationResult, Provider, RetryPolicy}; +pub use types::{ + BoxEmulation, BoxSupport, Client, ClientConfig, EvaluationFailure, EvaluationResult, Provider, + RetryPolicy, +}; use std::time::Duration; use reqwest::{StatusCode, header::RETRY_AFTER}; -use crate::{Error, EvaluationRequest, EvaluationResponse, Result}; +use crate::{Error, EvaluationRequest, EvaluationResponse, Question, Result}; const MAX_RETRIES: u32 = 100; const MAX_RESPONSE_BYTES: usize = 16 * 1024 * 1024; @@ -92,27 +96,50 @@ impl Client { .validate() .map_err(|error| evaluation_failure(error, 0, started.elapsed()))?; let mut attempts = 0_u32; + let asks_box = request + .questions + .values() + .any(|question| matches!(question, Question::Box(_))); + let outcome = match self.config.box_support() { + BoxSupport::Emulated(settings) if asks_box => { + self.evaluate_emulated(request, settings, &mut attempts) + .await + } + _ => self.send_validated(request, &mut attempts).await, + }; + match outcome { + Ok((response, request_id)) => Ok(EvaluationResult { + response, + request_id, + attempts, + latency: started.elapsed(), + }), + Err(error) => Err(evaluation_failure(error, attempts, started.elapsed())), + } + } + + /// Send one already validated request with the retry policy and validate + /// its response, adding every HTTP attempt to `attempts`. + async fn send_validated( + &self, + request: &EvaluationRequest, + attempts: &mut u32, + ) -> Result<(EvaluationResponse, Option)> { + let mut tries = 0_u32; loop { - attempts = attempts.saturating_add(1); + tries = tries.saturating_add(1); + *attempts = attempts.saturating_add(1); match self.send_once(request).await { Ok((response, request_id)) => { - self.validate_response(&response, request) - .map_err(|error| evaluation_failure(error, attempts, started.elapsed()))?; - return Ok(EvaluationResult { - response, - request_id, - attempts, - latency: started.elapsed(), - }); - } - Err(Failure::Terminal(error)) => { - return Err(evaluation_failure(error, attempts, started.elapsed())); + self.validate_response(&response, request)?; + return Ok((response, request_id)); } + Err(Failure::Terminal(error)) => return Err(error), Err(Failure::Retryable { error, retry_after }) => { - if attempts > self.config.retry.max_retries { - return Err(evaluation_failure(error, attempts, started.elapsed())); + if tries > self.config.retry.max_retries { + return Err(error); } - let delay = retry_after.unwrap_or_else(|| self.config.retry.delay(attempts)); + let delay = retry_after.unwrap_or_else(|| self.config.retry.delay(tries)); tokio::time::sleep(delay.min(self.config.retry.max_backoff)).await; } } @@ -302,6 +329,13 @@ impl ClientConfig { reason: "retry backoffs must be greater than zero".to_owned(), }); } + if let BoxSupport::Emulated(settings) = self.box_support() + && !(2..=8).contains(&settings.grid) + { + return Err(Error::InvalidConfig { + reason: "box emulation grid must be from 2 to 8 cells per side".to_owned(), + }); + } if self.retry.max_retries > MAX_RETRIES { return Err(Error::InvalidConfig { reason: format!("max retries must not exceed {MAX_RETRIES}"), diff --git a/crates/tinyinference-decisions/src/client/types.rs b/crates/tinyinference-decisions/src/client/types.rs index e4984c1e..25bdf2f7 100644 --- a/crates/tinyinference-decisions/src/client/types.rs +++ b/crates/tinyinference-decisions/src/client/types.rs @@ -48,6 +48,56 @@ impl Provider { } } +/// How the client answers [`Question::Box`](crate::Question::Box). +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum BoxSupport { + /// Send Box questions to the provider as they are. + Native, + /// Replace each Box question with Noul questions about crops of its image + /// and build the Box answer from theirs. The caller still sends Box + /// questions and receives [`Answer::Box`](crate::Answer::Box) answers. + Emulated(BoxEmulation), +} + +impl BoxSupport { + /// The default for `provider`: [`Self::Native`] when it + /// [answers Box](Provider::answers_box), else emulation with + /// [`BoxEmulation::default`]. + #[must_use] + pub fn for_provider(provider: Provider) -> Self { + if provider.answers_box() { + Self::Native + } else { + Self::Emulated(BoxEmulation::default()) + } + } +} + +/// Settings for emulating Box questions with Noul questions over image crops. +/// +/// Each Box question's image is cut into a `grid` by `grid` set of cells, and +/// every cell is asked about as its own image. With `refine`, the box found is +/// cut into the same grid again in a second, sequential round of requests. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct BoxEmulation { + /// Cells per side, from 2 to 8; each round asks `grid * grid` Noul + /// questions per Box question. + pub grid: u32, + /// Whether to run the second, refining round. + pub refine: bool, +} + +impl Default for BoxEmulation { + /// A 4 by 4 grid with refinement: up to 32 Noul questions per Box + /// question, and a box as fine as one sixteenth of a cell per side. + fn default() -> Self { + Self { + grid: 4, + refine: true, + } + } +} + /// Async Jev-compatible System One client. #[derive(Clone)] pub struct Client { @@ -82,6 +132,8 @@ pub struct ClientConfig { pub retry: RetryPolicy, /// Sanitized product attribution for the exact `TinyHumans` proxy endpoint. pub(super) sdk_name: Option, + /// Caller override of the provider's Box support. + pub(super) box_support: Option, } impl ClientConfig { @@ -97,6 +149,7 @@ impl ClientConfig { timeout: Duration::from_secs(30), retry: RetryPolicy::default(), sdk_name: None, + box_support: None, } } @@ -112,6 +165,7 @@ impl ClientConfig { timeout: Duration::from_secs(30), retry: RetryPolicy::default(), sdk_name: None, + box_support: None, } } @@ -127,6 +181,7 @@ impl ClientConfig { timeout: Duration::from_secs(30), retry: RetryPolicy::default(), sdk_name: None, + box_support: None, } } @@ -147,6 +202,7 @@ impl ClientConfig { timeout: Duration::from_secs(30), retry: RetryPolicy::default(), sdk_name: None, + box_support: None, } } @@ -172,6 +228,7 @@ impl ClientConfig { timeout: Duration::from_secs(30), retry: RetryPolicy::default(), sdk_name: None, + box_support: None, } } @@ -190,6 +247,7 @@ impl ClientConfig { timeout: Duration::from_secs(30), retry: RetryPolicy::default(), sdk_name: None, + box_support: None, } } @@ -226,6 +284,22 @@ impl ClientConfig { self } + /// Answer Box questions as `support` says instead of as the provider's + /// default ([`BoxSupport::for_provider`]). + #[must_use] + pub fn with_box_support(mut self, support: BoxSupport) -> Self { + self.box_support = Some(support); + self + } + + /// How this configuration answers Box questions: the override from + /// [`Self::with_box_support`], else the provider's default. + #[must_use] + pub fn box_support(&self) -> BoxSupport { + self.box_support + .unwrap_or_else(|| BoxSupport::for_provider(self.provider)) + } + /// Returns the exact endpoint override, when configured. #[must_use] pub fn endpoint_url(&self) -> Option<&str> { @@ -247,6 +321,7 @@ impl fmt::Debug for ClientConfig { .field("timeout", &self.timeout) .field("retry", &self.retry) .field("sdk_name", &self.sdk_name) + .field("box_support", &self.box_support()) .finish() } } diff --git a/crates/tinyinference-decisions/src/emulation/mod.rs b/crates/tinyinference-decisions/src/emulation/mod.rs index 02029ba0..449b6da3 100644 --- a/crates/tinyinference-decisions/src/emulation/mod.rs +++ b/crates/tinyinference-decisions/src/emulation/mod.rs @@ -8,8 +8,8 @@ //! refinement round grids that box again. Probes never reach the caller: the //! response carries exactly the caller's question ids. -mod crop; -mod geometry; +pub(crate) mod crop; +pub(crate) mod geometry; use std::collections::{BTreeMap, BTreeSet}; From 84942769c259e1162985056e5a9f7ea8dbf43026 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 04:53:40 +0000 Subject: [PATCH 08/19] test(client): add imports for emulated client test helpers Extend the emulated client test module's imports to bring in the client config, transport, and error types needed by the tests. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyinference-decisions/src/client/emulated_tests.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/tinyinference-decisions/src/client/emulated_tests.rs b/crates/tinyinference-decisions/src/client/emulated_tests.rs index 152853e4..e5202115 100644 --- a/crates/tinyinference-decisions/src/client/emulated_tests.rs +++ b/crates/tinyinference-decisions/src/client/emulated_tests.rs @@ -6,10 +6,14 @@ use base64::Engine as _; use reqwest::StatusCode; use serde_json::{Value, json}; +use super::super::{ + BoxSupport, ClientConfig, Failure, HttpTransport, IMAGE_OPT_IN_HEADER, Provider, + TransportResponse, +}; use super::*; use crate::emulation::crop::tests::{encode, is_red, picture}; use crate::emulation::geometry::Rect; -use crate::{Answer, BoxQuestion, ImageAsset, Noul, Question}; +use crate::{Answer, BoxQuestion, Error, ImageAsset, Noul, Question}; /// Answers every question from the request itself: a probe is positive when /// its crop shows red, a Box is answered with a fixed box, a Noul with 0.7. From 24f725f0260cae67056bf04b8de7ba1646782fc1 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 04:53:49 +0000 Subject: [PATCH 09/19] test(client): rename test helper client to echo_client Renamed the local test helper that builds a client backed by the echo transport so its name reflects what it returns. This avoids confusion with the client type and makes the emulated tests easier to read. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/client/emulated_tests.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/tinyinference-decisions/src/client/emulated_tests.rs b/crates/tinyinference-decisions/src/client/emulated_tests.rs index e5202115..0d3a0623 100644 --- a/crates/tinyinference-decisions/src/client/emulated_tests.rs +++ b/crates/tinyinference-decisions/src/client/emulated_tests.rs @@ -72,7 +72,7 @@ impl HttpTransport for EchoTransport { } } -fn client(config: ClientConfig, model: &str) -> (Client, Arc) { +fn echo_client(config: ClientConfig, model: &str) -> (Client, Arc) { let transport = Arc::new(EchoTransport { model: model.to_owned(), bodies: Mutex::new(Vec::new()), @@ -82,7 +82,7 @@ fn client(config: ClientConfig, model: &str) -> (Client, Arc) { } fn surplus() -> (Client, Arc) { - client(ClientConfig::surplus("key"), "typesafe/jev-1.14-20261014") + echo_client(ClientConfig::surplus("key"), "typesafe/jev-1.14-20261014") } /// An 800 by 800 page with a red square at x 420..470, y 220..260. @@ -179,7 +179,7 @@ async fn without_refinement_the_box_is_the_first_round_cell() { refine: false, }, )); - let (client, transport) = client(config, "typesafe/jev-1.14"); + let (client, transport) = echo_client(config, "typesafe/jev-1.14"); let result = client.evaluate(&page_request()).await.unwrap(); assert_eq!(transport.bodies.lock().unwrap().len(), 2); let Answer::Box(answer) = result.response.answers["square"] else { @@ -198,7 +198,7 @@ async fn without_refinement_the_box_is_the_first_round_cell() { #[tokio::test] async fn native_providers_send_box_questions_unchanged() { - let (client, transport) = client(ClientConfig::new("key"), "jev-1.14"); + let (client, transport) = echo_client(ClientConfig::new("key"), "jev-1.14"); let result = client.evaluate(&page_request()).await.unwrap(); assert!(sent_box(&transport)); assert_eq!(transport.bodies.lock().unwrap().len(), 1); @@ -209,13 +209,13 @@ async fn native_providers_send_box_questions_unchanged() { async fn box_support_can_be_overridden_either_way() { let native = ClientConfig::surplus("key").with_box_support(BoxSupport::Native); assert_eq!(native.box_support(), BoxSupport::Native); - let (client, transport) = client(native, "typesafe/jev-1.14"); + let (client, transport) = echo_client(native, "typesafe/jev-1.14"); client.evaluate(&page_request()).await.unwrap(); assert!(sent_box(&transport)); let emulated = ClientConfig::new("key").with_box_support(BoxSupport::Emulated(BoxEmulation::default())); - let (client, transport) = client(emulated, "jev-1.14"); + let (client, transport) = echo_client(emulated, "jev-1.14"); client.evaluate(&page_request()).await.unwrap(); assert!(!sent_box(&transport)); } @@ -278,7 +278,7 @@ async fn a_box_question_without_an_image_fails_before_any_request() { #[tokio::test] async fn a_sub_response_from_another_model_fails_with_its_attempts() { - let (client, transport) = client(ClientConfig::surplus("key"), "typesafe/jev-1.13"); + let (client, transport) = echo_client(ClientConfig::surplus("key"), "typesafe/jev-1.13"); let Err(Error::EvaluationFailure(failure)) = client.evaluate(&page_request()).await else { panic!("expected an evaluation failure"); }; From ffaf76b4c0378fe91549845318a7b9c06d93da1d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 04:54:05 +0000 Subject: [PATCH 10/19] feat(client): implement emulated box evaluation Wire the emulated evaluation path to the box emulator so requests whose box questions the provider cannot answer are resolved by sending probe batches through the normal retry and validation flow, with an optional refinement round. Also re-export BoxEmulation and BoxSupport from the crate root. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/client/emulated.rs | 54 +++++++++++++++++-- crates/tinyinference-decisions/src/lib.rs | 3 +- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/crates/tinyinference-decisions/src/client/emulated.rs b/crates/tinyinference-decisions/src/client/emulated.rs index cfe3e71c..7cb18ab4 100644 --- a/crates/tinyinference-decisions/src/client/emulated.rs +++ b/crates/tinyinference-decisions/src/client/emulated.rs @@ -1,16 +1,62 @@ //! Evaluating a request whose Box questions the provider cannot answer. +//! +//! [`crate::emulation`] plans the probe requests and builds the answers; this +//! module only sends them, one after another, through the same retry and +//! response validation as any other request. use super::{BoxEmulation, Client}; +use crate::emulation::{BoxEmulator, Probe, Responses, refine}; +use crate::request::image::MAX_TOTAL_IMAGE_BYTES; use crate::{EvaluationRequest, EvaluationResponse, Result}; impl Client { + /// Answer `request`, emulating its Box questions as `settings` says. + /// + /// Every HTTP attempt of every sub-request is added to `attempts`. The + /// returned request id is the first one a sub-request reported. pub(super) async fn evaluate_emulated( &self, - _request: &EvaluationRequest, - _settings: BoxEmulation, - _attempts: &mut u32, + request: &EvaluationRequest, + settings: BoxEmulation, + attempts: &mut u32, ) -> Result<(EvaluationResponse, Option)> { - todo!() + let emulator = BoxEmulator::new(request, settings.grid)?; + let mut responses = Responses::default(); + let first = emulator.first_round()?; + self.ask(&emulator, &first, true, &mut responses, attempts) + .await?; + let mut picks: Vec<_> = emulator + .picks(&first, &responses)? + .into_iter() + .flatten() + .collect(); + if settings.refine { + let second = emulator.refinement_round(&picks)?; + if !second.is_empty() { + self.ask(&emulator, &second, false, &mut responses, attempts) + .await?; + picks = refine(picks, emulator.picks(&second, &responses)?); + } + } + let (response, request_id) = emulator.response(responses, &picks); + self.validate_response(&response, request)?; + Ok((response, request_id)) + } + + async fn ask( + &self, + emulator: &BoxEmulator<'_>, + probes: &[Probe], + with_base: bool, + responses: &mut Responses, + attempts: &mut u32, + ) -> Result<()> { + for batch in emulator.batches(probes, with_base, MAX_TOTAL_IMAGE_BYTES)? { + batch.validate()?; + let (response, request_id) = self.send_validated(&batch, attempts).await?; + responses.add(response, request_id); + } + Ok(()) } } diff --git a/crates/tinyinference-decisions/src/lib.rs b/crates/tinyinference-decisions/src/lib.rs index a83039aa..dfa15ca7 100644 --- a/crates/tinyinference-decisions/src/lib.rs +++ b/crates/tinyinference-decisions/src/lib.rs @@ -52,7 +52,8 @@ mod response; pub mod sage; pub use client::{ - Client, ClientConfig, EvaluationFailure, EvaluationResult, Provider, RetryPolicy, + BoxEmulation, BoxSupport, Client, ClientConfig, EvaluationFailure, EvaluationResult, Provider, + RetryPolicy, }; pub use error::{Error, Result}; pub use request::{ From 32ab6a66994afe4e958a17586c6f57a8cc3cc712 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 04:54:14 +0000 Subject: [PATCH 11/19] style(tinyinference-decisions): reformat emulation code and tests Reformatted the box emulation module and its tests to satisfy rustfmt line-width rules, splitting long signatures, expressions, and assertions across lines. No behaviour changed. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/client/emulated_tests.rs | 26 ++++++++----------- .../src/emulation/mod.rs | 9 +++++-- .../src/emulation/mod_tests.rs | 16 +++++++++--- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/crates/tinyinference-decisions/src/client/emulated_tests.rs b/crates/tinyinference-decisions/src/client/emulated_tests.rs index 0d3a0623..4f7c9159 100644 --- a/crates/tinyinference-decisions/src/client/emulated_tests.rs +++ b/crates/tinyinference-decisions/src/client/emulated_tests.rs @@ -47,7 +47,11 @@ impl HttpTransport for EchoTransport { .decode(data) .unwrap(); let crop = image::load_from_memory(&bytes).unwrap().to_rgba8(); - let noul = if crop.pixels().any(is_red) { 0.95 } else { 0.05 }; + let noul = if crop.pixels().any(is_red) { + 0.95 + } else { + 0.05 + }; json!({"type": "noul", "noul": noul}) } _ => json!({"type": "noul", "noul": 0.7}), @@ -173,12 +177,11 @@ async fn surplus_answers_box_questions_from_refined_crop_probes() { #[tokio::test] async fn without_refinement_the_box_is_the_first_round_cell() { - let config = ClientConfig::surplus("key").with_box_support(BoxSupport::Emulated( - BoxEmulation { + let config = + ClientConfig::surplus("key").with_box_support(BoxSupport::Emulated(BoxEmulation { grid: 4, refine: false, - }, - )); + })); let (client, transport) = echo_client(config, "typesafe/jev-1.14"); let result = client.evaluate(&page_request()).await.unwrap(); assert_eq!(transport.bodies.lock().unwrap().len(), 2); @@ -232,21 +235,14 @@ fn box_support_defaults_follow_the_provider() { assert_eq!(ClientConfig::new("key").box_support(), BoxSupport::Native); let mut retargeted = ClientConfig::new("key"); retargeted.provider = Provider::Surplus; - assert!(matches!( - retargeted.box_support(), - BoxSupport::Emulated(_) - )); + assert!(matches!(retargeted.box_support(), BoxSupport::Emulated(_))); } #[test] fn the_emulation_grid_is_bounded() { for grid in [0, 1, 9] { - let config = ClientConfig::surplus("key").with_box_support(BoxSupport::Emulated( - BoxEmulation { - grid, - refine: true, - }, - )); + let config = ClientConfig::surplus("key") + .with_box_support(BoxSupport::Emulated(BoxEmulation { grid, refine: true })); assert!( matches!(Client::new(config), Err(Error::InvalidConfig { .. })), "grid {grid}" diff --git a/crates/tinyinference-decisions/src/emulation/mod.rs b/crates/tinyinference-decisions/src/emulation/mod.rs index 449b6da3..6c9459bc 100644 --- a/crates/tinyinference-decisions/src/emulation/mod.rs +++ b/crates/tinyinference-decisions/src/emulation/mod.rs @@ -289,8 +289,13 @@ impl<'a> BoxEmulator<'a> { /// /// Returns [`Error::InvalidResponse`] when a probe was not answered with a /// Noul. - pub(crate) fn picks(&self, probes: &[Probe], responses: &Responses) -> Result>> { - let mut cells: Vec<(Vec, Vec)> = vec![(Vec::new(), Vec::new()); self.targets.len()]; + pub(crate) fn picks( + &self, + probes: &[Probe], + responses: &Responses, + ) -> Result>> { + let mut cells: Vec<(Vec, Vec)> = + vec![(Vec::new(), Vec::new()); self.targets.len()]; for probe in probes { let Some(Answer::Noul(answer)) = responses.answers.get(&probe.id) else { return Err(Error::invalid_response(format!( diff --git a/crates/tinyinference-decisions/src/emulation/mod_tests.rs b/crates/tinyinference-decisions/src/emulation/mod_tests.rs index 327cda7c..afc04d34 100644 --- a/crates/tinyinference-decisions/src/emulation/mod_tests.rs +++ b/crates/tinyinference-decisions/src/emulation/mod_tests.rs @@ -81,7 +81,10 @@ fn a_key_contained_in_a_longer_named_key_is_not_ambiguous() { fn the_source_is_the_asset_held_by_a_backticked_state_field() { let request = request( json!({"page": "shot-a", "logo": "shot-b"}), - vec![("button", boxed(json!({"find": "the Verify button in `page`"})))], + vec![( + "button", + boxed(json!({"find": "the Verify button in `page`"})), + )], ) .with_image("shot-a", png(300, 300)) .with_image("shot-b", png(300, 300)); @@ -108,7 +111,11 @@ fn an_unnamed_or_ambiguous_source_is_a_request_error() { .with_image("shot-a", png(300, 300)) .with_image("shot-b", png(300, 300)) }; - for instructions in ["the Verify button", "the button in shot-a or shot-b", "`a` or `b`"] { + for instructions in [ + "the Verify button", + "the button in shot-a or shot-b", + "`a` or `b`", + ] { let error = source_of(&two(instructions)).unwrap_err(); assert!( matches!(&error, Error::InvalidRequest { reason } if reason.contains("`button`")), @@ -163,7 +170,10 @@ fn the_first_round_asks_one_noul_per_grid_cell_with_its_own_crop() { panic!("probes must be Noul questions"); }; let text = question.instructions.as_str().unwrap(); - assert!(text.contains("`__tinyinference_box0_round0_r1c1`"), "{text}"); + assert!( + text.contains("`__tinyinference_box0_round0_r1c1`"), + "{text}" + ); assert!(text.contains("the Verify button in `page`"), "{text}"); assert_eq!(probes[5].asset.format, ImageFormat::Png); assert_eq!(probes[5].bytes, probes[5].asset.checked_len().unwrap()); From ff0b05b59ebd8489353f32fb99341115431ade43 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 04:54:26 +0000 Subject: [PATCH 12/19] test(public-api): cover surplus box emulation defaults and overrides Adds a public API test asserting that a surplus client config emulates box questions by default and that callers can switch to native support, plus a clarifying comment on the grid layout in the geometry tests. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/emulation/geometry_tests.rs | 2 ++ .../tinyinference-decisions/tests/public_api.rs | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/crates/tinyinference-decisions/src/emulation/geometry_tests.rs b/crates/tinyinference-decisions/src/emulation/geometry_tests.rs index f623d2cf..f63e3d07 100644 --- a/crates/tinyinference-decisions/src/emulation/geometry_tests.rs +++ b/crates/tinyinference-decisions/src/emulation/geometry_tests.rs @@ -50,6 +50,7 @@ fn edges_are_fractions_of_the_whole_image() { #[test] fn pick_unions_the_positive_component_around_the_best_cell() { let cells = grid(rect(0, 0, 400, 400), 4); + // One row of the 4x4 grid per line. #[rustfmt::skip] let probabilities = [ 0.1, 0.6, 0.0, 0.0, @@ -67,6 +68,7 @@ fn pick_unions_the_positive_component_around_the_best_cell() { #[test] fn pick_joins_diagonal_neighbours() { let cells = grid(rect(0, 0, 400, 400), 4); + // One row of the 4x4 grid per line. #[rustfmt::skip] let probabilities = [ 0.9, 0.0, 0.0, 0.0, diff --git a/crates/tinyinference-decisions/tests/public_api.rs b/crates/tinyinference-decisions/tests/public_api.rs index 23f4d4ff..e406abf7 100644 --- a/crates/tinyinference-decisions/tests/public_api.rs +++ b/crates/tinyinference-decisions/tests/public_api.rs @@ -33,3 +33,19 @@ fn public_types_build_a_valid_jev_request() { ); request.validate().expect("public request should validate"); } + +#[test] +fn surplus_emulates_box_questions_by_default_and_callers_can_override_it() { + use tinyinference_decisions::{BoxEmulation, BoxSupport, ClientConfig, Provider}; + + let surplus = ClientConfig::surplus("key"); + assert_eq!(surplus.provider, Provider::Surplus); + assert!(!Provider::Surplus.answers_box()); + assert_eq!( + surplus.box_support(), + BoxSupport::Emulated(BoxEmulation::default()) + ); + let native = surplus.with_box_support(BoxSupport::Native); + assert_eq!(native.box_support(), BoxSupport::Native); + Client::new(native).expect("surplus configuration should be valid"); +} From f66ded2686a89d983ee1410ba297ea1fc9caf738 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 04:54:52 +0000 Subject: [PATCH 13/19] feat(tinyinference-decisions): add Surplus Intelligence provider and box emulation Adds a `ClientConfig::surplus` constructor targeting Surplus Intelligence, which accepts OpenRouter-style resolved model ids, along with a new `Provider::Surplus` variant. Box questions are now emulated for providers that cannot answer them by splitting the image into upscaled crops and issuing one Noul per crop, with usage summed across the sub-requests. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- CHANGELOG.md | 12 +++++ crates/tinyinference-decisions/src/README.md | 1 + .../src/client/README.md | 8 +++ crates/tinyinference-decisions/src/lib.rs | 2 + .../src/request/README.md | 3 ++ docs/tinyinference-decisions.md | 53 ++++++++++++++++++- 6 files changed, 77 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b72af2d..9948cc61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ ## Unreleased +- `tinyinference-decisions`: Surplus Intelligence provider. `ClientConfig::surplus` + posts to `https://api.surplusintelligence.ai/v1/decisions` and accepts + OpenRouter-style resolved model ids (`jev-1.14` answered as + `typesafe/jev-1.14-20261014`); `Provider::Surplus` is a new variant, so an + exhaustive `match` on `Provider` needs an arm for it. +- `tinyinference-decisions`: Box questions are emulated for providers that do + not answer them (`Provider::answers_box`, `BoxSupport`, `BoxEmulation`, + `ClientConfig::with_box_support` / `box_support`). The client cuts the image + into a grid of upscaled PNG crops, asks one Noul per crop, optionally refines + inside the box found, and returns ordinary `Answer::Box` answers with usage + summed over the sub-requests. Adds the `image` dependency (PNG and JPEG + only). - Security: re-pin `tinystoragedrivers-*` to 2.1.3, which moves `rusqlite` to 0.41.1 and `libsqlite3-sys` to 0.39.x (SQLite 3.54). Clears RUSTSEC-2026-0459 from the dev-dependency tree of `tinyinference-image`; no advisory ignore was added. diff --git a/crates/tinyinference-decisions/src/README.md b/crates/tinyinference-decisions/src/README.md index 4da3c7aa..a1a2d556 100644 --- a/crates/tinyinference-decisions/src/README.md +++ b/crates/tinyinference-decisions/src/README.md @@ -4,6 +4,7 @@ | --- | --- | | `client/` | HTTP execution, bounded retry, secret handling, and measurements | | `request/` | typed Choice, Score, and Noul request payloads and validation | +| `emulation/` | Box questions emulated with Noul questions over image crops | | `response/` | typed answers and request-relative response validation | | `error/` | crate-wide classified failures | | `sage/` | Levanto Sage decision requests, results, and HTTP transport | diff --git a/crates/tinyinference-decisions/src/client/README.md b/crates/tinyinference-decisions/src/client/README.md index bcd4a996..d9e14245 100644 --- a/crates/tinyinference-decisions/src/client/README.md +++ b/crates/tinyinference-decisions/src/client/README.md @@ -27,3 +27,11 @@ The endpoint is used exactly as given and must pass the same URL checks. There is no default model, so callers set `EvaluationRequest::model`; the response must echo that model id. An empty key is allowed for unauthenticated local servers and then no `Authorization` header is sent. + +`ClientConfig::surplus` targets Surplus Intelligence at +`https://api.surplusintelligence.ai/v1/decisions` and validates the response +model like OpenRouter. Surplus rejects Box questions, so its default +`BoxSupport` is `Emulated`: `emulated.rs` sends the probe requests planned by +`crate::emulation` one after another through the same retry loop and +response validation, then validates the synthesized response against the +caller's request. `with_box_support` overrides the provider default. diff --git a/crates/tinyinference-decisions/src/lib.rs b/crates/tinyinference-decisions/src/lib.rs index dfa15ca7..f41ece83 100644 --- a/crates/tinyinference-decisions/src/lib.rs +++ b/crates/tinyinference-decisions/src/lib.rs @@ -38,6 +38,8 @@ //! ``` //! //! [`sage`] contains Levanto Sage's Yes/No, Choice, Scale, Sort, and Tags API. +//! [`ClientConfig::surplus`] targets Surplus Intelligence; Box questions sent +//! to a provider without them are emulated (see [`BoxSupport`]). //! Use [`ClientConfig::self_hosted`] with [`EvaluationRequest::new`] to target //! a user-managed OpenJEV or other Jev-compatible decisions endpoint with an //! explicit model id and optional bearer authentication. diff --git a/crates/tinyinference-decisions/src/request/README.md b/crates/tinyinference-decisions/src/request/README.md index f2d5df27..b3622fe2 100644 --- a/crates/tinyinference-decisions/src/request/README.md +++ b/crates/tinyinference-decisions/src/request/README.md @@ -5,3 +5,6 @@ System One endpoint. `types.rs` holds the serde wire types, `mod.rs` validates the documented primitive bounds before transport, and `mod_tests.rs` pins both. `image.rs` builds inline image assets and checks them against the provider's image limits (format, side lengths, sizes) by reading only the file header. +Box questions stay typed here even for providers that cannot answer them; +the client rewrites them (see `../emulation/`) after this module validates the +caller's request. diff --git a/docs/tinyinference-decisions.md b/docs/tinyinference-decisions.md index ea9a3973..2ae3bb0a 100644 --- a/docs/tinyinference-decisions.md +++ b/docs/tinyinference-decisions.md @@ -26,9 +26,9 @@ println!("{:?}", result.response.answers["route"]); # } ``` -The API key is read from `TYPESAFE_API_KEY` or supplied through `ClientConfig`. Keys are redacted from `Debug` output and never included in errors. The client supports TypeSafe, OpenRouter, and OpenJEV endpoints, bounded retries, request and response validation, and explicit per-call failure metadata. +The API key is read from `TYPESAFE_API_KEY` or supplied through `ClientConfig`. Keys are redacted from `Debug` output and never included in errors. The client supports TypeSafe, OpenRouter, OpenJEV, and Surplus Intelligence endpoints, bounded retries, request and response validation, and explicit per-call failure metadata. -For OpenRouter, construct `ClientConfig::openrouter("")`. Tiny Humans proxy users can use `ClientConfig::tinyhumans_openrouter("")`. For OpenJEV, use `ClientConfig::openjev("")` with `EvaluationRequest::openjev(...)`, which selects OpenJEV's `openjev` model id. Custom endpoints can be configured with `.with_endpoint_url(...)`. For a self-hosted Jev-compatible decision model, use `ClientConfig::self_hosted("", "")` and set `EvaluationRequest::model` to the model the server answers as; the response must echo it. +For OpenRouter, construct `ClientConfig::openrouter("")`. Tiny Humans proxy users can use `ClientConfig::tinyhumans_openrouter("")`. For OpenJEV, use `ClientConfig::openjev("")` with `EvaluationRequest::openjev(...)`, which selects OpenJEV's `openjev` model id. For Surplus Intelligence, use `ClientConfig::surplus("")` with an explicit release such as `EvaluationRequest::new("jev-1.14", ...)`; it posts to `https://api.surplusintelligence.ai/v1/decisions`, and the response may name the resolved release (`typesafe/jev-1.14-20261014`), as on OpenRouter. Custom endpoints can be configured with `.with_endpoint_url(...)`. For a self-hosted Jev-compatible decision model, use `ClientConfig::self_hosted("", "")` and set `EvaluationRequest::model` to the model the server answers as; the response must echo it. The live example spends a real API call: @@ -80,6 +80,55 @@ all, and every asset key referenced by the state or a question. Images are neither resized nor rotated; that stays with the caller. A Box answer is always one box, so ask a Noul alongside it when the thing may be absent. +## Emulated Box questions + +Some providers serve Jev without the Box question: Surplus Intelligence +answers a request containing one with 400 (`questions. has a type that is +not one of noul | choice | score`), while Noul questions about images work. +`Provider::answers_box()` reports this, and `ClientConfig::box_support()` +resolves it to `BoxSupport::Native` or `BoxSupport::Emulated(BoxEmulation)`. +`ClientConfig::with_box_support(...)` overrides the provider's default in +either direction. Callers do not change anything else: they send Box +questions and receive `Answer::Box` answers whose ids match their questions. + +Emulation works inside `Client::evaluate`: + +1. Each Box question's image is the asset whose key its instructions mention; + else the asset held by a top-level state field it names in backticks + (`` `page` `` with `{"page": "shot-1"}`); else the request's only image. + Anything else is an `Error::InvalidRequest` naming the question, raised + before any request is sent. +2. The image is decoded and cut into a `grid` by `grid` set of cells (4 by 4 + by default). Each cell is upscaled (bilinear) until its shorter side is + 280 pixels, encoded as PNG, and asked about as a Noul: does this tile show + all or part of the original instructions? +3. Probes are packed into requests holding at most 32 MiB of images; a crop is + at most 16 MiB. They share a request with the caller's other questions + unless that request carries the image the crops were cut from, so a tile + is never judged next to the whole picture. Sub-requests run one after + another, each with the normal retry policy and response validation. +4. The box is the bounding rectangle of the cells with probability 0.5 or more + that connect (diagonals included) to the best cell, so a second instance + elsewhere does not stretch it. With no such cell it is the best cell + alone. Confidence is the best cell's probability. Edges are fractions of + the original image. +5. With `refine` (the default), the box is cut into the same grid again and + asked about in one more round. The refined box replaces the first only when + one of its cells reaches 0.5. Refinement is skipped when the box already + covers the whole image or its cells would be under 32 pixels; a box more + than 4 times longer than wide (or the reverse) is first grown about its + centre so its cells can be upscaled within the 1568-pixel maximum. + +Probe question ids and asset keys start with `__tinyinference_box`, which +emulating requests reserve. The result's `usage` sums every sub-request (a +field stays absent only when no sub-request reported it), `attempts` counts +every HTTP attempt, `latency` covers them all, and `request_id` is the first +one returned. A Box question costs up to `2 * grid * grid` Noul questions with +an image each, and up to three sequential HTTP requests in the usual case +(the caller's other questions with the original image, then each round). The +precision is one cell, or one cell of a cell after refinement; this is a +coarse locator, not a native detector. + ## Local decision models Use `ClientConfig::self_hosted` for a locally served OpenJEV, Decider, Surogate From 2685fc0688e6eb97451af9b3cbdfb42a4069e301 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 04:56:25 +0000 Subject: [PATCH 14/19] test(emulation): cover empty crop returning a request error Add a test asserting that cropping an image to an empty region yields an InvalidRequest error rather than panicking, guarding the boundary case where the crop rectangle falls outside the source dimensions. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/emulation/crop_tests.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/tinyinference-decisions/src/emulation/crop_tests.rs b/crates/tinyinference-decisions/src/emulation/crop_tests.rs index ed503560..81767c42 100644 --- a/crates/tinyinference-decisions/src/emulation/crop_tests.rs +++ b/crates/tinyinference-decisions/src/emulation/crop_tests.rs @@ -93,3 +93,13 @@ fn an_undecodable_image_names_its_asset() { "{error:?}" ); } + +#[test] +fn an_empty_crop_is_a_request_error_not_a_panic() { + let source = picture(400, 300, rect(0, 0, 1, 1)); + let image = decode("page", &ImageAsset::png(&encode(&source, Codec::Png))).unwrap(); + assert!(matches!( + crop(&image, rect(10, 10, 10, 50)), + Err(Error::InvalidRequest { .. }) + )); +} From f4501ce6bffd5fc91deab15027cae261e66d7f24 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 04:56:37 +0000 Subject: [PATCH 15/19] fix(emulation): reject empty crop rectangles Emulated Box crops with a zero width or height now return an invalid request error instead of being passed to the image cropper. The docs were also clarified to say attached images are sent as given, with emulated Box questions sending crops alongside rather than altering them. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyinference-decisions/src/emulation/crop.rs | 5 +++++ docs/tinyinference-decisions.md | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/tinyinference-decisions/src/emulation/crop.rs b/crates/tinyinference-decisions/src/emulation/crop.rs index bbb7672a..d211f844 100644 --- a/crates/tinyinference-decisions/src/emulation/crop.rs +++ b/crates/tinyinference-decisions/src/emulation/crop.rs @@ -32,6 +32,11 @@ pub(crate) fn decode(key: &str, asset: &ImageAsset) -> Result { /// break the maximum side, which then clamps the longer side. pub(crate) fn crop(image: &DynamicImage, rect: Rect) -> Result<(ImageAsset, usize)> { let (width, height) = (rect.width(), rect.height()); + if width == 0 || height == 0 { + return Err(Error::invalid_request( + "an emulated Box image crop must not be empty", + )); + } let mut crop = image.crop_imm(rect.left, rect.top, width, height); let shorter = width.min(height); if shorter < MIN_SIDE { diff --git a/docs/tinyinference-decisions.md b/docs/tinyinference-decisions.md index 2ae3bb0a..f831cf95 100644 --- a/docs/tinyinference-decisions.md +++ b/docs/tinyinference-decisions.md @@ -76,8 +76,9 @@ The client sends `TypeSafe-Danger-Zone-Images: true` on any request with an image or a Box question, as the provider requires. `validate` checks the documented limits before anything is sent: PNG or JPEG whose header matches the declared format, 280 to 1568 pixels per side, 16 MiB per image and 32 MiB in -all, and every asset key referenced by the state or a question. Images are -neither resized nor rotated; that stays with the caller. A Box answer is always +all, and every asset key referenced by the state or a question. Attached +images are sent as given, never resized or rotated (emulated Box questions, +below, send crops alongside rather than altering them). A Box answer is always one box, so ask a Noul alongside it when the thing may be absent. ## Emulated Box questions From d27f387080bf0f7ecfa79c1c512b70b2b1c8d79e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 04:57:52 +0000 Subject: [PATCH 16/19] feat(examples): add live surplus box example Add an example that runs the surplus box inference live, giving a runnable demonstration of the decision logic outside the test suite. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../examples/live_surplus_box.rs | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 crates/tinyinference-decisions/examples/live_surplus_box.rs diff --git a/crates/tinyinference-decisions/examples/live_surplus_box.rs b/crates/tinyinference-decisions/examples/live_surplus_box.rs new file mode 100644 index 00000000..3f2700ae --- /dev/null +++ b/crates/tinyinference-decisions/examples/live_surplus_box.rs @@ -0,0 +1,64 @@ +//! Live check of Box emulation on Surplus Intelligence, using a synthetic +//! image. +//! +//! Run with `SURPLUS_API_KEY` set (and optionally `SURPLUS_JEV_MODEL`, +//! default `jev-1.14`). Surplus does not accept Box questions, so the client +//! answers this one by cropping the picture and asking a Noul per cell. Each +//! run spends a few cents of the account's balance at most. + +use std::collections::BTreeMap; +use std::io::Cursor; + +use image::{ImageFormat as Encoding, Rgb, RgbImage}; +use serde_json::json; +use tinyinference_decisions::{ + Answer, BoxQuestion, Client, ClientConfig, EvaluationRequest, ImageAsset, Question, +}; + +/// The red square drawn on a white 640×480 picture, as pixel bounds. +const SQUARE: (u32, u32, u32, u32) = (400, 80, 520, 200); + +#[tokio::main] +async fn main() -> Result<(), Box> { + let key = std::env::var("SURPLUS_API_KEY")?; + let model = std::env::var("SURPLUS_JEV_MODEL").unwrap_or_else(|_| "jev-1.14".to_owned()); + let client = Client::new(ClientConfig::surplus(key))?; + + let (left, top, right, bottom) = SQUARE; + let picture = RgbImage::from_fn(640, 480, |x, y| { + if (left..right).contains(&x) && (top..bottom).contains(&y) { + Rgb([220, 20, 20]) + } else { + Rgb([255, 255, 255]) + } + }); + let mut png = Vec::new(); + picture.write_to(&mut Cursor::new(&mut png), Encoding::Png)?; + + let questions = BTreeMap::from([( + "square".to_owned(), + Question::Box(BoxQuestion { + instructions: json!({ "find": "the red square in `picture`" }), + }), + )]); + let request = EvaluationRequest::new(model, json!({ "picture": "picture-image" }), questions) + .with_image("picture-image", ImageAsset::png(&png)); + + let result = client.evaluate(&request).await?; + println!("model: {}", result.response.model); + println!( + "expected: left {:.3} right {:.3} top {:.3} bottom {:.3}", + f64::from(left) / 640.0, + f64::from(right) / 640.0, + f64::from(top) / 480.0, + f64::from(bottom) / 480.0, + ); + match result.response.answers.get("square") { + Some(Answer::Box(found)) => println!( + "box: left {:.3} right {:.3} top {:.3} bottom {:.3} (confidence {:.2})", + found.edges.left, found.edges.right, found.edges.top, found.edges.bottom, found.confidence, + ), + other => println!("unexpected answer: {other:?}"), + } + Ok(()) +} From 30d781f050ebdf218507e3ec0a5c67f7de08cd5c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 04:58:04 +0000 Subject: [PATCH 17/19] test(decisions): live Surplus Box emulation example Draws a red square on a white picture and asks Surplus (which refuses Box questions) to box it; the client answers through cell Nouls. Run 2026-11-15: expected x 0.625-0.812 y 0.167-0.417, got x 0.625-0.875 y 0.125-0.500, confidence 0.84, model typesafe/jev-1.14-20261014. Co-authored-by: Medulla --- crates/tinyinference-decisions/examples/live_surplus_box.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/tinyinference-decisions/examples/live_surplus_box.rs b/crates/tinyinference-decisions/examples/live_surplus_box.rs index 3f2700ae..9af1dfb1 100644 --- a/crates/tinyinference-decisions/examples/live_surplus_box.rs +++ b/crates/tinyinference-decisions/examples/live_surplus_box.rs @@ -56,7 +56,11 @@ async fn main() -> Result<(), Box> { match result.response.answers.get("square") { Some(Answer::Box(found)) => println!( "box: left {:.3} right {:.3} top {:.3} bottom {:.3} (confidence {:.2})", - found.edges.left, found.edges.right, found.edges.top, found.edges.bottom, found.confidence, + found.edges.left, + found.edges.right, + found.edges.top, + found.edges.bottom, + found.confidence, ), other => println!("unexpected answer: {other:?}"), } From 0121dc05aa13879c92111af9ca2f8d116150f104 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 05:11:16 +0000 Subject: [PATCH 18/19] fix(emulation): match asset keys as whole tokens Asset keys mentioned in Box instructions are now matched as whole tokens rather than substrings, so a key like `a` no longer matches inside `square` and `img-1` no longer matches inside `img-10`. Keys nested within a longer key's occurrence are also excluded, while keys named in separate values are each still found. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/emulation/mod.rs | 67 +++++++++++++++---- 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/crates/tinyinference-decisions/src/emulation/mod.rs b/crates/tinyinference-decisions/src/emulation/mod.rs index 6c9459bc..6d015ee5 100644 --- a/crates/tinyinference-decisions/src/emulation/mod.rs +++ b/crates/tinyinference-decisions/src/emulation/mod.rs @@ -361,8 +361,8 @@ pub(crate) fn refine(first: Vec, second: Vec>) -> Vec { /// The asset key of the image a Box question is about. /// -/// In order: the one asset key its instructions mention (a key contained in a -/// longer mentioned key does not count); else the one asset held by a +/// In order: the one asset key its instructions mention as a whole token (see +/// [`referenced`]); else the one asset held by a /// top-level state field its instructions name in backticks; else the /// request's only image. fn source_asset(request: &EvaluationRequest, id: &str, instructions: &Value) -> Result { @@ -371,19 +371,12 @@ fn source_asset(request: &EvaluationRequest, id: &str, instructions: &Value) -> "box question `{id}` names more than one image; emulating Box needs exactly one" )) }; + let mut found = BTreeSet::new(); + referenced(instructions, request.assets.keys(), &mut found); let mentioned: Vec<&String> = request .assets .keys() - .filter(|key| value_mentions(instructions, key)) - .collect(); - let mentioned: Vec<&String> = mentioned - .iter() - .filter(|key| { - !mentioned - .iter() - .any(|other| other != *key && other.contains(key.as_str())) - }) - .copied() + .filter(|key| found.contains(key.as_str())) .collect(); match mentioned.as_slice() { [key] => return Ok((*key).clone()), @@ -411,6 +404,56 @@ fn source_asset(request: &EvaluationRequest, id: &str, instructions: &Value) -> ))) } +/// Collect into `found` the asset keys that `value` references. +/// +/// A reference is an occurrence of the key that is not glued to a neighbouring +/// word character (`a` is not found in `square`, `img-1` not in `img-10`) and +/// does not sit inside the occurrence of a longer key (`img` in `img.png` +/// when both are keys). Each string is judged on its own, so `img-1` and +/// `img-10` named in separate values are both found. +fn referenced<'a>( + value: &Value, + keys: impl Iterator + Clone, + found: &mut BTreeSet<&'a str>, +) { + match value { + Value::String(text) => scan(text, keys, found), + Value::Array(items) => items + .iter() + .for_each(|item| referenced(item, keys.clone(), found)), + Value::Object(map) => map.iter().for_each(|(name, item)| { + scan(name, keys.clone(), found); + referenced(item, keys.clone(), found); + }), + _ => {} + } +} + +fn scan<'a>(text: &str, keys: impl Iterator, found: &mut BTreeSet<&'a str>) { + let word = |c: char| c.is_alphanumeric() || c == '_' || c == '-'; + let mut spans: Vec<(usize, usize, &'a str)> = Vec::new(); + for key in keys.filter(|key| !key.is_empty()) { + for (start, _) in text.match_indices(key.as_str()) { + let end = start + key.len(); + let glued = (word(key.chars().next().unwrap_or(' ')) + && text[..start].chars().next_back().is_some_and(word)) + || (word(key.chars().next_back().unwrap_or(' ')) + && text[end..].chars().next().is_some_and(word)); + if !glued { + spans.push((start, end, key.as_str())); + } + } + } + for &(start, end, key) in &spans { + let nested = spans.iter().any(|&(s, e, other)| { + other.len() > key.len() && s <= start && end <= e + }); + if !nested { + found.insert(key); + } + } +} + /// Every span between backticks in the string values of `value`. fn backticked(value: &Value, spans: &mut Vec) { match value { From 96b9d984dbad37136fb8b8e5df4590fa08d74049 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 15 Nov 2026 05:11:28 +0000 Subject: [PATCH 19/19] refactor(emulation): reformat nested span check Reformatted the nested-span predicate in the scan function to a single chained expression with no behaviour change. Added tests covering independently named prefix keys staying ambiguous and short keys inside ordinary words not being treated as references. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/emulation/mod.rs | 6 ++-- .../src/emulation/mod_tests.rs | 28 +++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/crates/tinyinference-decisions/src/emulation/mod.rs b/crates/tinyinference-decisions/src/emulation/mod.rs index 6d015ee5..aa1ffd88 100644 --- a/crates/tinyinference-decisions/src/emulation/mod.rs +++ b/crates/tinyinference-decisions/src/emulation/mod.rs @@ -445,9 +445,9 @@ fn scan<'a>(text: &str, keys: impl Iterator, found: &mut BTre } } for &(start, end, key) in &spans { - let nested = spans.iter().any(|&(s, e, other)| { - other.len() > key.len() && s <= start && end <= e - }); + let nested = spans + .iter() + .any(|&(s, e, other)| other.len() > key.len() && s <= start && end <= e); if !nested { found.insert(key); } diff --git a/crates/tinyinference-decisions/src/emulation/mod_tests.rs b/crates/tinyinference-decisions/src/emulation/mod_tests.rs index afc04d34..252d70ba 100644 --- a/crates/tinyinference-decisions/src/emulation/mod_tests.rs +++ b/crates/tinyinference-decisions/src/emulation/mod_tests.rs @@ -77,6 +77,34 @@ fn a_key_contained_in_a_longer_named_key_is_not_ambiguous() { assert_eq!(source_of(&request).unwrap(), "img-10"); } +#[test] +fn independently_named_prefix_keys_stay_ambiguous() { + let request = request( + json!({"a": "img-1", "b": "img-10"}), + vec![( + "button", + boxed(json!({"source": "img-1", "comparison": "img-10"})), + )], + ) + .with_image("img-1", png(300, 300)) + .with_image("img-10", png(300, 300)); + assert!(matches!( + source_of(&request), + Err(Error::InvalidRequest { .. }) + )); +} + +#[test] +fn a_short_key_inside_ordinary_words_is_not_a_reference() { + let request = request( + json!({"page": "b", "other": "a"}), + vec![("button", boxed(json!("the red square in `page`")))], + ) + .with_image("a", png(300, 300)) + .with_image("b", png(300, 300)); + assert_eq!(source_of(&request).unwrap(), "b"); +} + #[test] fn the_source_is_the_asset_held_by_a_backticked_state_field() { let request = request(