From ad86c62793c15ba0ad49698c54e939104c7f2b5d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 14 Nov 2026 11:52:05 +0000 Subject: [PATCH 1/6] feat(request): add image assets and box questions to evaluation requests EvaluationRequest now carries an assets map of base64-encoded images keyed by the reference string used in state or question instructions, plus a with_image builder and a needs_image_opt_in check that also covers box questions. A new Box question variant lets callers ask where something is in an attached image, and empty assets are skipped on the wire so text-only requests serialise as before. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- crates/tinyinference-decisions/Cargo.toml | 2 + .../src/request/types.rs | 70 +++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/crates/tinyinference-decisions/Cargo.toml b/crates/tinyinference-decisions/Cargo.toml index 5d01f398..dc37d71b 100644 --- a/crates/tinyinference-decisions/Cargo.toml +++ b/crates/tinyinference-decisions/Cargo.toml @@ -13,6 +13,8 @@ categories = ["api-bindings", "asynchronous"] [dependencies] async-trait = { workspace = true } +# Encodes attached images for the System One `assets` map. +base64 = "0.23" httpdate = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } diff --git a/crates/tinyinference-decisions/src/request/types.rs b/crates/tinyinference-decisions/src/request/types.rs index a8ca1329..9a77bf5b 100644 --- a/crates/tinyinference-decisions/src/request/types.rs +++ b/crates/tinyinference-decisions/src/request/types.rs @@ -14,6 +14,11 @@ pub struct EvaluationRequest { pub model: String, /// Independently evaluated questions keyed by caller-owned ids. pub questions: BTreeMap, + /// Images keyed by the reference string that `state` or a question uses + /// to point at them. Empty for a text-only request, and then absent from + /// the wire form, so text requests serialise exactly as before. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub assets: BTreeMap, } impl EvaluationRequest { @@ -33,9 +38,34 @@ impl EvaluationRequest { state: state.into(), model: model.into(), questions, + assets: BTreeMap::new(), } } + /// Attach an image under `key`. + /// + /// `key` is the reference string: put it somewhere in `state` (for + /// example `{"photo": key}`) or in a question's instructions so Jev knows + /// where the image belongs. An image nobody references is rejected by + /// [`Self::validate`]. Attaching any asset makes the client send + /// `TypeSafe-Danger-Zone-Images: true`. + #[must_use] + pub fn with_image(mut self, key: impl Into, image: ImageAsset) -> Self { + self.assets.insert(key.into(), Asset::Image(image)); + self + } + + /// Whether the request carries an image or asks a [`Question::Box`], the + /// two things the provider admits only with its image opt-in header. + #[must_use] + pub fn needs_image_opt_in(&self) -> bool { + !self.assets.is_empty() + || self + .questions + .values() + .any(|question| matches!(question, Question::Box(_))) + } + /// Build a request using the stable Jev alias. #[must_use] pub fn jev(state: impl Into, questions: BTreeMap) -> Self { @@ -59,6 +89,46 @@ pub enum Question { Score(Score), /// Estimate the probability that a yes/no condition holds. Noul(Noul), + /// Locate one thing in an image (Jev 1.14 or later). The answer is a + /// single box; ask a [`Noul`] alongside it to learn whether the thing is + /// there at all, because a box is always returned. + Box(BoxQuestion), +} + +/// A question asking where something is in an attached image. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct BoxQuestion { + /// What to find, and in which image: name the state field holding the + /// image in backticks, for example ``"the search button in `page`"``. + pub instructions: Value, +} + +/// One attached asset. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum Asset { + /// A still PNG or JPEG image. + Image(ImageAsset), +} + +/// A still image sent inline as base64. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct ImageAsset { + /// Encoding of the image bytes. + pub format: ImageFormat, + /// Standard padded base64 of the image file, with no `data:` prefix. + pub data_base64: String, +} + +/// Image encodings the provider accepts. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ImageFormat { + /// PNG. + Png, + /// JPEG, written `jpg` on the wire. + #[serde(rename = "jpg", alias = "jpeg")] + Jpg, } /// A closed-set selection question. From 855c19257a692142576bb18118378fa6f6264cfd Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 14 Nov 2026 11:53:30 +0000 Subject: [PATCH 2/6] feat(request): add image assets and box questions Requests can now carry base64-encoded image assets referenced from state or question text, plus a new Box question type that asks for bounding boxes. Validation rejects unreferenced or oversized assets, and the client sends the provider's image opt-in header whenever a request includes an image or Box question. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- Cargo.lock | 1 + .../tinyinference-decisions/src/client/mod.rs | 6 + crates/tinyinference-decisions/src/lib.rs | 9 +- .../src/request/image.rs | 118 ++++++++++++++++++ .../src/request/image_tests.rs | 88 +++++++++++++ .../src/request/mod.rs | 42 ++++++- .../src/request/mod_tests.rs | 112 +++++++++++++++++ .../src/response/mod.rs | 16 ++- .../src/response/types.rs | 38 ++++++ 9 files changed, 426 insertions(+), 4 deletions(-) create mode 100644 crates/tinyinference-decisions/src/request/image.rs create mode 100644 crates/tinyinference-decisions/src/request/image_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 7c5124e9..c06e9e9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1700,6 +1700,7 @@ version = "0.5.0" dependencies = [ "async-trait", "axum", + "base64 0.23.1", "httpdate", "reqwest", "serde", diff --git a/crates/tinyinference-decisions/src/client/mod.rs b/crates/tinyinference-decisions/src/client/mod.rs index 95d319e5..67a17142 100644 --- a/crates/tinyinference-decisions/src/client/mod.rs +++ b/crates/tinyinference-decisions/src/client/mod.rs @@ -16,6 +16,9 @@ use crate::{Error, EvaluationRequest, EvaluationResponse, Result}; const MAX_RETRIES: u32 = 100; const MAX_RESPONSE_BYTES: usize = 16 * 1024 * 1024; +/// Header the provider requires on every request that carries an image or a +/// Box question; without it such a request is rejected with 400. +const IMAGE_OPT_IN_HEADER: &str = "TypeSafe-Danger-Zone-Images"; impl Client { /// Construct a client from an explicit configuration. @@ -143,6 +146,9 @@ impl Client { ) }); let mut builder = self.http.post(&url).json(request); + if request.needs_image_opt_in() { + builder = builder.header(IMAGE_OPT_IN_HEADER, "true"); + } let api_key = self.config.api_key.expose(); if !(self.config.provider == Provider::SelfHosted && api_key.trim().is_empty()) { builder = builder.bearer_auth(api_key); diff --git a/crates/tinyinference-decisions/src/lib.rs b/crates/tinyinference-decisions/src/lib.rs index 1d24c74b..939c6a93 100644 --- a/crates/tinyinference-decisions/src/lib.rs +++ b/crates/tinyinference-decisions/src/lib.rs @@ -54,5 +54,10 @@ pub use client::{ Client, ClientConfig, EvaluationFailure, EvaluationResult, Provider, RetryPolicy, }; pub use error::{Error, Result}; -pub use request::{Choice, EvaluationRequest, Noul, NoulCriteria, Question, Score}; -pub use response::{Answer, ChoiceAnswer, EvaluationResponse, NoulAnswer, ScoreAnswer, Usage}; +pub use request::{ + Asset, BoxQuestion, Choice, EvaluationRequest, ImageAsset, ImageFormat, Noul, NoulCriteria, + Question, Score, +}; +pub use response::{ + Answer, BoxAnswer, BoxEdges, ChoiceAnswer, EvaluationResponse, NoulAnswer, ScoreAnswer, Usage, +}; diff --git a/crates/tinyinference-decisions/src/request/image.rs b/crates/tinyinference-decisions/src/request/image.rs new file mode 100644 index 00000000..d7726a21 --- /dev/null +++ b/crates/tinyinference-decisions/src/request/image.rs @@ -0,0 +1,118 @@ +//! Building inline image assets and checking them against the provider's +//! documented image limits before anything is sent. + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; + +use super::types::{ImageAsset, ImageFormat}; +use crate::{Error, Result}; + +/// Smallest accepted width or height, in pixels. +pub(crate) const MIN_SIDE: u32 = 280; +/// Largest accepted width or height, in pixels. +pub(crate) const MAX_SIDE: u32 = 1568; +/// Largest accepted image file. +pub(crate) const MAX_IMAGE_BYTES: usize = 16 * 1024 * 1024; +/// Largest accepted total of all image files in one request. +pub(crate) const MAX_TOTAL_IMAGE_BYTES: usize = 32 * 1024 * 1024; +/// Longest accepted asset reference string, in bytes. +pub(crate) const MAX_REFERENCE_BYTES: usize = 1024; + +const PNG_SIGNATURE: &[u8] = b"\x89PNG\r\n\x1a\n"; + +impl ImageAsset { + /// An image from the bytes of a PNG file. + #[must_use] + pub fn png(bytes: &[u8]) -> Self { + Self { + format: ImageFormat::Png, + data_base64: STANDARD.encode(bytes), + } + } + + /// An image from the bytes of a JPEG file. + #[must_use] + pub fn jpeg(bytes: &[u8]) -> Self { + Self { + format: ImageFormat::Jpg, + data_base64: STANDARD.encode(bytes), + } + } + + /// Decode the image and check it against the provider's limits, returning + /// its size in bytes. + pub(crate) fn checked_len(&self) -> Result { + let bytes = STANDARD + .decode(&self.data_base64) + .map_err(|_| Error::invalid_request("image data_base64 must be padded base64"))?; + if bytes.len() > MAX_IMAGE_BYTES { + return Err(Error::invalid_request("image must be at most 16 MiB")); + } + let (width, height) = dimensions(self.format, &bytes)?; + if !(MIN_SIDE..=MAX_SIDE).contains(&width) || !(MIN_SIDE..=MAX_SIDE).contains(&height) { + return Err(Error::invalid_request(format!( + "image sides must be {MIN_SIDE} to {MAX_SIDE} pixels, got {width}x{height}" + ))); + } + Ok(bytes.len()) + } +} + +/// Width and height read from the image header. +pub(crate) fn dimensions(format: ImageFormat, bytes: &[u8]) -> Result<(u32, u32)> { + match format { + ImageFormat::Png => png_dimensions(bytes), + ImageFormat::Jpg => jpeg_dimensions(bytes), + } + .ok_or_else(|| match format { + ImageFormat::Png => Error::invalid_request("image is not a readable PNG"), + ImageFormat::Jpg => Error::invalid_request("image is not a readable JPEG"), + }) +} + +fn png_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { + // The signature is followed by the IHDR chunk: length, "IHDR", width, height. + if !bytes.starts_with(PNG_SIGNATURE) || bytes.get(12..16)? != b"IHDR" { + return None; + } + let width = u32::from_be_bytes(bytes.get(16..20)?.try_into().ok()?); + let height = u32::from_be_bytes(bytes.get(20..24)?.try_into().ok()?); + Some((width, height)) +} + +fn jpeg_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { + if !bytes.starts_with(&[0xFF, 0xD8]) { + return None; + } + let mut at = 2; + loop { + // Skip fill bytes before a marker. + while *bytes.get(at)? == 0xFF && *bytes.get(at + 1)? == 0xFF { + at += 1; + } + if *bytes.get(at)? != 0xFF { + return None; + } + let marker = *bytes.get(at + 1)?; + // Markers without a length field. + if marker == 0x01 || (0xD0..=0xD7).contains(&marker) { + at += 2; + continue; + } + let length = usize::from(u16::from_be_bytes([*bytes.get(at + 2)?, *bytes.get(at + 3)?])); + // Start-of-frame markers carry the size; C4, C8 and CC do not. + if matches!(marker, 0xC0..=0xCF) && !matches!(marker, 0xC4 | 0xC8 | 0xCC) { + let height = u16::from_be_bytes([*bytes.get(at + 5)?, *bytes.get(at + 6)?]); + let width = u16::from_be_bytes([*bytes.get(at + 7)?, *bytes.get(at + 8)?]); + return Some((u32::from(width), u32::from(height))); + } + if length < 2 { + return None; + } + at += 2 + length; + } +} + +#[cfg(test)] +#[path = "image_tests.rs"] +pub(crate) mod tests; diff --git a/crates/tinyinference-decisions/src/request/image_tests.rs b/crates/tinyinference-decisions/src/request/image_tests.rs new file mode 100644 index 00000000..da8f1286 --- /dev/null +++ b/crates/tinyinference-decisions/src/request/image_tests.rs @@ -0,0 +1,88 @@ +//! Image asset construction, header reading, and limit checks. + +use super::*; + +/// The first bytes of a PNG file of the given size: enough for the header. +pub(crate) fn png_header(width: u32, height: u32) -> Vec { + let mut bytes = PNG_SIGNATURE.to_vec(); + bytes.extend_from_slice(&13_u32.to_be_bytes()); + bytes.extend_from_slice(b"IHDR"); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&[8, 6, 0, 0, 0]); + bytes +} + +/// The first bytes of a baseline JPEG of the given size, with an APP0 segment +/// before the frame so the reader has to walk a segment. +fn jpeg_header(width: u16, height: u16) -> Vec { + let mut bytes = vec![0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10]; + bytes.extend_from_slice(b"JFIF\0\x01\x01\0\0\x01\0\x01\0\0"); + bytes.extend_from_slice(&[0xFF, 0xC0, 0x00, 0x11, 0x08]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&[0x03, 0x01, 0x22, 0x00]); + bytes +} + +#[test] +fn png_asset_is_plain_base64_and_reads_its_size() { + let asset = ImageAsset::png(&png_header(400, 300)); + assert_eq!(asset.format, ImageFormat::Png); + assert!(!asset.data_base64.starts_with("data:")); + assert_eq!(asset.checked_len().unwrap(), png_header(400, 300).len()); +} + +#[test] +fn jpeg_asset_reads_its_size_past_other_segments() { + let bytes = jpeg_header(1024, 768); + assert_eq!(dimensions(ImageFormat::Jpg, &bytes).unwrap(), (1024, 768)); + assert!(ImageAsset::jpeg(&bytes).checked_len().is_ok()); +} + +#[test] +fn jpeg_reader_skips_fill_bytes_and_standalone_markers() { + let mut bytes = vec![0xFF, 0xD8, 0xFF, 0xFF, 0xD0]; + bytes.extend_from_slice(&jpeg_header(300, 300)[2..]); + assert_eq!(dimensions(ImageFormat::Jpg, &bytes).unwrap(), (300, 300)); +} + +#[test] +fn rejects_sides_outside_the_documented_range() { + for (width, height) in [(279, 400), (400, 279), (1569, 400), (400, 1569)] { + let error = ImageAsset::png(&png_header(width, height)) + .checked_len() + .unwrap_err(); + assert!(error.to_string().contains("280 to 1568"), "{error}"); + } + assert!(ImageAsset::png(&png_header(280, 1568)).checked_len().is_ok()); +} + +#[test] +fn rejects_bytes_that_do_not_match_the_declared_format() { + let error = ImageAsset::png(&jpeg_header(400, 400)) + .checked_len() + .unwrap_err(); + assert!(error.to_string().contains("not a readable PNG")); + let error = ImageAsset::jpeg(&png_header(400, 400)) + .checked_len() + .unwrap_err(); + assert!(error.to_string().contains("not a readable JPEG")); + assert!(dimensions(ImageFormat::Jpg, &[0xFF, 0xD8, 0x00]).is_err()); + assert!(dimensions(ImageFormat::Jpg, &[0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x01]).is_err()); + assert!(dimensions(ImageFormat::Png, &PNG_SIGNATURE[..4]).is_err()); +} + +#[test] +fn rejects_invalid_base64_and_oversized_files() { + let asset = ImageAsset { + format: ImageFormat::Png, + data_base64: "data:image/png;base64,AAAA".into(), + }; + assert!(asset.checked_len().unwrap_err().to_string().contains("base64")); + + let mut huge = png_header(400, 400); + huge.resize(MAX_IMAGE_BYTES + 1, 0); + let error = ImageAsset::png(&huge).checked_len().unwrap_err(); + assert!(error.to_string().contains("16 MiB")); +} diff --git a/crates/tinyinference-decisions/src/request/mod.rs b/crates/tinyinference-decisions/src/request/mod.rs index a8989be2..9bc756a5 100644 --- a/crates/tinyinference-decisions/src/request/mod.rs +++ b/crates/tinyinference-decisions/src/request/mod.rs @@ -4,9 +4,13 @@ #[path = "mod_tests.rs"] mod test; +mod image; mod types; -pub use types::{Choice, EvaluationRequest, Noul, NoulCriteria, Question, Score}; +pub use types::{ + Asset, BoxQuestion, Choice, EvaluationRequest, ImageAsset, ImageFormat, Noul, NoulCriteria, + Question, Score, +}; use crate::{Error, Result}; @@ -40,6 +44,41 @@ impl EvaluationRequest { } question.validate()?; } + self.validate_assets() + } + + fn validate_assets(&self) -> Result<()> { + if self.assets.is_empty() { + return Ok(()); + } + // Every reference must appear somewhere the model reads, or the + // provider rejects the request; the wire text of state and questions + // is exactly where it looks. + let readable = serde_json::to_string(&(&self.state, &self.questions)) + .map_err(|_| Error::invalid_request("state and questions must serialise"))?; + let mut total = 0_usize; + for (key, asset) in &self.assets { + if key.trim().is_empty() { + return Err(Error::invalid_request("asset references must not be empty")); + } + if key.len() > image::MAX_REFERENCE_BYTES { + return Err(Error::invalid_request( + "asset references must be at most 1024 bytes", + )); + } + if !readable.contains(key.as_str()) { + return Err(Error::invalid_request(format!( + "asset `{key}` is not referenced by the state or any question" + ))); + } + let Asset::Image(image) = asset; + total = total.saturating_add(image.checked_len()?); + } + if total > image::MAX_TOTAL_IMAGE_BYTES { + return Err(Error::invalid_request( + "images must total at most 32 MiB per request", + )); + } Ok(()) } } @@ -50,6 +89,7 @@ impl Question { Self::Choice(question) => question.validate(), Self::Score(question) => question.validate(), Self::Noul(question) => question.validate(), + Self::Box(question) => validate_instructions(&question.instructions), } } } diff --git a/crates/tinyinference-decisions/src/request/mod_tests.rs b/crates/tinyinference-decisions/src/request/mod_tests.rs index 95eb027c..55e9fd11 100644 --- a/crates/tinyinference-decisions/src/request/mod_tests.rs +++ b/crates/tinyinference-decisions/src/request/mod_tests.rs @@ -257,3 +257,115 @@ fn rejects_blank_score_and_noul_descriptions() { ); assert!(noul.validate().is_err()); } + +fn image_request(state: serde_json::Value) -> EvaluationRequest { + EvaluationRequest::new( + "jev-preview", + state, + BTreeMap::from([( + "found".to_owned(), + Question::Noul(Noul { + instructions: json!("Does `photo` show a bus?"), + criteria: None, + }), + )]), + ) + .with_image( + "a3f1-photo", + ImageAsset::png(&super::image::tests::png_header(400, 300)), + ) +} + +#[test] +fn a_text_request_has_no_assets_on_the_wire() { + let request = EvaluationRequest::jev(json!({"message": "review this"}), questions()); + let wire = serde_json::to_value(&request).unwrap(); + assert!(wire.get("assets").is_none()); + assert!(!request.needs_image_opt_in()); + assert_eq!( + serde_json::from_value::(wire).unwrap(), + request + ); +} + +#[test] +fn an_image_request_pins_the_documented_asset_shape() { + let request = image_request(json!({"photo": "a3f1-photo"})); + request.validate().unwrap(); + assert!(request.needs_image_opt_in()); + let wire = serde_json::to_value(&request).unwrap(); + let asset = &wire["assets"]["a3f1-photo"]; + assert_eq!(asset["type"], "image"); + assert_eq!(asset["format"], "png"); + assert!(asset["data_base64"].as_str().unwrap().starts_with("iVBORw0KGgo")); + assert_eq!( + serde_json::from_value::(wire).unwrap(), + request + ); + let jpeg: ImageFormat = serde_json::from_value(json!("jpeg")).unwrap(); + assert_eq!(jpeg, ImageFormat::Jpg); + assert_eq!(serde_json::to_value(jpeg).unwrap(), json!("jpg")); +} + +#[test] +fn an_unreferenced_asset_is_rejected() { + let error = image_request(json!({"photo": "something else"})) + .validate() + .unwrap_err(); + assert!(error.to_string().contains("not referenced"), "{error}"); +} + +#[test] +fn asset_references_must_be_nonempty_and_short() { + let mut request = image_request(json!({"photo": "a3f1-photo"})); + let image = request.assets.remove("a3f1-photo").unwrap(); + request.assets.insert(" ".into(), image.clone()); + assert!(request.validate().unwrap_err().to_string().contains("must not be empty")); + + let long = "k".repeat(1025); + let mut request = image_request(json!({ "photo": long.clone() })); + request.assets.clear(); + request.assets.insert(long, image); + assert!(request.validate().unwrap_err().to_string().contains("1024 bytes")); +} + +#[test] +fn images_may_not_total_more_than_the_request_limit() { + let mut big = super::image::tests::png_header(400, 400); + big.resize(12 * 1024 * 1024, 0); + let mut request = image_request(json!({"a": "img-a", "b": "img-b", "c": "img-c"})); + request.assets.clear(); + for key in ["img-a", "img-b", "img-c"] { + request = request.with_image(key, ImageAsset::png(&big)); + } + let error = request.validate().unwrap_err(); + assert!(error.to_string().contains("32 MiB"), "{error}"); +} + +#[test] +fn a_box_question_needs_the_opt_in_and_nonempty_instructions() { + let request = EvaluationRequest::new( + "jev-preview", + json!({"page": "p1"}), + BTreeMap::from([( + "button".to_owned(), + Question::Box(BoxQuestion { + instructions: json!("the verify button in `page`"), + }), + )]), + ); + assert!(request.needs_image_opt_in()); + request.validate().unwrap(); + assert_eq!( + serde_json::to_value(&request).unwrap()["questions"]["button"], + json!({"type": "box", "instructions": "the verify button in `page`"}) + ); + let mut empty = request; + empty.questions.insert( + "button".into(), + Question::Box(BoxQuestion { + instructions: json!(" "), + }), + ); + assert!(empty.validate().is_err()); +} diff --git a/crates/tinyinference-decisions/src/response/mod.rs b/crates/tinyinference-decisions/src/response/mod.rs index e04fc53e..903d0e07 100644 --- a/crates/tinyinference-decisions/src/response/mod.rs +++ b/crates/tinyinference-decisions/src/response/mod.rs @@ -6,7 +6,9 @@ mod test; mod types; -pub use types::{Answer, ChoiceAnswer, EvaluationResponse, NoulAnswer, ScoreAnswer, Usage}; +pub use types::{ + Answer, BoxAnswer, BoxEdges, ChoiceAnswer, EvaluationResponse, NoulAnswer, ScoreAnswer, Usage, +}; use std::collections::BTreeSet; @@ -158,6 +160,18 @@ fn validate_pair(question: &Question, answer: &Answer) -> Result<()> { (Question::Noul(_), Answer::Noul(answer)) => { validate_probability(answer.noul, "noul")?; } + (Question::Box(_), Answer::Box(answer)) => { + validate_probability(answer.confidence, "box confidence")?; + let edges = answer.edges; + for edge in [edges.top, edges.bottom, edges.left, edges.right] { + validate_probability(edge, "box edge")?; + } + if edges.left > edges.right || edges.top > edges.bottom { + return Err(Error::invalid_response( + "box edges must satisfy left <= right and top <= bottom", + )); + } + } _ => { return Err(Error::invalid_response( "answer type must match its request question type", diff --git a/crates/tinyinference-decisions/src/response/types.rs b/crates/tinyinference-decisions/src/response/types.rs index 8f13963c..ea7f2f8c 100644 --- a/crates/tinyinference-decisions/src/response/types.rs +++ b/crates/tinyinference-decisions/src/response/types.rs @@ -78,6 +78,44 @@ pub enum Answer { Score(ScoreAnswer), /// Probability of yes. Noul(NoulAnswer), + /// Where the asked-for thing is in the image. + Box(BoxAnswer), +} + +/// Answer to a Box question. +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)] +pub struct BoxAnswer { + /// The box, as fractions of the image's width and height. + #[serde(rename = "box")] + pub edges: BoxEdges, + /// Confidence in the box, from zero to one. + pub confidence: f64, +} + +/// The edges of a box, each a fraction (0 to 1) of the image measured from its +/// top-left corner: `top` and `bottom` of its height, `left` and `right` of +/// its width. +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)] +pub struct BoxEdges { + /// Top edge, a fraction of the height. + pub top: f64, + /// Bottom edge, a fraction of the height. + pub bottom: f64, + /// Left edge, a fraction of the width. + pub left: f64, + /// Right edge, a fraction of the width. + pub right: f64, +} + +impl BoxEdges { + /// The centre of the box as `(x, y)` fractions of width and height. + #[must_use] + pub fn centre(&self) -> (f64, f64) { + ( + f64::midpoint(self.left, self.right), + f64::midpoint(self.top, self.bottom), + ) + } } /// Answer to a Choice question. From c466d81b6d5643019610b2aeae373c8acc0383a7 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 14 Nov 2026 11:54:02 +0000 Subject: [PATCH 3/6] test(request): cover image opt-in header and asset serialization Adds a client test asserting the image opt-in header and assets payload appear only when an image is attached, and exposes the image module to the crate so tests can build PNG fixtures. Also reformats a few existing assertions. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/client/mod_tests.rs | 22 ++++++ .../src/request/image.rs | 5 +- .../src/request/image_tests.rs | 14 +++- .../src/request/mod.rs | 2 +- .../src/request/mod_tests.rs | 23 +++++- .../src/response/mod_tests.rs | 72 +++++++++++++++++++ 6 files changed, 131 insertions(+), 7 deletions(-) diff --git a/crates/tinyinference-decisions/src/client/mod_tests.rs b/crates/tinyinference-decisions/src/client/mod_tests.rs index 43dfe67f..2d984e82 100644 --- a/crates/tinyinference-decisions/src/client/mod_tests.rs +++ b/crates/tinyinference-decisions/src/client/mod_tests.rs @@ -163,6 +163,28 @@ async fn sends_the_documented_endpoint_and_bearer_header() { assert!(requests[0].body.contains("\"model\":\"jev-latest\"")); } +#[tokio::test] +async fn sends_the_image_opt_in_header_only_with_an_image() { + let (client, requests) = mock_client( + config("http://127.0.0.1:1".into()), + vec![response(200, &success(), ""), response(200, &success(), "")], + ); + client.evaluate(&request()).await.unwrap(); + let image = crate::request::image::tests::png_header(400, 300); + let mut with_image = request(); + with_image.state = json!({"message": "review this", "photo": "img-1"}); + let with_image = with_image.with_image("img-1", crate::ImageAsset::png(&image)); + client.evaluate(&with_image).await.unwrap(); + let requests = requests.lock().await; + assert!(requests[0].headers.get(IMAGE_OPT_IN_HEADER).is_none()); + assert!(!requests[0].body.contains("assets")); + assert_eq!( + requests[1].headers.get(IMAGE_OPT_IN_HEADER).unwrap(), + "true" + ); + assert!(requests[1].body.contains("\"assets\":{\"img-1\"")); +} + #[tokio::test] async fn an_exact_endpoint_override_is_not_extended_with_a_provider_path() { let endpoint = "https://example.com/api/alpha/decisions"; diff --git a/crates/tinyinference-decisions/src/request/image.rs b/crates/tinyinference-decisions/src/request/image.rs index d7726a21..4919b065 100644 --- a/crates/tinyinference-decisions/src/request/image.rs +++ b/crates/tinyinference-decisions/src/request/image.rs @@ -99,7 +99,10 @@ fn jpeg_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { at += 2; continue; } - let length = usize::from(u16::from_be_bytes([*bytes.get(at + 2)?, *bytes.get(at + 3)?])); + let length = usize::from(u16::from_be_bytes([ + *bytes.get(at + 2)?, + *bytes.get(at + 3)?, + ])); // Start-of-frame markers carry the size; C4, C8 and CC do not. if matches!(marker, 0xC0..=0xCF) && !matches!(marker, 0xC4 | 0xC8 | 0xCC) { let height = u16::from_be_bytes([*bytes.get(at + 5)?, *bytes.get(at + 6)?]); diff --git a/crates/tinyinference-decisions/src/request/image_tests.rs b/crates/tinyinference-decisions/src/request/image_tests.rs index da8f1286..872c4f1a 100644 --- a/crates/tinyinference-decisions/src/request/image_tests.rs +++ b/crates/tinyinference-decisions/src/request/image_tests.rs @@ -55,7 +55,11 @@ fn rejects_sides_outside_the_documented_range() { .unwrap_err(); assert!(error.to_string().contains("280 to 1568"), "{error}"); } - assert!(ImageAsset::png(&png_header(280, 1568)).checked_len().is_ok()); + assert!( + ImageAsset::png(&png_header(280, 1568)) + .checked_len() + .is_ok() + ); } #[test] @@ -79,7 +83,13 @@ fn rejects_invalid_base64_and_oversized_files() { format: ImageFormat::Png, data_base64: "data:image/png;base64,AAAA".into(), }; - assert!(asset.checked_len().unwrap_err().to_string().contains("base64")); + assert!( + asset + .checked_len() + .unwrap_err() + .to_string() + .contains("base64") + ); let mut huge = png_header(400, 400); huge.resize(MAX_IMAGE_BYTES + 1, 0); diff --git a/crates/tinyinference-decisions/src/request/mod.rs b/crates/tinyinference-decisions/src/request/mod.rs index 9bc756a5..22a63ead 100644 --- a/crates/tinyinference-decisions/src/request/mod.rs +++ b/crates/tinyinference-decisions/src/request/mod.rs @@ -4,7 +4,7 @@ #[path = "mod_tests.rs"] mod test; -mod image; +pub(crate) mod image; mod types; pub use types::{ diff --git a/crates/tinyinference-decisions/src/request/mod_tests.rs b/crates/tinyinference-decisions/src/request/mod_tests.rs index 55e9fd11..5b857edf 100644 --- a/crates/tinyinference-decisions/src/request/mod_tests.rs +++ b/crates/tinyinference-decisions/src/request/mod_tests.rs @@ -297,7 +297,12 @@ fn an_image_request_pins_the_documented_asset_shape() { let asset = &wire["assets"]["a3f1-photo"]; assert_eq!(asset["type"], "image"); assert_eq!(asset["format"], "png"); - assert!(asset["data_base64"].as_str().unwrap().starts_with("iVBORw0KGgo")); + assert!( + asset["data_base64"] + .as_str() + .unwrap() + .starts_with("iVBORw0KGgo") + ); assert_eq!( serde_json::from_value::(wire).unwrap(), request @@ -320,13 +325,25 @@ fn asset_references_must_be_nonempty_and_short() { let mut request = image_request(json!({"photo": "a3f1-photo"})); let image = request.assets.remove("a3f1-photo").unwrap(); request.assets.insert(" ".into(), image.clone()); - assert!(request.validate().unwrap_err().to_string().contains("must not be empty")); + assert!( + request + .validate() + .unwrap_err() + .to_string() + .contains("must not be empty") + ); let long = "k".repeat(1025); let mut request = image_request(json!({ "photo": long.clone() })); request.assets.clear(); request.assets.insert(long, image); - assert!(request.validate().unwrap_err().to_string().contains("1024 bytes")); + assert!( + request + .validate() + .unwrap_err() + .to_string() + .contains("1024 bytes") + ); } #[test] diff --git a/crates/tinyinference-decisions/src/response/mod_tests.rs b/crates/tinyinference-decisions/src/response/mod_tests.rs index e40d965a..54134f83 100644 --- a/crates/tinyinference-decisions/src/response/mod_tests.rs +++ b/crates/tinyinference-decisions/src/response/mod_tests.rs @@ -376,3 +376,75 @@ fn response_answer_types_round_trip_with_optional_usage() { without_usage ); } + +fn box_request() -> EvaluationRequest { + EvaluationRequest::new( + "jev-preview", + json!({"page": "p1"}), + BTreeMap::from([( + "button".to_owned(), + Question::Box(crate::BoxQuestion { + instructions: json!("the verify button in `page`"), + }), + )]), + ) +} + +fn box_response(edges: serde_json::Value, confidence: f64) -> EvaluationResponse { + serde_json::from_value(json!({ + "model": "jev-preview", + "answers": {"button": {"type": "box", "box": edges, "confidence": confidence}}, + "usage": {"input_tokens": 900, "output_tokens": 0} + })) + .unwrap() +} + +#[test] +fn a_box_answer_decodes_and_reports_its_centre() { + let response = box_response( + json!({"top": 0.8, "bottom": 0.9, "left": 0.7, "right": 0.9}), + 0.93, + ); + response.validate_for(&box_request()).unwrap(); + let Answer::Box(answer) = &response.answers["button"] else { + panic!("expected a box answer"); + }; + let (x, y) = answer.edges.centre(); + assert!((x - 0.8).abs() < 1e-9 && (y - 0.85).abs() < 1e-9); +} + +#[test] +fn a_box_answer_with_inverted_or_out_of_range_edges_is_rejected() { + let inverted = box_response( + json!({"top": 0.9, "bottom": 0.1, "left": 0.1, "right": 0.2}), + 0.5, + ); + assert!(inverted.validate_for(&box_request()).is_err()); + let outside = box_response( + json!({"top": 0.1, "bottom": 1.2, "left": 0.1, "right": 0.2}), + 0.5, + ); + assert!(outside.validate_for(&box_request()).is_err()); + let unsure = box_response( + json!({"top": 0.1, "bottom": 0.2, "left": 0.1, "right": 0.2}), + 1.5, + ); + assert!(unsure.validate_for(&box_request()).is_err()); +} + +#[test] +fn a_box_answer_to_a_noul_question_is_a_type_mismatch() { + let mut request = box_request(); + request.questions.insert( + "button".into(), + Question::Noul(Noul { + instructions: json!("Is there a button?"), + criteria: None, + }), + ); + let response = box_response( + json!({"top": 0.1, "bottom": 0.2, "left": 0.1, "right": 0.2}), + 0.5, + ); + assert!(response.validate_for(&request).is_err()); +} From 8a16bfdf4a1407a267585289f6782d77694ffdd6 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 14 Nov 2026 11:54:18 +0000 Subject: [PATCH 4/6] docs(tinyinference-decisions): document image attachments and box questions Adds a guide section covering image attachments and Box questions, including a usage example, the danger-zone header the client sends, and the validation limits applied before transport. The request module README now also mentions the image asset builder and the renamed test file. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/request/README.md | 4 +- docs/tinyinference-decisions.md | 42 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/crates/tinyinference-decisions/src/request/README.md b/crates/tinyinference-decisions/src/request/README.md index 433c4a83..f2d5df27 100644 --- a/crates/tinyinference-decisions/src/request/README.md +++ b/crates/tinyinference-decisions/src/request/README.md @@ -2,4 +2,6 @@ This module defines the exact state-and-questions payload sent to TypeSafe's System One endpoint. `types.rs` holds the serde wire types, `mod.rs` validates -the documented primitive bounds before transport, and `test.rs` pins both. +the documented primitive bounds before transport, and `mod_tests.rs` pins both. +`image.rs` builds inline image assets and checks them against the provider's +image limits (format, side lengths, sizes) by reading only the file header. diff --git a/docs/tinyinference-decisions.md b/docs/tinyinference-decisions.md index a2eff416..ea9a3973 100644 --- a/docs/tinyinference-decisions.md +++ b/docs/tinyinference-decisions.md @@ -38,6 +38,48 @@ TYPESAFE_API_KEY='' cargo run -p tinyinference-decisions --example basic The crate is part of the TinyInference workspace and is licensed GPL-3.0-only. +## Images and Box questions + +Jev 1.13 and later read images, and Jev 1.14 (`jev-preview`) adds the Box +question, which returns where one thing is in an image. Attach an image with +`EvaluationRequest::with_image(key, ImageAsset::png(&bytes))` and put `key` +somewhere in `state` (for example `{"page": key}`), so the model knows where it +belongs. Questions point at the state field in backticks. + +```rust,no_run +use std::collections::BTreeMap; +use serde_json::json; +use tinyinference_decisions::{ + Answer, BoxQuestion, Client, EvaluationRequest, ImageAsset, Question, +}; + +# async fn run(png: Vec) -> Result<(), Box> { +let request = EvaluationRequest::new( + "jev-preview", + json!({"page": "shot-1"}), + BTreeMap::from([( + "button".to_owned(), + Question::Box(BoxQuestion { instructions: json!("the Verify button in `page`") }), + )]), +) +.with_image("shot-1", ImageAsset::png(&png)); +let result = Client::from_env()?.evaluate(&request).await?; +if let Some(Answer::Box(answer)) = result.response.answers.get("button") { + let (x, y) = answer.edges.centre(); // fractions of width and height + println!("{x} {y} at confidence {}", answer.confidence); +} +# Ok(()) +# } +``` + +The client sends `TypeSafe-Danger-Zone-Images: true` on any request with an +image or a Box question, as the provider requires. `validate` checks the +documented limits before anything is sent: PNG or JPEG whose header matches the +declared format, 280 to 1568 pixels per side, 16 MiB per image and 32 MiB in +all, and every asset key referenced by the state or a question. Images are +neither resized nor rotated; that stays with the caller. A Box answer is always +one box, so ask a Noul alongside it when the thing may be absent. + ## Local decision models Use `ClientConfig::self_hosted` for a locally served OpenJEV, Decider, Surogate From 8dcf9ea07e8ce5adc0edfb78ff7b0766e2a86f14 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 14 Nov 2026 14:33:41 +0000 Subject: [PATCH 5/6] fix(tinyinference-decisions): match asset references against decoded json values Asset reference validation now walks the decoded JSON values and object keys instead of searching the escaped serialized text, so keys containing backslashes, quotes or control characters are matched correctly. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../src/request/mod.rs | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/crates/tinyinference-decisions/src/request/mod.rs b/crates/tinyinference-decisions/src/request/mod.rs index 22a63ead..4d42fc80 100644 --- a/crates/tinyinference-decisions/src/request/mod.rs +++ b/crates/tinyinference-decisions/src/request/mod.rs @@ -53,8 +53,10 @@ impl EvaluationRequest { } // Every reference must appear somewhere the model reads, or the // provider rejects the request; the wire text of state and questions - // is exactly where it looks. - let readable = serde_json::to_string(&(&self.state, &self.questions)) + // is exactly where it looks. Search the decoded string values and + // object keys, not the escaped JSON text, so keys containing `\`, `"` + // or control characters match. + let readable = serde_json::to_value((&self.state, &self.questions)) .map_err(|_| Error::invalid_request("state and questions must serialise"))?; let mut total = 0_usize; for (key, asset) in &self.assets { @@ -66,7 +68,7 @@ impl EvaluationRequest { "asset references must be at most 1024 bytes", )); } - if !readable.contains(key.as_str()) { + if !value_mentions(&readable, key) { return Err(Error::invalid_request(format!( "asset `{key}` is not referenced by the state or any question" ))); @@ -83,6 +85,18 @@ impl EvaluationRequest { } } +/// Whether any decoded string value or object key inside `value` contains `needle`. +fn value_mentions(value: &serde_json::Value, needle: &str) -> bool { + match value { + serde_json::Value::String(text) => text.contains(needle), + serde_json::Value::Array(items) => items.iter().any(|item| value_mentions(item, needle)), + serde_json::Value::Object(map) => map + .iter() + .any(|(key, item)| key.contains(needle) || value_mentions(item, needle)), + _ => false, + } +} + impl Question { fn validate(&self) -> Result<()> { match self { From 67b2797794235c4abd10acc94f2df59873b14fbe Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 14 Nov 2026 14:33:50 +0000 Subject: [PATCH 6/6] test(request): cover asset references containing json escapes Add a test that rekeys an asset to paths containing backslashes, quotes and tabs, confirming validation matches against the decoded state rather than the raw JSON text. Auto-committed-on: cuttlefish Co-authored-by: Medulla --- .../tinyinference-decisions/src/request/mod_tests.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/tinyinference-decisions/src/request/mod_tests.rs b/crates/tinyinference-decisions/src/request/mod_tests.rs index 5b857edf..974cc699 100644 --- a/crates/tinyinference-decisions/src/request/mod_tests.rs +++ b/crates/tinyinference-decisions/src/request/mod_tests.rs @@ -320,6 +320,16 @@ fn an_unreferenced_asset_is_rejected() { assert!(error.to_string().contains("not referenced"), "{error}"); } +#[test] +fn asset_references_with_json_escapes_match_decoded_state() { + for key in ["C:\\shots\\a.png", "say \"cheese\".png", "tab\there.png"] { + let mut request = image_request(json!({ "photo": key })); + let image = request.assets.remove("a3f1-photo").unwrap(); + request.assets.insert(key.to_owned(), image); + request.validate().unwrap(); + } +} + #[test] fn asset_references_must_be_nonempty_and_short() { let mut request = image_request(json!({"photo": "a3f1-photo"}));