diff --git a/Cargo.lock b/Cargo.lock index 7c5124e9..c06e9e9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1700,6 +1700,7 @@ version = "0.5.0" dependencies = [ "async-trait", "axum", + "base64 0.23.1", "httpdate", "reqwest", "serde", diff --git a/crates/tinyinference-decisions/Cargo.toml b/crates/tinyinference-decisions/Cargo.toml index 5d01f398..dc37d71b 100644 --- a/crates/tinyinference-decisions/Cargo.toml +++ b/crates/tinyinference-decisions/Cargo.toml @@ -13,6 +13,8 @@ categories = ["api-bindings", "asynchronous"] [dependencies] async-trait = { workspace = true } +# Encodes attached images for the System One `assets` map. +base64 = "0.23" httpdate = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } diff --git a/crates/tinyinference-decisions/src/client/mod.rs b/crates/tinyinference-decisions/src/client/mod.rs index 95d319e5..67a17142 100644 --- a/crates/tinyinference-decisions/src/client/mod.rs +++ b/crates/tinyinference-decisions/src/client/mod.rs @@ -16,6 +16,9 @@ use crate::{Error, EvaluationRequest, EvaluationResponse, Result}; const MAX_RETRIES: u32 = 100; const MAX_RESPONSE_BYTES: usize = 16 * 1024 * 1024; +/// Header the provider requires on every request that carries an image or a +/// Box question; without it such a request is rejected with 400. +const IMAGE_OPT_IN_HEADER: &str = "TypeSafe-Danger-Zone-Images"; impl Client { /// Construct a client from an explicit configuration. @@ -143,6 +146,9 @@ impl Client { ) }); let mut builder = self.http.post(&url).json(request); + if request.needs_image_opt_in() { + builder = builder.header(IMAGE_OPT_IN_HEADER, "true"); + } let api_key = self.config.api_key.expose(); if !(self.config.provider == Provider::SelfHosted && api_key.trim().is_empty()) { builder = builder.bearer_auth(api_key); diff --git a/crates/tinyinference-decisions/src/client/mod_tests.rs b/crates/tinyinference-decisions/src/client/mod_tests.rs index 43dfe67f..2d984e82 100644 --- a/crates/tinyinference-decisions/src/client/mod_tests.rs +++ b/crates/tinyinference-decisions/src/client/mod_tests.rs @@ -163,6 +163,28 @@ async fn sends_the_documented_endpoint_and_bearer_header() { assert!(requests[0].body.contains("\"model\":\"jev-latest\"")); } +#[tokio::test] +async fn sends_the_image_opt_in_header_only_with_an_image() { + let (client, requests) = mock_client( + config("http://127.0.0.1:1".into()), + vec![response(200, &success(), ""), response(200, &success(), "")], + ); + client.evaluate(&request()).await.unwrap(); + let image = crate::request::image::tests::png_header(400, 300); + let mut with_image = request(); + with_image.state = json!({"message": "review this", "photo": "img-1"}); + let with_image = with_image.with_image("img-1", crate::ImageAsset::png(&image)); + client.evaluate(&with_image).await.unwrap(); + let requests = requests.lock().await; + assert!(requests[0].headers.get(IMAGE_OPT_IN_HEADER).is_none()); + assert!(!requests[0].body.contains("assets")); + assert_eq!( + requests[1].headers.get(IMAGE_OPT_IN_HEADER).unwrap(), + "true" + ); + assert!(requests[1].body.contains("\"assets\":{\"img-1\"")); +} + #[tokio::test] async fn an_exact_endpoint_override_is_not_extended_with_a_provider_path() { let endpoint = "https://example.com/api/alpha/decisions"; diff --git a/crates/tinyinference-decisions/src/lib.rs b/crates/tinyinference-decisions/src/lib.rs index 1d24c74b..939c6a93 100644 --- a/crates/tinyinference-decisions/src/lib.rs +++ b/crates/tinyinference-decisions/src/lib.rs @@ -54,5 +54,10 @@ pub use client::{ Client, ClientConfig, EvaluationFailure, EvaluationResult, Provider, RetryPolicy, }; pub use error::{Error, Result}; -pub use request::{Choice, EvaluationRequest, Noul, NoulCriteria, Question, Score}; -pub use response::{Answer, ChoiceAnswer, EvaluationResponse, NoulAnswer, ScoreAnswer, Usage}; +pub use request::{ + Asset, BoxQuestion, Choice, EvaluationRequest, ImageAsset, ImageFormat, Noul, NoulCriteria, + Question, Score, +}; +pub use response::{ + Answer, BoxAnswer, BoxEdges, ChoiceAnswer, EvaluationResponse, NoulAnswer, ScoreAnswer, Usage, +}; diff --git a/crates/tinyinference-decisions/src/request/README.md b/crates/tinyinference-decisions/src/request/README.md index 433c4a83..f2d5df27 100644 --- a/crates/tinyinference-decisions/src/request/README.md +++ b/crates/tinyinference-decisions/src/request/README.md @@ -2,4 +2,6 @@ This module defines the exact state-and-questions payload sent to TypeSafe's System One endpoint. `types.rs` holds the serde wire types, `mod.rs` validates -the documented primitive bounds before transport, and `test.rs` pins both. +the documented primitive bounds before transport, and `mod_tests.rs` pins both. +`image.rs` builds inline image assets and checks them against the provider's +image limits (format, side lengths, sizes) by reading only the file header. diff --git a/crates/tinyinference-decisions/src/request/image.rs b/crates/tinyinference-decisions/src/request/image.rs new file mode 100644 index 00000000..4919b065 --- /dev/null +++ b/crates/tinyinference-decisions/src/request/image.rs @@ -0,0 +1,121 @@ +//! Building inline image assets and checking them against the provider's +//! documented image limits before anything is sent. + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; + +use super::types::{ImageAsset, ImageFormat}; +use crate::{Error, Result}; + +/// Smallest accepted width or height, in pixels. +pub(crate) const MIN_SIDE: u32 = 280; +/// Largest accepted width or height, in pixels. +pub(crate) const MAX_SIDE: u32 = 1568; +/// Largest accepted image file. +pub(crate) const MAX_IMAGE_BYTES: usize = 16 * 1024 * 1024; +/// Largest accepted total of all image files in one request. +pub(crate) const MAX_TOTAL_IMAGE_BYTES: usize = 32 * 1024 * 1024; +/// Longest accepted asset reference string, in bytes. +pub(crate) const MAX_REFERENCE_BYTES: usize = 1024; + +const PNG_SIGNATURE: &[u8] = b"\x89PNG\r\n\x1a\n"; + +impl ImageAsset { + /// An image from the bytes of a PNG file. + #[must_use] + pub fn png(bytes: &[u8]) -> Self { + Self { + format: ImageFormat::Png, + data_base64: STANDARD.encode(bytes), + } + } + + /// An image from the bytes of a JPEG file. + #[must_use] + pub fn jpeg(bytes: &[u8]) -> Self { + Self { + format: ImageFormat::Jpg, + data_base64: STANDARD.encode(bytes), + } + } + + /// Decode the image and check it against the provider's limits, returning + /// its size in bytes. + pub(crate) fn checked_len(&self) -> Result { + let bytes = STANDARD + .decode(&self.data_base64) + .map_err(|_| Error::invalid_request("image data_base64 must be padded base64"))?; + if bytes.len() > MAX_IMAGE_BYTES { + return Err(Error::invalid_request("image must be at most 16 MiB")); + } + let (width, height) = dimensions(self.format, &bytes)?; + if !(MIN_SIDE..=MAX_SIDE).contains(&width) || !(MIN_SIDE..=MAX_SIDE).contains(&height) { + return Err(Error::invalid_request(format!( + "image sides must be {MIN_SIDE} to {MAX_SIDE} pixels, got {width}x{height}" + ))); + } + Ok(bytes.len()) + } +} + +/// Width and height read from the image header. +pub(crate) fn dimensions(format: ImageFormat, bytes: &[u8]) -> Result<(u32, u32)> { + match format { + ImageFormat::Png => png_dimensions(bytes), + ImageFormat::Jpg => jpeg_dimensions(bytes), + } + .ok_or_else(|| match format { + ImageFormat::Png => Error::invalid_request("image is not a readable PNG"), + ImageFormat::Jpg => Error::invalid_request("image is not a readable JPEG"), + }) +} + +fn png_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { + // The signature is followed by the IHDR chunk: length, "IHDR", width, height. + if !bytes.starts_with(PNG_SIGNATURE) || bytes.get(12..16)? != b"IHDR" { + return None; + } + let width = u32::from_be_bytes(bytes.get(16..20)?.try_into().ok()?); + let height = u32::from_be_bytes(bytes.get(20..24)?.try_into().ok()?); + Some((width, height)) +} + +fn jpeg_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { + if !bytes.starts_with(&[0xFF, 0xD8]) { + return None; + } + let mut at = 2; + loop { + // Skip fill bytes before a marker. + while *bytes.get(at)? == 0xFF && *bytes.get(at + 1)? == 0xFF { + at += 1; + } + if *bytes.get(at)? != 0xFF { + return None; + } + let marker = *bytes.get(at + 1)?; + // Markers without a length field. + if marker == 0x01 || (0xD0..=0xD7).contains(&marker) { + at += 2; + continue; + } + let length = usize::from(u16::from_be_bytes([ + *bytes.get(at + 2)?, + *bytes.get(at + 3)?, + ])); + // Start-of-frame markers carry the size; C4, C8 and CC do not. + if matches!(marker, 0xC0..=0xCF) && !matches!(marker, 0xC4 | 0xC8 | 0xCC) { + let height = u16::from_be_bytes([*bytes.get(at + 5)?, *bytes.get(at + 6)?]); + let width = u16::from_be_bytes([*bytes.get(at + 7)?, *bytes.get(at + 8)?]); + return Some((u32::from(width), u32::from(height))); + } + if length < 2 { + return None; + } + at += 2 + length; + } +} + +#[cfg(test)] +#[path = "image_tests.rs"] +pub(crate) mod tests; diff --git a/crates/tinyinference-decisions/src/request/image_tests.rs b/crates/tinyinference-decisions/src/request/image_tests.rs new file mode 100644 index 00000000..872c4f1a --- /dev/null +++ b/crates/tinyinference-decisions/src/request/image_tests.rs @@ -0,0 +1,98 @@ +//! Image asset construction, header reading, and limit checks. + +use super::*; + +/// The first bytes of a PNG file of the given size: enough for the header. +pub(crate) fn png_header(width: u32, height: u32) -> Vec { + let mut bytes = PNG_SIGNATURE.to_vec(); + bytes.extend_from_slice(&13_u32.to_be_bytes()); + bytes.extend_from_slice(b"IHDR"); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&[8, 6, 0, 0, 0]); + bytes +} + +/// The first bytes of a baseline JPEG of the given size, with an APP0 segment +/// before the frame so the reader has to walk a segment. +fn jpeg_header(width: u16, height: u16) -> Vec { + let mut bytes = vec![0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10]; + bytes.extend_from_slice(b"JFIF\0\x01\x01\0\0\x01\0\x01\0\0"); + bytes.extend_from_slice(&[0xFF, 0xC0, 0x00, 0x11, 0x08]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&[0x03, 0x01, 0x22, 0x00]); + bytes +} + +#[test] +fn png_asset_is_plain_base64_and_reads_its_size() { + let asset = ImageAsset::png(&png_header(400, 300)); + assert_eq!(asset.format, ImageFormat::Png); + assert!(!asset.data_base64.starts_with("data:")); + assert_eq!(asset.checked_len().unwrap(), png_header(400, 300).len()); +} + +#[test] +fn jpeg_asset_reads_its_size_past_other_segments() { + let bytes = jpeg_header(1024, 768); + assert_eq!(dimensions(ImageFormat::Jpg, &bytes).unwrap(), (1024, 768)); + assert!(ImageAsset::jpeg(&bytes).checked_len().is_ok()); +} + +#[test] +fn jpeg_reader_skips_fill_bytes_and_standalone_markers() { + let mut bytes = vec![0xFF, 0xD8, 0xFF, 0xFF, 0xD0]; + bytes.extend_from_slice(&jpeg_header(300, 300)[2..]); + assert_eq!(dimensions(ImageFormat::Jpg, &bytes).unwrap(), (300, 300)); +} + +#[test] +fn rejects_sides_outside_the_documented_range() { + for (width, height) in [(279, 400), (400, 279), (1569, 400), (400, 1569)] { + let error = ImageAsset::png(&png_header(width, height)) + .checked_len() + .unwrap_err(); + assert!(error.to_string().contains("280 to 1568"), "{error}"); + } + assert!( + ImageAsset::png(&png_header(280, 1568)) + .checked_len() + .is_ok() + ); +} + +#[test] +fn rejects_bytes_that_do_not_match_the_declared_format() { + let error = ImageAsset::png(&jpeg_header(400, 400)) + .checked_len() + .unwrap_err(); + assert!(error.to_string().contains("not a readable PNG")); + let error = ImageAsset::jpeg(&png_header(400, 400)) + .checked_len() + .unwrap_err(); + assert!(error.to_string().contains("not a readable JPEG")); + assert!(dimensions(ImageFormat::Jpg, &[0xFF, 0xD8, 0x00]).is_err()); + assert!(dimensions(ImageFormat::Jpg, &[0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x01]).is_err()); + assert!(dimensions(ImageFormat::Png, &PNG_SIGNATURE[..4]).is_err()); +} + +#[test] +fn rejects_invalid_base64_and_oversized_files() { + let asset = ImageAsset { + format: ImageFormat::Png, + data_base64: "data:image/png;base64,AAAA".into(), + }; + assert!( + asset + .checked_len() + .unwrap_err() + .to_string() + .contains("base64") + ); + + let mut huge = png_header(400, 400); + huge.resize(MAX_IMAGE_BYTES + 1, 0); + let error = ImageAsset::png(&huge).checked_len().unwrap_err(); + assert!(error.to_string().contains("16 MiB")); +} diff --git a/crates/tinyinference-decisions/src/request/mod.rs b/crates/tinyinference-decisions/src/request/mod.rs index a8989be2..4d42fc80 100644 --- a/crates/tinyinference-decisions/src/request/mod.rs +++ b/crates/tinyinference-decisions/src/request/mod.rs @@ -4,9 +4,13 @@ #[path = "mod_tests.rs"] mod test; +pub(crate) mod image; mod types; -pub use types::{Choice, EvaluationRequest, Noul, NoulCriteria, Question, Score}; +pub use types::{ + Asset, BoxQuestion, Choice, EvaluationRequest, ImageAsset, ImageFormat, Noul, NoulCriteria, + Question, Score, +}; use crate::{Error, Result}; @@ -40,16 +44,66 @@ impl EvaluationRequest { } question.validate()?; } + self.validate_assets() + } + + fn validate_assets(&self) -> Result<()> { + if self.assets.is_empty() { + return Ok(()); + } + // Every reference must appear somewhere the model reads, or the + // provider rejects the request; the wire text of state and questions + // is exactly where it looks. Search the decoded string values and + // object keys, not the escaped JSON text, so keys containing `\`, `"` + // or control characters match. + let readable = serde_json::to_value((&self.state, &self.questions)) + .map_err(|_| Error::invalid_request("state and questions must serialise"))?; + let mut total = 0_usize; + for (key, asset) in &self.assets { + if key.trim().is_empty() { + return Err(Error::invalid_request("asset references must not be empty")); + } + if key.len() > image::MAX_REFERENCE_BYTES { + return Err(Error::invalid_request( + "asset references must be at most 1024 bytes", + )); + } + if !value_mentions(&readable, key) { + return Err(Error::invalid_request(format!( + "asset `{key}` is not referenced by the state or any question" + ))); + } + let Asset::Image(image) = asset; + total = total.saturating_add(image.checked_len()?); + } + if total > image::MAX_TOTAL_IMAGE_BYTES { + return Err(Error::invalid_request( + "images must total at most 32 MiB per request", + )); + } Ok(()) } } +/// Whether any decoded string value or object key inside `value` contains `needle`. +fn value_mentions(value: &serde_json::Value, needle: &str) -> bool { + match value { + serde_json::Value::String(text) => text.contains(needle), + serde_json::Value::Array(items) => items.iter().any(|item| value_mentions(item, needle)), + serde_json::Value::Object(map) => map + .iter() + .any(|(key, item)| key.contains(needle) || value_mentions(item, needle)), + _ => false, + } +} + impl Question { fn validate(&self) -> Result<()> { match self { Self::Choice(question) => question.validate(), Self::Score(question) => question.validate(), Self::Noul(question) => question.validate(), + Self::Box(question) => validate_instructions(&question.instructions), } } } diff --git a/crates/tinyinference-decisions/src/request/mod_tests.rs b/crates/tinyinference-decisions/src/request/mod_tests.rs index 95eb027c..974cc699 100644 --- a/crates/tinyinference-decisions/src/request/mod_tests.rs +++ b/crates/tinyinference-decisions/src/request/mod_tests.rs @@ -257,3 +257,142 @@ fn rejects_blank_score_and_noul_descriptions() { ); assert!(noul.validate().is_err()); } + +fn image_request(state: serde_json::Value) -> EvaluationRequest { + EvaluationRequest::new( + "jev-preview", + state, + BTreeMap::from([( + "found".to_owned(), + Question::Noul(Noul { + instructions: json!("Does `photo` show a bus?"), + criteria: None, + }), + )]), + ) + .with_image( + "a3f1-photo", + ImageAsset::png(&super::image::tests::png_header(400, 300)), + ) +} + +#[test] +fn a_text_request_has_no_assets_on_the_wire() { + let request = EvaluationRequest::jev(json!({"message": "review this"}), questions()); + let wire = serde_json::to_value(&request).unwrap(); + assert!(wire.get("assets").is_none()); + assert!(!request.needs_image_opt_in()); + assert_eq!( + serde_json::from_value::(wire).unwrap(), + request + ); +} + +#[test] +fn an_image_request_pins_the_documented_asset_shape() { + let request = image_request(json!({"photo": "a3f1-photo"})); + request.validate().unwrap(); + assert!(request.needs_image_opt_in()); + let wire = serde_json::to_value(&request).unwrap(); + let asset = &wire["assets"]["a3f1-photo"]; + assert_eq!(asset["type"], "image"); + assert_eq!(asset["format"], "png"); + assert!( + asset["data_base64"] + .as_str() + .unwrap() + .starts_with("iVBORw0KGgo") + ); + assert_eq!( + serde_json::from_value::(wire).unwrap(), + request + ); + let jpeg: ImageFormat = serde_json::from_value(json!("jpeg")).unwrap(); + assert_eq!(jpeg, ImageFormat::Jpg); + assert_eq!(serde_json::to_value(jpeg).unwrap(), json!("jpg")); +} + +#[test] +fn an_unreferenced_asset_is_rejected() { + let error = image_request(json!({"photo": "something else"})) + .validate() + .unwrap_err(); + assert!(error.to_string().contains("not referenced"), "{error}"); +} + +#[test] +fn asset_references_with_json_escapes_match_decoded_state() { + for key in ["C:\\shots\\a.png", "say \"cheese\".png", "tab\there.png"] { + let mut request = image_request(json!({ "photo": key })); + let image = request.assets.remove("a3f1-photo").unwrap(); + request.assets.insert(key.to_owned(), image); + request.validate().unwrap(); + } +} + +#[test] +fn asset_references_must_be_nonempty_and_short() { + let mut request = image_request(json!({"photo": "a3f1-photo"})); + let image = request.assets.remove("a3f1-photo").unwrap(); + request.assets.insert(" ".into(), image.clone()); + assert!( + request + .validate() + .unwrap_err() + .to_string() + .contains("must not be empty") + ); + + let long = "k".repeat(1025); + let mut request = image_request(json!({ "photo": long.clone() })); + request.assets.clear(); + request.assets.insert(long, image); + assert!( + request + .validate() + .unwrap_err() + .to_string() + .contains("1024 bytes") + ); +} + +#[test] +fn images_may_not_total_more_than_the_request_limit() { + let mut big = super::image::tests::png_header(400, 400); + big.resize(12 * 1024 * 1024, 0); + let mut request = image_request(json!({"a": "img-a", "b": "img-b", "c": "img-c"})); + request.assets.clear(); + for key in ["img-a", "img-b", "img-c"] { + request = request.with_image(key, ImageAsset::png(&big)); + } + let error = request.validate().unwrap_err(); + assert!(error.to_string().contains("32 MiB"), "{error}"); +} + +#[test] +fn a_box_question_needs_the_opt_in_and_nonempty_instructions() { + let request = EvaluationRequest::new( + "jev-preview", + json!({"page": "p1"}), + BTreeMap::from([( + "button".to_owned(), + Question::Box(BoxQuestion { + instructions: json!("the verify button in `page`"), + }), + )]), + ); + assert!(request.needs_image_opt_in()); + request.validate().unwrap(); + assert_eq!( + serde_json::to_value(&request).unwrap()["questions"]["button"], + json!({"type": "box", "instructions": "the verify button in `page`"}) + ); + let mut empty = request; + empty.questions.insert( + "button".into(), + Question::Box(BoxQuestion { + instructions: json!(" "), + }), + ); + assert!(empty.validate().is_err()); +} diff --git a/crates/tinyinference-decisions/src/request/types.rs b/crates/tinyinference-decisions/src/request/types.rs index a8ca1329..9a77bf5b 100644 --- a/crates/tinyinference-decisions/src/request/types.rs +++ b/crates/tinyinference-decisions/src/request/types.rs @@ -14,6 +14,11 @@ pub struct EvaluationRequest { pub model: String, /// Independently evaluated questions keyed by caller-owned ids. pub questions: BTreeMap, + /// Images keyed by the reference string that `state` or a question uses + /// to point at them. Empty for a text-only request, and then absent from + /// the wire form, so text requests serialise exactly as before. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub assets: BTreeMap, } impl EvaluationRequest { @@ -33,9 +38,34 @@ impl EvaluationRequest { state: state.into(), model: model.into(), questions, + assets: BTreeMap::new(), } } + /// Attach an image under `key`. + /// + /// `key` is the reference string: put it somewhere in `state` (for + /// example `{"photo": key}`) or in a question's instructions so Jev knows + /// where the image belongs. An image nobody references is rejected by + /// [`Self::validate`]. Attaching any asset makes the client send + /// `TypeSafe-Danger-Zone-Images: true`. + #[must_use] + pub fn with_image(mut self, key: impl Into, image: ImageAsset) -> Self { + self.assets.insert(key.into(), Asset::Image(image)); + self + } + + /// Whether the request carries an image or asks a [`Question::Box`], the + /// two things the provider admits only with its image opt-in header. + #[must_use] + pub fn needs_image_opt_in(&self) -> bool { + !self.assets.is_empty() + || self + .questions + .values() + .any(|question| matches!(question, Question::Box(_))) + } + /// Build a request using the stable Jev alias. #[must_use] pub fn jev(state: impl Into, questions: BTreeMap) -> Self { @@ -59,6 +89,46 @@ pub enum Question { Score(Score), /// Estimate the probability that a yes/no condition holds. Noul(Noul), + /// Locate one thing in an image (Jev 1.14 or later). The answer is a + /// single box; ask a [`Noul`] alongside it to learn whether the thing is + /// there at all, because a box is always returned. + Box(BoxQuestion), +} + +/// A question asking where something is in an attached image. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct BoxQuestion { + /// What to find, and in which image: name the state field holding the + /// image in backticks, for example ``"the search button in `page`"``. + pub instructions: Value, +} + +/// One attached asset. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum Asset { + /// A still PNG or JPEG image. + Image(ImageAsset), +} + +/// A still image sent inline as base64. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct ImageAsset { + /// Encoding of the image bytes. + pub format: ImageFormat, + /// Standard padded base64 of the image file, with no `data:` prefix. + pub data_base64: String, +} + +/// Image encodings the provider accepts. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ImageFormat { + /// PNG. + Png, + /// JPEG, written `jpg` on the wire. + #[serde(rename = "jpg", alias = "jpeg")] + Jpg, } /// A closed-set selection question. diff --git a/crates/tinyinference-decisions/src/response/mod.rs b/crates/tinyinference-decisions/src/response/mod.rs index e04fc53e..903d0e07 100644 --- a/crates/tinyinference-decisions/src/response/mod.rs +++ b/crates/tinyinference-decisions/src/response/mod.rs @@ -6,7 +6,9 @@ mod test; mod types; -pub use types::{Answer, ChoiceAnswer, EvaluationResponse, NoulAnswer, ScoreAnswer, Usage}; +pub use types::{ + Answer, BoxAnswer, BoxEdges, ChoiceAnswer, EvaluationResponse, NoulAnswer, ScoreAnswer, Usage, +}; use std::collections::BTreeSet; @@ -158,6 +160,18 @@ fn validate_pair(question: &Question, answer: &Answer) -> Result<()> { (Question::Noul(_), Answer::Noul(answer)) => { validate_probability(answer.noul, "noul")?; } + (Question::Box(_), Answer::Box(answer)) => { + validate_probability(answer.confidence, "box confidence")?; + let edges = answer.edges; + for edge in [edges.top, edges.bottom, edges.left, edges.right] { + validate_probability(edge, "box edge")?; + } + if edges.left > edges.right || edges.top > edges.bottom { + return Err(Error::invalid_response( + "box edges must satisfy left <= right and top <= bottom", + )); + } + } _ => { return Err(Error::invalid_response( "answer type must match its request question type", diff --git a/crates/tinyinference-decisions/src/response/mod_tests.rs b/crates/tinyinference-decisions/src/response/mod_tests.rs index e40d965a..54134f83 100644 --- a/crates/tinyinference-decisions/src/response/mod_tests.rs +++ b/crates/tinyinference-decisions/src/response/mod_tests.rs @@ -376,3 +376,75 @@ fn response_answer_types_round_trip_with_optional_usage() { without_usage ); } + +fn box_request() -> EvaluationRequest { + EvaluationRequest::new( + "jev-preview", + json!({"page": "p1"}), + BTreeMap::from([( + "button".to_owned(), + Question::Box(crate::BoxQuestion { + instructions: json!("the verify button in `page`"), + }), + )]), + ) +} + +fn box_response(edges: serde_json::Value, confidence: f64) -> EvaluationResponse { + serde_json::from_value(json!({ + "model": "jev-preview", + "answers": {"button": {"type": "box", "box": edges, "confidence": confidence}}, + "usage": {"input_tokens": 900, "output_tokens": 0} + })) + .unwrap() +} + +#[test] +fn a_box_answer_decodes_and_reports_its_centre() { + let response = box_response( + json!({"top": 0.8, "bottom": 0.9, "left": 0.7, "right": 0.9}), + 0.93, + ); + response.validate_for(&box_request()).unwrap(); + let Answer::Box(answer) = &response.answers["button"] else { + panic!("expected a box answer"); + }; + let (x, y) = answer.edges.centre(); + assert!((x - 0.8).abs() < 1e-9 && (y - 0.85).abs() < 1e-9); +} + +#[test] +fn a_box_answer_with_inverted_or_out_of_range_edges_is_rejected() { + let inverted = box_response( + json!({"top": 0.9, "bottom": 0.1, "left": 0.1, "right": 0.2}), + 0.5, + ); + assert!(inverted.validate_for(&box_request()).is_err()); + let outside = box_response( + json!({"top": 0.1, "bottom": 1.2, "left": 0.1, "right": 0.2}), + 0.5, + ); + assert!(outside.validate_for(&box_request()).is_err()); + let unsure = box_response( + json!({"top": 0.1, "bottom": 0.2, "left": 0.1, "right": 0.2}), + 1.5, + ); + assert!(unsure.validate_for(&box_request()).is_err()); +} + +#[test] +fn a_box_answer_to_a_noul_question_is_a_type_mismatch() { + let mut request = box_request(); + request.questions.insert( + "button".into(), + Question::Noul(Noul { + instructions: json!("Is there a button?"), + criteria: None, + }), + ); + let response = box_response( + json!({"top": 0.1, "bottom": 0.2, "left": 0.1, "right": 0.2}), + 0.5, + ); + assert!(response.validate_for(&request).is_err()); +} diff --git a/crates/tinyinference-decisions/src/response/types.rs b/crates/tinyinference-decisions/src/response/types.rs index 8f13963c..ea7f2f8c 100644 --- a/crates/tinyinference-decisions/src/response/types.rs +++ b/crates/tinyinference-decisions/src/response/types.rs @@ -78,6 +78,44 @@ pub enum Answer { Score(ScoreAnswer), /// Probability of yes. Noul(NoulAnswer), + /// Where the asked-for thing is in the image. + Box(BoxAnswer), +} + +/// Answer to a Box question. +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)] +pub struct BoxAnswer { + /// The box, as fractions of the image's width and height. + #[serde(rename = "box")] + pub edges: BoxEdges, + /// Confidence in the box, from zero to one. + pub confidence: f64, +} + +/// The edges of a box, each a fraction (0 to 1) of the image measured from its +/// top-left corner: `top` and `bottom` of its height, `left` and `right` of +/// its width. +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)] +pub struct BoxEdges { + /// Top edge, a fraction of the height. + pub top: f64, + /// Bottom edge, a fraction of the height. + pub bottom: f64, + /// Left edge, a fraction of the width. + pub left: f64, + /// Right edge, a fraction of the width. + pub right: f64, +} + +impl BoxEdges { + /// The centre of the box as `(x, y)` fractions of width and height. + #[must_use] + pub fn centre(&self) -> (f64, f64) { + ( + f64::midpoint(self.left, self.right), + f64::midpoint(self.top, self.bottom), + ) + } } /// Answer to a Choice question. diff --git a/docs/tinyinference-decisions.md b/docs/tinyinference-decisions.md index a2eff416..ea9a3973 100644 --- a/docs/tinyinference-decisions.md +++ b/docs/tinyinference-decisions.md @@ -38,6 +38,48 @@ TYPESAFE_API_KEY='' cargo run -p tinyinference-decisions --example basic The crate is part of the TinyInference workspace and is licensed GPL-3.0-only. +## Images and Box questions + +Jev 1.13 and later read images, and Jev 1.14 (`jev-preview`) adds the Box +question, which returns where one thing is in an image. Attach an image with +`EvaluationRequest::with_image(key, ImageAsset::png(&bytes))` and put `key` +somewhere in `state` (for example `{"page": key}`), so the model knows where it +belongs. Questions point at the state field in backticks. + +```rust,no_run +use std::collections::BTreeMap; +use serde_json::json; +use tinyinference_decisions::{ + Answer, BoxQuestion, Client, EvaluationRequest, ImageAsset, Question, +}; + +# async fn run(png: Vec) -> Result<(), Box> { +let request = EvaluationRequest::new( + "jev-preview", + json!({"page": "shot-1"}), + BTreeMap::from([( + "button".to_owned(), + Question::Box(BoxQuestion { instructions: json!("the Verify button in `page`") }), + )]), +) +.with_image("shot-1", ImageAsset::png(&png)); +let result = Client::from_env()?.evaluate(&request).await?; +if let Some(Answer::Box(answer)) = result.response.answers.get("button") { + let (x, y) = answer.edges.centre(); // fractions of width and height + println!("{x} {y} at confidence {}", answer.confidence); +} +# Ok(()) +# } +``` + +The client sends `TypeSafe-Danger-Zone-Images: true` on any request with an +image or a Box question, as the provider requires. `validate` checks the +documented limits before anything is sent: PNG or JPEG whose header matches the +declared format, 280 to 1568 pixels per side, 16 MiB per image and 32 MiB in +all, and every asset key referenced by the state or a question. Images are +neither resized nor rotated; that stays with the caller. A Box answer is always +one box, so ask a Noul alongside it when the thing may be absent. + ## Local decision models Use `ClientConfig::self_hosted` for a locally served OpenJEV, Decider, Surogate