Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions crates/tinyinference-decisions/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ categories = ["api-bindings", "asynchronous"]

[dependencies]
async-trait = { workspace = true }
# Encodes attached images for the System One `assets` map.
base64 = "0.23"
httpdate = { workspace = true }
reqwest = { workspace = true }
serde = { workspace = true }
Expand Down
6 changes: 6 additions & 0 deletions crates/tinyinference-decisions/src/client/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ use crate::{Error, EvaluationRequest, EvaluationResponse, Result};

const MAX_RETRIES: u32 = 100;
const MAX_RESPONSE_BYTES: usize = 16 * 1024 * 1024;
/// Header the provider requires on every request that carries an image or a
/// Box question; without it such a request is rejected with 400.
const IMAGE_OPT_IN_HEADER: &str = "TypeSafe-Danger-Zone-Images";

impl Client {
/// Construct a client from an explicit configuration.
Expand Down Expand Up @@ -143,6 +146,9 @@ impl Client {
)
});
let mut builder = self.http.post(&url).json(request);
if request.needs_image_opt_in() {
builder = builder.header(IMAGE_OPT_IN_HEADER, "true");
}
let api_key = self.config.api_key.expose();
if !(self.config.provider == Provider::SelfHosted && api_key.trim().is_empty()) {
builder = builder.bearer_auth(api_key);
Expand Down
22 changes: 22 additions & 0 deletions crates/tinyinference-decisions/src/client/mod_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,28 @@ async fn sends_the_documented_endpoint_and_bearer_header() {
assert!(requests[0].body.contains("\"model\":\"jev-latest\""));
}

#[tokio::test]
async fn sends_the_image_opt_in_header_only_with_an_image() {
let (client, requests) = mock_client(
config("http://127.0.0.1:1".into()),
vec![response(200, &success(), ""), response(200, &success(), "")],
);
client.evaluate(&request()).await.unwrap();
let image = crate::request::image::tests::png_header(400, 300);
let mut with_image = request();
with_image.state = json!({"message": "review this", "photo": "img-1"});
let with_image = with_image.with_image("img-1", crate::ImageAsset::png(&image));
client.evaluate(&with_image).await.unwrap();
let requests = requests.lock().await;
assert!(requests[0].headers.get(IMAGE_OPT_IN_HEADER).is_none());
assert!(!requests[0].body.contains("assets"));
assert_eq!(
requests[1].headers.get(IMAGE_OPT_IN_HEADER).unwrap(),
"true"
);
assert!(requests[1].body.contains("\"assets\":{\"img-1\""));
}

#[tokio::test]
async fn an_exact_endpoint_override_is_not_extended_with_a_provider_path() {
let endpoint = "https://example.com/api/alpha/decisions";
Expand Down
9 changes: 7 additions & 2 deletions crates/tinyinference-decisions/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,5 +54,10 @@ pub use client::{
Client, ClientConfig, EvaluationFailure, EvaluationResult, Provider, RetryPolicy,
};
pub use error::{Error, Result};
pub use request::{Choice, EvaluationRequest, Noul, NoulCriteria, Question, Score};
pub use response::{Answer, ChoiceAnswer, EvaluationResponse, NoulAnswer, ScoreAnswer, Usage};
pub use request::{
Asset, BoxQuestion, Choice, EvaluationRequest, ImageAsset, ImageFormat, Noul, NoulCriteria,
Question, Score,
};
pub use response::{
Answer, BoxAnswer, BoxEdges, ChoiceAnswer, EvaluationResponse, NoulAnswer, ScoreAnswer, Usage,
};
4 changes: 3 additions & 1 deletion crates/tinyinference-decisions/src/request/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,6 @@

This module defines the exact state-and-questions payload sent to TypeSafe's
System One endpoint. `types.rs` holds the serde wire types, `mod.rs` validates
the documented primitive bounds before transport, and `test.rs` pins both.
the documented primitive bounds before transport, and `mod_tests.rs` pins both.
`image.rs` builds inline image assets and checks them against the provider's
image limits (format, side lengths, sizes) by reading only the file header.
121 changes: 121 additions & 0 deletions crates/tinyinference-decisions/src/request/image.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
//! Building inline image assets and checking them against the provider's
//! documented image limits before anything is sent.

use base64::Engine as _;
use base64::engine::general_purpose::STANDARD;

use super::types::{ImageAsset, ImageFormat};
use crate::{Error, Result};

/// Smallest accepted width or height, in pixels.
pub(crate) const MIN_SIDE: u32 = 280;
/// Largest accepted width or height, in pixels.
pub(crate) const MAX_SIDE: u32 = 1568;
/// Largest accepted image file.
pub(crate) const MAX_IMAGE_BYTES: usize = 16 * 1024 * 1024;
/// Largest accepted total of all image files in one request.
pub(crate) const MAX_TOTAL_IMAGE_BYTES: usize = 32 * 1024 * 1024;
/// Longest accepted asset reference string, in bytes.
pub(crate) const MAX_REFERENCE_BYTES: usize = 1024;

const PNG_SIGNATURE: &[u8] = b"\x89PNG\r\n\x1a\n";

impl ImageAsset {
/// An image from the bytes of a PNG file.
#[must_use]
pub fn png(bytes: &[u8]) -> Self {
Self {
format: ImageFormat::Png,
data_base64: STANDARD.encode(bytes),
}
}

/// An image from the bytes of a JPEG file.
#[must_use]
pub fn jpeg(bytes: &[u8]) -> Self {
Self {
format: ImageFormat::Jpg,
data_base64: STANDARD.encode(bytes),
}
}

/// Decode the image and check it against the provider's limits, returning
/// its size in bytes.
pub(crate) fn checked_len(&self) -> Result<usize> {
let bytes = STANDARD
.decode(&self.data_base64)
.map_err(|_| Error::invalid_request("image data_base64 must be padded base64"))?;
if bytes.len() > MAX_IMAGE_BYTES {
return Err(Error::invalid_request("image must be at most 16 MiB"));
}
let (width, height) = dimensions(self.format, &bytes)?;
if !(MIN_SIDE..=MAX_SIDE).contains(&width) || !(MIN_SIDE..=MAX_SIDE).contains(&height) {
return Err(Error::invalid_request(format!(
"image sides must be {MIN_SIDE} to {MAX_SIDE} pixels, got {width}x{height}"
)));
}
Ok(bytes.len())
}
}

/// Width and height read from the image header.
pub(crate) fn dimensions(format: ImageFormat, bytes: &[u8]) -> Result<(u32, u32)> {
match format {
ImageFormat::Png => png_dimensions(bytes),
ImageFormat::Jpg => jpeg_dimensions(bytes),
}
.ok_or_else(|| match format {
ImageFormat::Png => Error::invalid_request("image is not a readable PNG"),
ImageFormat::Jpg => Error::invalid_request("image is not a readable JPEG"),
})
}

fn png_dimensions(bytes: &[u8]) -> Option<(u32, u32)> {
// The signature is followed by the IHDR chunk: length, "IHDR", width, height.
if !bytes.starts_with(PNG_SIGNATURE) || bytes.get(12..16)? != b"IHDR" {
return None;
}
let width = u32::from_be_bytes(bytes.get(16..20)?.try_into().ok()?);
let height = u32::from_be_bytes(bytes.get(20..24)?.try_into().ok()?);
Some((width, height))
}

fn jpeg_dimensions(bytes: &[u8]) -> Option<(u32, u32)> {
if !bytes.starts_with(&[0xFF, 0xD8]) {
return None;
}
let mut at = 2;
loop {
// Skip fill bytes before a marker.
while *bytes.get(at)? == 0xFF && *bytes.get(at + 1)? == 0xFF {
at += 1;
}
if *bytes.get(at)? != 0xFF {
return None;
}
let marker = *bytes.get(at + 1)?;
// Markers without a length field.
if marker == 0x01 || (0xD0..=0xD7).contains(&marker) {
at += 2;
continue;
}
let length = usize::from(u16::from_be_bytes([
*bytes.get(at + 2)?,
*bytes.get(at + 3)?,
]));
// Start-of-frame markers carry the size; C4, C8 and CC do not.
if matches!(marker, 0xC0..=0xCF) && !matches!(marker, 0xC4 | 0xC8 | 0xCC) {
let height = u16::from_be_bytes([*bytes.get(at + 5)?, *bytes.get(at + 6)?]);
let width = u16::from_be_bytes([*bytes.get(at + 7)?, *bytes.get(at + 8)?]);
return Some((u32::from(width), u32::from(height)));
}
if length < 2 {
return None;
}
at += 2 + length;
}
}

#[cfg(test)]
#[path = "image_tests.rs"]
pub(crate) mod tests;
98 changes: 98 additions & 0 deletions crates/tinyinference-decisions/src/request/image_tests.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
//! Image asset construction, header reading, and limit checks.

use super::*;

/// The first bytes of a PNG file of the given size: enough for the header.
pub(crate) fn png_header(width: u32, height: u32) -> Vec<u8> {
let mut bytes = PNG_SIGNATURE.to_vec();
bytes.extend_from_slice(&13_u32.to_be_bytes());
bytes.extend_from_slice(b"IHDR");
bytes.extend_from_slice(&width.to_be_bytes());
bytes.extend_from_slice(&height.to_be_bytes());
bytes.extend_from_slice(&[8, 6, 0, 0, 0]);
bytes
}

/// The first bytes of a baseline JPEG of the given size, with an APP0 segment
/// before the frame so the reader has to walk a segment.
fn jpeg_header(width: u16, height: u16) -> Vec<u8> {
let mut bytes = vec![0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10];
bytes.extend_from_slice(b"JFIF\0\x01\x01\0\0\x01\0\x01\0\0");
bytes.extend_from_slice(&[0xFF, 0xC0, 0x00, 0x11, 0x08]);
bytes.extend_from_slice(&height.to_be_bytes());
bytes.extend_from_slice(&width.to_be_bytes());
bytes.extend_from_slice(&[0x03, 0x01, 0x22, 0x00]);
bytes
}

#[test]
fn png_asset_is_plain_base64_and_reads_its_size() {
let asset = ImageAsset::png(&png_header(400, 300));
assert_eq!(asset.format, ImageFormat::Png);
assert!(!asset.data_base64.starts_with("data:"));
assert_eq!(asset.checked_len().unwrap(), png_header(400, 300).len());
}

#[test]
fn jpeg_asset_reads_its_size_past_other_segments() {
let bytes = jpeg_header(1024, 768);
assert_eq!(dimensions(ImageFormat::Jpg, &bytes).unwrap(), (1024, 768));
assert!(ImageAsset::jpeg(&bytes).checked_len().is_ok());
}

#[test]
fn jpeg_reader_skips_fill_bytes_and_standalone_markers() {
let mut bytes = vec![0xFF, 0xD8, 0xFF, 0xFF, 0xD0];
bytes.extend_from_slice(&jpeg_header(300, 300)[2..]);
assert_eq!(dimensions(ImageFormat::Jpg, &bytes).unwrap(), (300, 300));
}

#[test]
fn rejects_sides_outside_the_documented_range() {
for (width, height) in [(279, 400), (400, 279), (1569, 400), (400, 1569)] {
let error = ImageAsset::png(&png_header(width, height))
.checked_len()
.unwrap_err();
assert!(error.to_string().contains("280 to 1568"), "{error}");
}
assert!(
ImageAsset::png(&png_header(280, 1568))
.checked_len()
.is_ok()
);
}

#[test]
fn rejects_bytes_that_do_not_match_the_declared_format() {
let error = ImageAsset::png(&jpeg_header(400, 400))
.checked_len()
.unwrap_err();
assert!(error.to_string().contains("not a readable PNG"));
let error = ImageAsset::jpeg(&png_header(400, 400))
.checked_len()
.unwrap_err();
assert!(error.to_string().contains("not a readable JPEG"));
assert!(dimensions(ImageFormat::Jpg, &[0xFF, 0xD8, 0x00]).is_err());
assert!(dimensions(ImageFormat::Jpg, &[0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x01]).is_err());
assert!(dimensions(ImageFormat::Png, &PNG_SIGNATURE[..4]).is_err());
}

#[test]
fn rejects_invalid_base64_and_oversized_files() {
let asset = ImageAsset {
format: ImageFormat::Png,
data_base64: "data:image/png;base64,AAAA".into(),
};
assert!(
asset
.checked_len()
.unwrap_err()
.to_string()
.contains("base64")
);

let mut huge = png_header(400, 400);
huge.resize(MAX_IMAGE_BYTES + 1, 0);
let error = ImageAsset::png(&huge).checked_len().unwrap_err();
assert!(error.to_string().contains("16 MiB"));
}
56 changes: 55 additions & 1 deletion crates/tinyinference-decisions/src/request/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,13 @@
#[path = "mod_tests.rs"]
mod test;

pub(crate) mod image;
mod types;

pub use types::{Choice, EvaluationRequest, Noul, NoulCriteria, Question, Score};
pub use types::{
Asset, BoxQuestion, Choice, EvaluationRequest, ImageAsset, ImageFormat, Noul, NoulCriteria,
Question, Score,
};

use crate::{Error, Result};

Expand Down Expand Up @@ -40,16 +44,66 @@ impl EvaluationRequest {
}
question.validate()?;
}
self.validate_assets()
}

fn validate_assets(&self) -> Result<()> {
if self.assets.is_empty() {
return Ok(());
}
// Every reference must appear somewhere the model reads, or the
// provider rejects the request; the wire text of state and questions
// is exactly where it looks. Search the decoded string values and
// object keys, not the escaped JSON text, so keys containing `\`, `"`
// or control characters match.
let readable = serde_json::to_value((&self.state, &self.questions))
.map_err(|_| Error::invalid_request("state and questions must serialise"))?;
let mut total = 0_usize;
for (key, asset) in &self.assets {
if key.trim().is_empty() {
return Err(Error::invalid_request("asset references must not be empty"));
}
if key.len() > image::MAX_REFERENCE_BYTES {
return Err(Error::invalid_request(
"asset references must be at most 1024 bytes",
));
}
if !value_mentions(&readable, key) {
return Err(Error::invalid_request(format!(
"asset `{key}` is not referenced by the state or any question"
)));
}
let Asset::Image(image) = asset;
total = total.saturating_add(image.checked_len()?);
}
if total > image::MAX_TOTAL_IMAGE_BYTES {
return Err(Error::invalid_request(
"images must total at most 32 MiB per request",
));
}
Ok(())
}
}

/// Whether any decoded string value or object key inside `value` contains `needle`.
fn value_mentions(value: &serde_json::Value, needle: &str) -> bool {
match value {
serde_json::Value::String(text) => text.contains(needle),
serde_json::Value::Array(items) => items.iter().any(|item| value_mentions(item, needle)),
serde_json::Value::Object(map) => map
.iter()
.any(|(key, item)| key.contains(needle) || value_mentions(item, needle)),
_ => false,
}
}

impl Question {
fn validate(&self) -> Result<()> {
match self {
Self::Choice(question) => question.validate(),
Self::Score(question) => question.validate(),
Self::Noul(question) => question.validate(),
Self::Box(question) => validate_instructions(&question.instructions),
}
}
}
Expand Down
Loading
Loading