diff --git a/Cargo.lock b/Cargo.lock index 1acbe63f6..623f30b34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,6 +111,12 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "cc" version = "1.10.0" @@ -581,6 +587,14 @@ dependencies = [ "vstd", ] +[[package]] +name = "clazel-webp" +version = "0.0.0" +dependencies = [ + "clazel-image-shape", + "image-webp", +] + [[package]] name = "clazel-window" version = "0.0.0" @@ -1111,6 +1125,16 @@ dependencies = [ "unicode-joining-type", ] +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + [[package]] name = "indexmap" version = "2.14.2" @@ -1733,6 +1757,12 @@ dependencies = [ "psl-types", ] +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quote" version = "1.0.47" diff --git a/crates/clazel-webp/Cargo.toml b/crates/clazel-webp/Cargo.toml new file mode 100644 index 000000000..c7705889a --- /dev/null +++ b/crates/clazel-webp/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "clazel-webp" +version = "0.0.0" +edition.workspace = true +license.workspace = true +publish.workspace = true +description = "A WebP decoder (Phase 6 item 6.8): the RIFF container, VP8X, ANIM and ANMF with every frame composited, and VP8L lossless decoding (transforms, colour cache, meta prefix codes, backward references); lossy VP8 is item 6.9 and is refused. A resumable step function over caller-owned buffers, implementing clazel-image-shape's Decoder." + +[dependencies] +# The decoder interface (item 6.3), our own crate with no dependencies, so no +# third-party code is shipped (CLAUDE.md rule 8). +clazel-image-shape = { path = "../clazel-image-shape" } + +# The reference decoder of the differential test (tests/conformance_webp.rs), +# never compiled into the browser: image-rs's `image-webp`, written by others, +# driven for the pixels and frames of each file (docs/DEPENDENCIES.md, class +# `test`). +[dev-dependencies] +image-webp = "=0.2.4" + +# Verification tier (docs/VERIFICATION.md). `tier` is what is proven about the +# shipped code today; `target-tier` is the plan's reading of item 6.8: bytes +# of a page-chosen file drive every index and counter, so Tier 1 (Kani +# harnesses in src/proofs.rs: prefix code construction over any code lengths, +# each transform over any row up to a stated width, and the step function's +# parts). The tier stays 0 until CI's full verification has run those +# harnesses green. +[package.metadata.clazel] +tier = 0 +target-tier = 1 + +[lints] +workspace = true diff --git a/crates/clazel-webp/README.md b/crates/clazel-webp/README.md new file mode 100644 index 000000000..4826fa008 --- /dev/null +++ b/crates/clazel-webp/README.md @@ -0,0 +1,159 @@ +# clazel-webp + +A WebP decoder for [Clazel](../../README.md), a web browser whose parsers of +untrusted input carry machine-checked proofs. Phase 6 item 6.8 +(`docs/phases/phase-6.md`): the RIFF container, animation, and **lossless** +(VP8L) decoding. Lossy VP8 is item 6.9; here a `VP8 ` chunk is recognised and +refused with `Error::Unsupported`. It runs in the renderer's sandbox, on bytes +a page chose, and implements the one interface of every Phase 6 decoder, +`clazel_image_shape::Decoder` (item 6.3): a resumable step function over +caller-owned buffers, nothing allocated while decoding. + +Its only dependency is `clazel-image-shape`. `image-webp` is a dev-dependency, +used only by the differential test. + +## What it does + +RFC 9649 (the WebP container, and the lossless bitstream): `RIFF`/`WEBP`, the +simple lossless format, the extended format (`VP8X`; `ICCP`, `ALPH`, `EXIF`, +`XMP ` and unknown chunks skipped), `ANIM` and `ANMF`. **Every frame** of an +animation is decoded and composited onto the canvas with the previous +frame's disposal and the frame's blending method; after each +`Status::FrameDone` the pixel buffer holds the canvas as displayed with that +frame and `Webp::frame()` has the frame's rectangle, duration, blending and +disposal. VP8L: the four transforms (predictor with its fourteen modes, +colour, subtract green, colour indexing with pixel bundling), the colour +cache (1 to 11 bits), meta prefix codes, simple and normal code length codes, +LZ77 backward references with the distance map. + +| File | Holds | +|------|-------| +| `src/lib.rs` | the pinned standard, the section numbers, the listed choices | +| `src/decoder.rs` | the container state machine, `Webp`, `Decoder`, `Webp::state_inv`, compositing | +| `src/vp8l.rs` | the lossless bitstream as a state machine over atomic actions | +| `src/huffman.rs` | canonical prefix codes: construction, decoding without tables | +| `src/transform.rs` | the inverse transforms, one row at a time, in place | +| `src/distance.rs` | the LZ77 prefix coding and the distance map (Figure 20) | +| `src/compose.rs` | alpha blending | +| `src/bits.rs`, `src/work.rs` | the bit reader, the work buffer's layout | +| `src/proofs.rs` | the Kani harnesses | + +The work buffer is `work_len(config)`: about 11.6 kB (colour cache, code +lengths, colour table), 12 bytes per block of 4 by 4 pixels (the predictor, +colour transform and entropy images), 6.4 kB per prefix code group held (at +most 1,024, and at most one per block of 4 by 4), and for an animation a +frame buffer as large as the canvas. A still image is decoded straight into +the pixel buffer. + +## Listed choices (`src/lib.rs` has them with the section numbers) + +Animation canvas transparent black and the `ANIM` background ignored, +disposal to transparent; alpha blending on stored values with libwebp's +integer arithmetic (a destination of alpha 0 takes the source exactly); +`alpha_is_used` ignored; `ANMF`/`ANIM` skipped in a still image, image data +outside a frame in an animation an error; `ALPH` skipped; the RIFF size ends +the stream; prefix codes empty, over-subscribed or incomplete are errors and a +single symbol takes zero bits; predictor modes 14 and 15 predict like mode 0; +at most 1,024 prefix code groups (more is `Unsupported`); a backward +reference before the first pixel or past the last is an error. + +## Kani harnesses (`src/proofs.rs`) + +Sixty-nine. The step harnesses are one per phase, role or transform: with +the phase symbolic, CBMC ran out of memory (over 4 GB locally; on CI the +runner was lost after two hours). What keeps each in memory is listed at the +top of `src/proofs.rs` (MiniSat for the pixel, finish and container ones, +smaller unwinding, a 4-colour table where it is indexed symbolically). +**Not yet run green on CI**: locally, under Kani 0.68.0 with a 4 GB cap, each +was run on its own (results in the pull request); CI's approved proof jobs +are the result that counts. + +| Harness | Statement (every one: no panic, no overflow, no out-of-bounds index) | Bound | +|---------|-----------|-------| +| `huffman_build_any` | prefix code construction: an error if a length is over 15; otherwise complete (Kraft sum exactly 1) or a single symbol, each used symbol once in the table | **any code lengths**, 6 symbols | +| `huffman_decode_any` | decoding: a symbol of the alphabet with the code's length, never more bits than available, `More` only under 15 bits | any built code, any 32 bits, any count available | +| `distance_value_any` | the LZ77 prefix coding: 1 to 1,048,576 | every prefix code 0 to 39, any extra bits | +| `distance_map_any` | the distance of a code is at least 1, `code - 120` above 120, within 7 rows and 8 columns below | every code, every width up to 2^14 | +| `predict_any` | each predictor mode's value (L, T, TR, TL, black, Average2 of L and T, Select is L or T) | every mode, any four pixels | +| `predictor_row_any` | the inverse predictor of a row: the other row untouched, the top-left pixel is black plus residual | any row of 3 pixels in 2 rows, any block bits 2 to 9, any modes, any pixels | +| `color_row_any` | the inverse colour transform: green and alpha unchanged | any row of 3, any transform data | +| `green_row_any` | the inverse subtract green: exactly red and blue gain green | any row of 3 | +| `palette_row_any` | colour indexing with bundling: each pixel is the table entry its packed bits name, or transparent black past the table | every bundling width, rows of 1 to 8 pixels, up to 4 colours | +| `palette_in_place_any` | the same, two rows expanded in place, last row first, equal to the expansion from a copy | every bundling width, 4 pixels in 2 rows | +| `blend_px_any` | alpha blending: alpha 255 gives the source, 0 the destination, otherwise alpha does not go down | any two pixels | +| `act_*` (40, `act_harness!`) | one atomic action of the lossless decoder, in one phase each (header, transforms, cache, meta prefix, every step of reading a prefix code, and pixels: literal symbols, lengths, distances, cache lookups, a backward reference of at most 4 pixels), keeps `Vp8l::inv` on `Ok`; the two distance phases have one harness per role | any state with the invariant, any bits, any contents of every scratch region; images of at most 4 pixels; `act_build_code`: an alphabet of at most 8 symbols; palette role: a table of 4 colours | +| `finish_*` (9) | finishing an image keeps `Vp8l::inv`: the colour table, entropy image and two transform images, and the main image with no transform or with one of each kind (inverting several is a loop whose body depends only on that transform and the pixels) | the same, up to 4 colours, 2 rows; `finish_main_predictor` exactly 2 by 2 pixels (`predictor_row_any` covers rows of 3); `finish_main_index`: a table of 4 colours | +| `container_*` (9, `container_harness!`) | one byte of the container, in each phase but the lossless payload, keeps `Webp::state_inv` on `Ok` | any state with the invariant, a 2 by 2 canvas | + +### What Kani covers, and what it does not + +- **The step function itself is not machine-checked, its parts are.** Item + 6.3's template `decoder_step_harness!` was not instantiated: GIF's + experience (item 6.7) is that CBMC does not finish on a whole state machine + of this size, and this one is larger (a work buffer of 15 kB even at the + harness's sizes). `step` is `Parts::new` (a unit test and the lengths in + `inv`), then a loop of `Webp::byte`, which is `dispatch` (the container + harness) and, in a lossless payload, `Vp8l::push` and `Vp8l::advance`, a + loop of `Vp8l::act` (one harness per phase) and `finish_image` (one per + role). What the template would add and is argued here, not + proven: that `inv` is kept over several bytes (induction from one), that + `consumed <= chunk.len()` and that `NeedMore` consumes the chunk (a loop's + index), and that nothing outside the buffers is written (safe Rust + bounds-checks every index, and no index uses unchecked arithmetic: the + workspace lints deny it). +- **Small bounds.** Under Kani the colour cache is 8 entries (`work.rs`), so a + green alphabet is 288 symbols; images are at most 4 pixels; a backward + reference at most 4 pixels. The code is the same code with other + constants; the real sizes are run by the unit tests, the differential test + and Miri. +- **Separate arrays**: the harnesses give the scratch regions as separate + arrays, not one work buffer cut by `Parts::new`. +- **Not stated**: that the pixels are right (the differential test's and the + hand-made streams'), that the work is bounded in time (an image of 2^26 + pixels coded with zero-bit prefix codes decodes from a handful of bytes: + the limits of item 6.0(j) bound it), and that every state `inv` admits is + reachable (it over-approximates, so the harnesses are sound). +- The tier stays 0 until CI's full verification has run these green + (CLAUDE.md rule 7). + +## Tests + +- **Unit tests** (22): the bit reader, the layout, prefix codes (the RFC + example, one symbol, bad codes), the distance map and Table 4, each + transform and predictor border, blending. +- **`tests/conformance_webp.rs`** (15 tests): Wuffs' WebP test data + (`tests/data/wuffs-webp`, commit `b36a81d`, MIT or Apache-2.0 at our + choice; libwebp's own vectors have no licence and are not vendored) + against `image-webp` 0.2.4: **the 11 lossless files agree, exactly** + (the largest 1,165 by 859); the animation agrees on its first frame and + within 1 per channel on the alpha-blended ones (`image-webp` rounds blending + more precisely than libwebp's formula, which is followed: listed); the 2 + lossy files are refused as unsupported, each with its reason. Every file at + chunk sizes 1 to 4096 (with and without empty chunks) equals the + whole-file decode, with the invariant checked at every step and canary + bytes past the buffers untouched. 1,440 deterministic byte mutants + never panic (968 end in an error, the rest decode something). +- **`tests/handmade/`**: a VP8L encoder written for the tests (from the RFC, + sharing no code with the decoder) makes **531 images** that must decode to + exactly the source, byte by byte, whole, and as `image-webp` reads them: + ten orders of transforms at six sizes and six seeds (360), every predictor + mode (6), pixel bundling at every palette size from 1 to 256 and nine + widths (81), caches of 0 to 11 bits with and without meta prefix codes (72), + backward references of every distance code and simple and normal prefix + codes (12). Container cases: a still `VP8X` with chunks to skip and + trailing data; an animation of five frames with blending, no blending and + both disposals, checked against a compositor written in the test; eight + malformed files and four malformed streams, each refused with the right + reason. +- **Miri**: every test runs a stride (`cfg!(miri)`): the 156-byte file, chunk + size 1, 2 mutants, one hand-made image of at most 8 by 4 pixels. Miri runs + 25,000 to 30,000 times slower than native; measured with `--cfg miri` + natively, the 156-byte file decodes in 0.85 ms. + +## Not done + +Lossy VP8 and `ALPH` (item 6.9). The loop count and frame timing policy +(6.19). Prefix code group counts above 1,024 (or above the image's blocks). +libwebp's own test vectors (no licence) and `dwebp`'s output: `image-webp` +is the reference, and libwebp's blending is reproduced from its source's +formula, not compared with it. Kani has not been run on this crate. diff --git a/crates/clazel-webp/src/bits.rs b/crates/clazel-webp/src/bits.rs new file mode 100644 index 000000000..f8b9ee1cf --- /dev/null +++ b/crates/clazel-webp/src/bits.rs @@ -0,0 +1,125 @@ +//! The bit reader: bits of the stream, least significant first (RFC 9649 +//! section 3.3's `ReadBits`), pushed one byte at a time. +//! +//! RFC 9649 section 3.3 ("Nomenclature") and 3.7.2.3: bits are read from the +//! least significant bit of each byte first. + +/// The unread bits of the stream. A byte is pushed only when 8 more fit in +/// the 64-bit accumulator; the decoder drains it to under 32 bits after +/// every push. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Bits { + /// The unread bits, the next one in bit 0. Bits at and above `n` are 0. + acc: u64, + /// How many bits of `acc` are unread (0 to 64). + n: u8, + /// The last byte of the chunk was pushed: no more bits will come. + pub(crate) end: bool, +} + +impl Bits { + /// An empty reader. + pub(crate) const fn new() -> Bits { + Bits { + acc: 0, + n: 0, + end: false, + } + } + + /// The invariant: the accumulator holds no bit at or past `n`. + pub(crate) fn inv(&self) -> bool { + self.n <= 64 + && self + .acc + .checked_shr(u32::from(self.n)) + .is_none_or(|rest| rest == 0) + } + + /// How many bits are unread. + pub(crate) fn len(&self) -> u8 { + self.n + } + + /// Appends the next byte of the stream. (Does nothing if the + /// accumulator is too full; the decoder drains it after every push, so + /// that does not happen.) + pub(crate) fn push(&mut self, b: u8) { + if self.n <= 56 { + let shifted = u64::from(b).checked_shl(u32::from(self.n)).unwrap_or(0); + self.acc |= shifted; + self.n = self.n.saturating_add(8); + } + } + + /// The next 32 bits (bits not yet pushed read as 0), the next one in + /// bit 0. + pub(crate) fn peek32(&self) -> u32 { + u32::try_from(self.acc & 0xFFFF_FFFF).unwrap_or(0) + } + + /// Drops `k` bits (at most the unread ones). + pub(crate) fn skip(&mut self, k: u8) { + let k = k.min(self.n); + self.acc = self.acc.checked_shr(u32::from(k)).unwrap_or(0); + self.n = self.n.saturating_sub(k); + } + + /// Reads `k` bits (0 to 32) if that many are unread. + pub(crate) fn take(&mut self, k: u8) -> Option { + if k > 32 || self.n < k { + return None; + } + let mask = if k == 0 { + 0 + } else { + u32::MAX + .checked_shr(u32::from(32u8.saturating_sub(k))) + .unwrap_or(0) + }; + let v = self.peek32() & mask; + self.skip(k); + Some(v) + } +} + +#[cfg(kani)] +impl Bits { + /// Any reader, the invariant not assumed. + pub(crate) fn any() -> Bits { + Bits { + acc: kani::any(), + n: kani::any(), + end: kani::any(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_lsb_first() { + let mut b = Bits::new(); + b.push(0b1010_0110); + b.push(0xFF); + assert_eq!(b.take(3), Some(0b110)); + assert_eq!(b.take(5), Some(0b10100)); + assert_eq!(b.take(8), Some(0xFF)); + assert_eq!(b.take(1), None); + assert!(b.inv()); + } + + #[test] + fn zero_and_wide_reads() { + let mut b = Bits::new(); + for _ in 0..4 { + b.push(0xAB); + } + assert_eq!(b.take(0), Some(0)); + assert_eq!(b.take(32), Some(0xABAB_ABAB)); + assert_eq!(b.len(), 0); + assert_eq!(b.take(33), None); + } +} diff --git a/crates/clazel-webp/src/compose.rs b/crates/clazel-webp/src/compose.rs new file mode 100644 index 000000000..5205ebceb --- /dev/null +++ b/crates/clazel-webp/src/compose.rs @@ -0,0 +1,87 @@ +//! Putting a frame on the canvas: alpha blending (RFC 9649 section 2.7.1.1, +//! "Blending method" and the formula in its notes). + +use crate::transform::{Px, put, px}; + +/// Blends `src` over `dst`, neither premultiplied (the formula of +/// section 2.7.1.1: `blend.A = src.A + dst.A * (1 - src.A / 255)`, and the +/// colours weighted by their alphas, divided by `blend.A`). +/// +/// Integer arithmetic, listed choices: +/// - a source alpha of 0 gives `dst`, of 255 gives `src`, and a destination +/// alpha of 0 gives `src` (all exact in the formula); +/// - otherwise libwebp's (`anim_decode.c`, `BlendPixelNonPremult`): +/// `dst_factor = dst.A * (256 - src.A) >> 8`, `blend.A = src.A + +/// dst_factor`, `scale = 2^24 / blend.A`, and each colour channel +/// `(src * src.A + dst * dst_factor) * scale >> 24`. +/// +/// The RFC says blending "SHOULD be done in linear color space"; browsers +/// (libwebp) do it on the stored values, as here. +pub(crate) fn blend_px(src: Px, dst: Px) -> Px { + let [sr, sg, sb, sa] = src; + let [dr, dg, db, da] = dst; + if sa == 0 { + return dst; + } + if sa == 255 || da == 0 { + return src; + } + let sa = u32::from(sa); + let factor = u32::from(da).wrapping_mul(256u32.wrapping_sub(sa)) >> 8; + let blend_a = sa.wrapping_add(factor).clamp(1, 255); + let scale = (1u32 << 24).checked_div(blend_a).unwrap_or(0); + let channel = |s: u8, d: u8| -> u8 { + let sum = u32::from(s) + .wrapping_mul(sa) + .wrapping_add(u32::from(d).wrapping_mul(factor)); + u8::try_from(sum.wrapping_mul(scale) >> 24).unwrap_or(255) + }; + [ + channel(sr, dr), + channel(sg, dg), + channel(sb, db), + u8::try_from(blend_a).unwrap_or(255), + ] +} + +/// Blends each pixel of `src` over the pixel of `dst` at the same place (as +/// many pixels as the shorter of the two holds). +pub(crate) fn blend_row(dst: &mut [u8], src: &[u8]) { + for i in 0..src.len() / 4 { + let b = blend_px(px(src, i), px(dst, i)); + put(dst, i, b); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extremes() { + let dst = [10, 20, 30, 200]; + assert_eq!(blend_px([1, 2, 3, 0], dst), dst); + assert_eq!(blend_px([1, 2, 3, 255], dst), [1, 2, 3, 255]); + assert_eq!(blend_px([7, 8, 9, 100], [0, 0, 0, 0]), [7, 8, 9, 100]); + } + + #[test] + fn half_over_opaque() { + // 50% white over opaque black: alpha stays 255 (128 + 255 * 128 >> + // 8 = 128 + 127), colour 255 * 128 * scale >> 24 about 128. + let out = blend_px([255, 255, 255, 128], [0, 0, 0, 255]); + assert_eq!(out[3], 255); + assert!((127..=128).contains(&out[0])); + assert_eq!(out[0], out[1]); + } + + #[test] + fn never_exceeds() { + for sa in [1u8, 2, 3, 100, 254] { + for da in [1u8, 128, 255] { + let out = blend_px([255, 255, 255, sa], [255, 255, 255, da]); + assert!(out[0] >= 254, "{sa} {da} {out:?}"); + } + } + } +} diff --git a/crates/clazel-webp/src/decoder.rs b/crates/clazel-webp/src/decoder.rs new file mode 100644 index 000000000..91a3e73e6 --- /dev/null +++ b/crates/clazel-webp/src/decoder.rs @@ -0,0 +1,939 @@ +//! The WebP container (RFC 9649 section 2) as a byte-at-a-time state +//! machine, driving the lossless decoder of `vp8l.rs` and compositing the +//! frames of an animation; implements [`clazel_image_shape::Decoder`]. +//! +//! Citations are to RFC 9649 by section; the listed choices are in `lib.rs`. + +use clazel_image_shape::{ + Buffers, Config, ConfigStatus, Decoder, Error, Format, FrameCount, Status, Step, +}; + +use crate::compose::blend_row; +use crate::vp8l::Vp8l; +use crate::work::{Parts, work_len}; + +fn bad(why: &'static str) -> Error { + Error::Malformed(why) +} + +/// How a frame's transparent pixels meet the canvas (section 2.7.1.1, +/// "Blending method (B)"). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Blend { + /// 0: alpha-blend the frame over the canvas. + Alpha, + /// 1: overwrite the frame's rectangle. + Overwrite, +} + +/// What happens to the canvas after a frame is shown (section 2.7.1.1, +/// "Disposal method (D)"). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Dispose { + /// 0: leave the canvas as it is. + None, + /// 1: fill the frame's rectangle with the background. Listed choice: + /// transparent black, as browsers do, not the `ANIM` chunk's colour. + Background, +} + +/// The facts of a frame, as the stream gives them; read them after +/// [`Status::FrameDone`] with [`Webp::frame`]. A still image's one frame +/// covers the canvas, with a duration of 0, [`Blend::Overwrite`] and +/// [`Dispose::None`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FrameInfo { + /// Left edge on the canvas (twice the stored `Frame X`). + pub left: u32, + /// Top edge on the canvas. + pub top: u32, + /// Width in pixels. + pub width: u32, + /// Height in pixels. + pub height: u32, + /// How long to show the frame, in milliseconds (0 is implementation + /// defined: 6.19's policy). + pub duration_ms: u32, + /// The blending method. + pub blend: Blend, + /// The disposal method. + pub dispose: Dispose, +} + +/// The global animation parameters of an `ANIM` chunk (section 2.7.1.1). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Animation { + /// The background colour hint, red, green, blue, alpha (the chunk has it + /// in the order blue, green, red, alpha). Not used for compositing. + pub background: [u8; 4], + /// The loop count; 0 means forever. Not used here (6.19's policy). + pub loops: u16, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Phase { + /// The 12 bytes `RIFF`, size, `WEBP`, gathered. + Riff, + /// A chunk header, 8 bytes, gathered. + ChunkHdr, + /// The first 10 bytes of the `VP8X` payload, gathered. + Vp8x, + /// The first 6 bytes of an `ANIM` payload, gathered. + Anim, + /// The first 16 bytes of an `ANMF` payload, gathered. + Anmf, + /// A `VP8L` payload, to the lossless decoder. + Payload, + /// The rest of a payload to skip. + Skip, + /// The padding byte after an odd-sized chunk. + Pad, + /// The end of the RIFF data or a still image's frame: no more frames. + Ended, + /// An error happened. + Spent, +} + +/// What one byte did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Event { + Continue, + Frame, + End, +} + +/// The WebP decoder's state: fixed-size, no allocation. +#[derive(Debug, Clone, Copy)] +pub struct Webp { + config: Config, + /// The file is an animation (the config says its frame count is not + /// known): the work buffer has a frame buffer. + animated: bool, + phase: Phase, + hold: [u8; 16], + have: u8, + /// Bytes of the RIFF payload not yet read, of the current chunk's + /// payload, and of the current `ANMF` chunk's frame data. + riff_left: u32, + chunk_left: u32, + anmf_left: u32, + /// The current chunk has a padding byte after it. + padded: bool, + /// Inside the frame data of an `ANMF` chunk. + in_anmf: bool, + /// No chunk has been read yet. + first_chunk: bool, + /// An `ANIM` chunk was read. + seen_anim: bool, + anim: Animation, + /// The current frame's bitstream was decoded into the frame buffer. + have_image: bool, + /// The frame being read (animations), and the last one completed. + cur: FrameInfo, + last: FrameInfo, + has_frame: bool, + frames: u32, + vp8l: Vp8l, +} + +impl Webp { + /// The last completed frame's facts, if there is one. After + /// `Status::FrameDone` it is that frame's. + pub fn frame(&self) -> Option<&FrameInfo> { + self.has_frame.then_some(&self.last) + } + + /// The animation's global parameters, once its `ANIM` chunk was read. + pub fn animation(&self) -> Option<&Animation> { + self.seen_anim.then_some(&self.anim) + } + + /// How many frames were completed. + pub fn frames_done(&self) -> u32 { + self.frames + } + + fn fail(&mut self, e: Error) -> Error { + self.phase = Phase::Spent; + e + } + + fn hold_at(&self, i: usize) -> u8 { + self.hold.get(i).copied().unwrap_or(0) + } + + fn u32_at(&self, i: usize) -> u32 { + u32::from_le_bytes([ + self.hold_at(i), + self.hold_at(i.saturating_add(1)), + self.hold_at(i.saturating_add(2)), + self.hold_at(i.saturating_add(3)), + ]) + } + + /// A 24-bit little-endian value (section 2.7: `uint24`). + fn u24_at(&self, i: usize) -> u32 { + u32::from_le_bytes([ + self.hold_at(i), + self.hold_at(i.saturating_add(1)), + self.hold_at(i.saturating_add(2)), + 0, + ]) + } + + /// Stores `b` in `hold`; true when `need` bytes are there (and `have` + /// goes back to 0). + fn gather(&mut self, b: u8, need: u8) -> bool { + if let Some(slot) = self.hold.get_mut(usize::from(self.have)) { + *slot = b; + } + self.have = self.have.saturating_add(1); + if self.have >= need { + self.have = 0; + true + } else { + false + } + } + + /// The canvas size in pixels. + fn canvas(&self) -> (u32, u32) { + (self.config.width(), self.config.height()) + } + + /// One byte of the stream. + pub(crate) fn byte(&mut self, b: u8, parts: &mut Parts<'_>) -> Result { + if self.phase == Phase::Riff { + // Section 2.4: `RIFF`, the file size ("starting at offset 8", + // so counting `WEBP`), `WEBP`. + if self.gather(b, 12) { + if self.hold.get(..4) != Some(b"RIFF") || self.hold.get(8..12) != Some(b"WEBP") { + return Err(bad("not a RIFF WEBP file")); + } + self.riff_left = self + .u32_at(4) + .checked_sub(4) + .ok_or(bad("a RIFF size too small to hold WEBP"))?; + if self.riff_left == 0 { + return Err(bad("a WebP file with no chunks")); + } + // No chunk is open yet (as `start` left them). + self.chunk_left = 0; + self.padded = false; + self.in_anmf = false; + self.phase = Phase::ChunkHdr; + } + return Ok(Event::Continue); + } + self.riff_left = self + .riff_left + .checked_sub(1) + .ok_or(bad("bytes past the end of the RIFF data"))?; + if self.in_anmf { + self.anmf_left = self + .anmf_left + .checked_sub(1) + .ok_or(bad("bytes past the end of the frame data"))?; + } + let event = self.dispatch(b, parts)?; + if self.riff_left == 0 { + // The RIFF data is used up (section 2.4: trailing data is + // ignored). + return match event { + Event::Frame => { + self.phase = Phase::Ended; + Ok(Event::Frame) + } + _ if self.frames == 0 => Err(bad("a WebP file with no image data")), + _ => { + self.phase = Phase::Ended; + Ok(Event::End) + } + }; + } + Ok(event) + } + + fn dispatch(&mut self, b: u8, parts: &mut Parts<'_>) -> Result { + match self.phase { + Phase::ChunkHdr => { + if self.gather(b, 8) { + return self.chunk_header(parts); + } + Ok(Event::Continue) + } + Phase::Pad => self.sub_done(parts), + Phase::Vp8x | Phase::Anim | Phase::Anmf | Phase::Payload | Phase::Skip => { + self.chunk_left = self + .chunk_left + .checked_sub(1) + .ok_or(bad("bytes past the end of a chunk"))?; + let phase = self.phase; + match phase { + Phase::Payload => return self.payload_byte(b, parts), + Phase::Vp8x => { + if self.gather(b, 10) { + self.on_vp8x(parts)?; + } + } + Phase::Anim => { + if self.gather(b, 6) { + self.on_anim(); + } + } + Phase::Anmf if self.gather(b, 16) => return self.on_anmf(), + _ => {} + } + if self.chunk_left == 0 { + return self.chunk_end(parts); + } + // The rest of a gathered payload is skipped. + if matches!(self.phase, Phase::Vp8x | Phase::Anim) && self.have == 0 { + self.phase = Phase::Skip; + } + Ok(Event::Continue) + } + Phase::Riff | Phase::Ended | Phase::Spent => Err(bad("a byte in a finished decoder")), + } + } + + /// A chunk header is complete (section 2.3: FourCC, size). + fn chunk_header(&mut self, parts: &mut Parts<'_>) -> Result { + let size = self.u32_at(4); + let mut fourcc = [0u8; 4]; + for (i, f) in fourcc.iter_mut().enumerate() { + *f = self.hold_at(i); + } + // The payload and its padding byte must fit in what contains the + // chunk (the RIFF data, or an ANMF chunk's frame data). + let total = size + .checked_add(size & 1) + .ok_or(bad("a chunk size overflow"))?; + let limit = if self.in_anmf { + self.anmf_left + } else { + self.riff_left + }; + if total > limit { + return Err(bad("a chunk that extends past what contains it")); + } + self.chunk_left = size; + self.padded = size & 1 == 1; + let first = self.first_chunk; + self.first_chunk = false; + // Section 2.7: "'VP8X' ... MUST appear in the order described". + match &fourcc { + b"VP8X" => { + if !first { + return Err(bad("a VP8X chunk that is not the first chunk")); + } + if size < 10 { + return Err(bad("a VP8X chunk of under 10 bytes")); + } + self.phase = Phase::Vp8x; + return Ok(Event::Continue); + } + _ if first && self.animated => { + return Err(bad("an animation whose first chunk is not VP8X")); + } + b"VP8L" | b"VP8 " if first => {} + _ if first => return Err(bad("the first chunk is not VP8X, VP8 or VP8L")), + _ => {} + } + match &fourcc { + // Sections 2.5 and 2.6; lossy decoding is item 6.9. + b"VP8 " if self.animated && !self.in_anmf => { + Err(bad("image data outside a frame in an animation")) + } + b"VP8 " => Err(Error::Unsupported( + "lossy WebP (a VP8 chunk): decoding it is item 6.9", + )), + b"VP8L" if self.animated && !self.in_anmf => { + Err(bad("image data outside a frame in an animation")) + } + b"VP8L" => { + if self.have_image { + return Err(bad("two bitstreams in one frame")); + } + if size == 0 { + return Err(bad("an empty VP8L chunk")); + } + let (w, h) = if self.animated { + (self.cur.width, self.cur.height) + } else { + self.canvas() + }; + self.vp8l = Vp8l::new(w, h); + self.phase = Phase::Payload; + Ok(Event::Continue) + } + // Section 2.7.1.1: `ANIM` "MUST appear if the Animation flag is + // set"; "If the Animation flag is not set and this chunk is + // present, it MUST be ignored". + b"ANIM" if self.animated && !self.in_anmf => { + if size < 6 { + return Err(bad("an ANIM chunk of under 6 bytes")); + } + self.phase = Phase::Anim; + Ok(Event::Continue) + } + b"ANMF" if self.animated && !self.in_anmf => { + if !self.seen_anim { + return Err(bad("an ANMF chunk before the ANIM chunk")); + } + // The frame data is padded chunks, so an ANMF chunk's size + // is even (section 2.7.1.1's note); 16 is the header. + if size < 16 || size & 1 == 1 { + return Err(bad("an ANMF chunk that is odd or under 16 bytes")); + } + self.phase = Phase::Anmf; + Ok(Event::Continue) + } + // `ALPH`, `ICCP`, `EXIF`, `XMP ` and unknown chunks are skipped + // (sections 2.7.1.2 to 2.7.1.6); `ALPH` belongs to `VP8 ` + // (item 6.9) and "A frame containing a 'VP8L' Chunk SHOULD NOT + // contain this chunk". + _ => { + if self.animated && !self.in_anmf && &fourcc == b"ALPH" { + return Err(bad("an ALPH chunk outside a frame in an animation")); + } + self.phase = Phase::Skip; + if self.chunk_left == 0 { + // Nothing to skip: the chunk is over (an empty chunk has + // no padding byte). + self.padded = false; + return self.sub_done(parts); + } + Ok(Event::Continue) + } + } + } + + /// The `VP8X` payload's first 10 bytes are in (section 2.7, Figure 7): + /// flags, 3 reserved bytes, canvas width and height minus one. + fn on_vp8x(&mut self, parts: &mut Parts<'_>) -> Result<(), Error> { + let flags = self.hold_at(0); + let animation = flags & 0x02 != 0; + let w = self.u24_at(4).saturating_add(1); + let h = self.u24_at(7).saturating_add(1); + if (w, h) != self.canvas() { + return Err(bad("the VP8X canvas is not the size the config says")); + } + if animation != self.animated { + return Err(bad("the VP8X animation flag is not what the config says")); + } + if self.animated { + // The canvas starts transparent black (listed choice). + parts.pixels.fill(0); + } + Ok(()) + } + + /// The `ANIM` payload's first 6 bytes (section 2.7.1.1, Figure 8). + fn on_anim(&mut self) { + self.anim = Animation { + background: [ + self.hold_at(2), + self.hold_at(1), + self.hold_at(0), + self.hold_at(3), + ], + loops: u16::from_le_bytes([self.hold_at(4), self.hold_at(5)]), + }; + self.seen_anim = true; + } + + /// The `ANMF` payload's first 16 bytes (section 2.7.1.1, Figure 9). + fn on_anmf(&mut self) -> Result { + let left = self.u24_at(0).saturating_mul(2); + let top = self.u24_at(3).saturating_mul(2); + let width = self.u24_at(6).saturating_add(1); + let height = self.u24_at(9).saturating_add(1); + let flags = self.hold_at(15); + let (cw, ch) = self.canvas(); + // Section 2.7.2: "VP8X.canvasWidth >= frame_right MUST be TRUE". + if left.saturating_add(width) > cw || top.saturating_add(height) > ch { + return Err(bad("a frame that extends past the canvas")); + } + self.cur = FrameInfo { + left, + top, + width, + height, + duration_ms: self.u24_at(12), + blend: if flags & 2 != 0 { + Blend::Overwrite + } else { + Blend::Alpha + }, + dispose: if flags & 1 != 0 { + Dispose::Background + } else { + Dispose::None + }, + }; + self.have_image = false; + self.in_anmf = true; + self.anmf_left = self.chunk_left; + self.chunk_left = 0; + self.padded = false; + self.phase = Phase::ChunkHdr; + if self.anmf_left == 0 { + return Err(bad("a frame with no image data")); + } + Ok(Event::Continue) + } + + /// One byte of a `VP8L` payload. + fn payload_byte(&mut self, b: u8, parts: &mut Parts<'_>) -> Result { + let last = self.chunk_left == 0; + self.vp8l.push(b, last); + let done = if self.animated { + let bytes = usize::try_from(self.cur.width) + .ok() + .and_then(|w| w.checked_mul(usize::try_from(self.cur.height).ok()?)) + .and_then(|n| n.checked_mul(4)) + .ok_or(bad("a frame size overflow"))?; + let target = parts + .frame + .get_mut(..bytes) + .ok_or(bad("a frame larger than the canvas"))?; + self.vp8l.advance(&mut parts.scratch, target)? + } else { + self.vp8l.advance(&mut parts.scratch, &mut *parts.pixels)? + }; + if !done { + if last { + return Err(bad("the VP8L data ends before the image is complete")); + } + return Ok(Event::Continue); + } + if !self.animated { + // A still image: its frame is the canvas. + let (width, height) = self.canvas(); + self.last = FrameInfo { + left: 0, + top: 0, + width, + height, + duration_ms: 0, + blend: Blend::Overwrite, + dispose: Dispose::None, + }; + self.has_frame = true; + self.frames = 1; + self.phase = Phase::Ended; + return Ok(Event::Frame); + } + self.have_image = true; + if self.chunk_left == 0 { + self.chunk_end(parts) + } else { + self.phase = Phase::Skip; + Ok(Event::Continue) + } + } + + /// A chunk's payload is used up: its padding byte, or what is next. + fn chunk_end(&mut self, parts: &mut Parts<'_>) -> Result { + if self.padded { + self.padded = false; + self.phase = Phase::Pad; + Ok(Event::Continue) + } else { + self.sub_done(parts) + } + } + + /// A chunk with its padding is used up: the next chunk header, or the + /// end of a frame. + fn sub_done(&mut self, parts: &mut Parts<'_>) -> Result { + self.phase = Phase::ChunkHdr; + if self.in_anmf && self.anmf_left == 0 { + return self.finish_frame(parts); + } + Ok(Event::Continue) + } + + /// The frame data of an `ANMF` chunk is used up: dispose of the previous + /// frame, then draw this one (section 2.7.2: "Before rendering each + /// frame, the previous frame's Disposal method is applied"). + fn finish_frame(&mut self, parts: &mut Parts<'_>) -> Result { + if !self.have_image { + return Err(bad("a frame with no image data")); + } + let (cw, _) = self.canvas(); + let cw = usize::try_from(cw).map_err(|_| bad("a canvas width overflow"))?; + if self.has_frame && self.last.dispose == Dispose::Background { + let f = self.last; + for row in 0..usize::try_from(f.height).unwrap_or(0) { + if let Some(span) = row_span(f.left, f.top, f.width, row, cw) + && let Some(px) = parts.pixels.get_mut(span) + { + px.fill(0); + } + } + } + let f = self.cur; + let fw = usize::try_from(f.width).map_err(|_| bad("a frame width overflow"))?; + for row in 0..usize::try_from(f.height).unwrap_or(0) { + let src = fw + .checked_mul(row) + .and_then(|o| o.checked_mul(4)) + .and_then(|o| parts.frame.get(o..o.checked_add(fw.checked_mul(4)?)?)); + let dst = row_span(f.left, f.top, f.width, row, cw) + .and_then(|span| parts.pixels.get_mut(span)); + let (Some(src), Some(dst)) = (src, dst) else { + return Err(bad("a frame row outside the buffers")); + }; + match f.blend { + Blend::Overwrite => dst.copy_from_slice(src), + Blend::Alpha => blend_row(dst, src), + } + } + self.last = f; + self.has_frame = true; + self.frames = self.frames.saturating_add(1); + self.have_image = false; + self.in_anmf = false; + Ok(Event::Frame) + } + + /// The invariant of the parts of the state that index. + fn state_inv(&self, groups: usize) -> bool { + let (cw, ch) = self.canvas(); + self.phase == Phase::Spent + || (self.have <= 16 + // Checked, not saturating: `chunk_header` checks the size and + // its padding byte with `checked_add`, and a saturated sum + // would not survive a byte. + && self + .chunk_left + .checked_add(u32::from(self.padded)) + .is_some_and(|t| { + t <= if self.in_anmf { + self.anmf_left + } else { + self.riff_left + } + }) + && (!self.in_anmf || self.anmf_left <= self.riff_left) + // Between chunks no chunk is open. + && (!matches!(self.phase, Phase::ChunkHdr | Phase::Pad) + || (self.chunk_left == 0 && !self.padded)) + && self.cur.left.saturating_add(self.cur.width) <= cw + && self.cur.top.saturating_add(self.cur.height) <= ch + && self.last.left.saturating_add(self.last.width) <= cw + && self.last.top.saturating_add(self.last.height) <= ch + && self.vp8l.inv(groups)) + } +} + +/// The byte range of row `row` of a frame at (`left`, `top`) of `width` +/// pixels in a canvas `cw` pixels wide. +fn row_span( + left: u32, + top: u32, + width: u32, + row: usize, + cw: usize, +) -> Option> { + let y = usize::try_from(top).ok()?.checked_add(row)?; + let start = y + .checked_mul(cw)? + .checked_add(usize::try_from(left).ok()?)? + .checked_mul(4)?; + let end = start.checked_add(usize::try_from(width).ok()?.checked_mul(4)?)?; + Some(start..end) +} + +/// Whether `prefix` (the first bytes of a file) agrees with `want` at `at`: +/// `Ok(true)` if all of `want` is there and equal, `Ok(false)` if the prefix +/// ends first, an error if a byte differs. +fn matches_at(prefix: &[u8], at: usize, want: &[u8]) -> Result { + for (i, w) in want.iter().enumerate() { + match prefix.get(at.saturating_add(i)) { + Some(b) if b == w => {} + Some(_) => return Err(bad("not a WebP file")), + None => return Ok(false), + } + } + Ok(true) +} + +/// A little-endian `u16` of `prefix` at `at`, if it is there. +fn le16(prefix: &[u8], at: usize) -> Option { + prefix + .get(at..)? + .first_chunk::<2>() + .copied() + .map(u16::from_le_bytes) +} + +/// A 24-bit little-endian value of `prefix` at `at`, if it is there. +fn le24(prefix: &[u8], at: usize) -> Option { + let [a, b, c] = *prefix.get(at..)?.first_chunk::<3>()?; + Some(u32::from_le_bytes([a, b, c, 0])) +} + +impl Decoder for Webp { + fn decode_config(prefix: &[u8]) -> Result { + // Section 2.4: `RIFF`, 4 bytes of size, `WEBP`; section 2.3: the + // first chunk's FourCC and size. + let more = Ok(ConfigStatus::NeedMore); + if !matches_at(prefix, 0, b"RIFF")? | !matches_at(prefix, 8, b"WEBP")? { + return more; + } + // The first chunk is `VP8X`, `VP8L` or `VP8 ` (section 2.5 to 2.7): + // all begin `VP8`. + if !matches_at(prefix, 12, b"VP8")? { + return more; + } + let Some(&kind) = prefix.get(15) else { + return more; + }; + let Some(size) = prefix + .get(16..) + .and_then(|s| s.first_chunk::<4>()) + .copied() + .map(u32::from_le_bytes) + else { + return more; + }; + let ready = |w: u32, h: u32, frames: FrameCount| -> Result { + Ok(ConfigStatus::Ready(Config::new( + Format::Webp, + w, + h, + frames, + )?)) + }; + match kind { + // Section 2.7, Figure 7: flags, 3 reserved bytes, canvas width + // and height minus one, 24 bits each. + b'X' => { + if size < 10 { + return Err(bad("a VP8X chunk of under 10 bytes")); + } + let (Some(flags), Some(w), Some(h)) = + (prefix.get(20).copied(), le24(prefix, 24), le24(prefix, 27)) + else { + return more; + }; + let frames = if flags & 0x02 != 0 { + FrameCount::Unknown + } else { + FrameCount::Known(1) + }; + ready(w.saturating_add(1), h.saturating_add(1), frames) + } + // Section 3.4: the signature 0x2f, then 14 bits of width minus + // one and 14 of height minus one, 1 of alpha, 3 of version. + b'L' => { + if let Some(&sig) = prefix.get(20) + && sig != 0x2F + { + return Err(bad("the VP8L signature byte is not 0x2f")); + } + let Some(b) = prefix.get(21..).and_then(|s| s.first_chunk::<4>()) else { + return more; + }; + let v = u32::from_le_bytes(*b); + if v >> 29 != 0 { + return Err(bad("a VP8L version other than 0")); + } + ready( + (v & 0x3FFF).saturating_add(1), + ((v >> 14) & 0x3FFF).saturating_add(1), + FrameCount::Known(1), + ) + } + // RFC 6386 section 9.1: a 3-byte frame tag (bit 0 clear is a key + // frame), the start code 9d 01 2a, then 14 bits of width and 14 + // of height (the top two bits of each are a scale). + b' ' => { + if let Some(&tag) = prefix.get(20) + && tag & 1 != 0 + { + return Err(bad("a VP8 frame that is not a key frame")); + } + if !matches_at(prefix, 23, &[0x9D, 0x01, 0x2A])? { + return more; + } + let (Some(w), Some(h)) = (le16(prefix, 26), le16(prefix, 28)) else { + return more; + }; + ready( + u32::from(w & 0x3FFF), + u32::from(h & 0x3FFF), + FrameCount::Known(1), + ) + } + _ => Err(bad("the first chunk is not VP8X, VP8 or VP8L")), + } + } + + fn work_len(config: &Config) -> Result { + let animated = matches!(config.frames(), FrameCount::Unknown); + work_len(config.width(), config.height(), animated).ok_or(bad("work buffer size overflow")) + } + + fn start(config: &Config) -> Result { + if config.format() != Format::Webp { + return Err(Error::Unsupported("not a WebP config")); + } + config.rgba_len()?; + let blank = FrameInfo { + left: 0, + top: 0, + width: 0, + height: 0, + duration_ms: 0, + blend: Blend::Overwrite, + dispose: Dispose::None, + }; + Ok(Webp { + config: *config, + animated: matches!(config.frames(), FrameCount::Unknown), + phase: Phase::Riff, + hold: [0; 16], + have: 0, + riff_left: 0, + chunk_left: 0, + anmf_left: 0, + padded: false, + in_anmf: false, + first_chunk: true, + seen_anim: false, + anim: Animation { + background: [0; 4], + loops: 0, + }, + have_image: false, + cur: blank, + last: blank, + has_frame: false, + frames: 0, + vp8l: Vp8l::new(0, 0), + }) + } + + fn config(&self) -> &Config { + &self.config + } + + fn step(&mut self, chunk: &[u8], buffers: &mut Buffers<'_>) -> Result { + match self.phase { + Phase::Spent => return Err(Error::Spent), + Phase::Ended => return Err(Error::NoMoreFrames), + _ => {} + } + let (cw, ch) = self.canvas(); + let Some(mut parts) = Parts::new( + &mut *buffers.pixels, + &mut *buffers.work, + cw, + ch, + self.animated, + ) else { + return Err(self.fail(Error::Buffers)); + }; + for (i, &b) in chunk.iter().enumerate() { + match self.byte(b, &mut parts) { + Ok(Event::Continue) => {} + Ok(Event::Frame) => { + return Ok(Step { + consumed: i.saturating_add(1), + status: Status::FrameDone, + }); + } + Ok(Event::End) => return Err(Error::NoMoreFrames), + Err(e) => return Err(self.fail(e)), + } + } + Ok(Step { + consumed: chunk.len(), + status: Status::NeedMore, + }) + } + + fn inv(&self, buffers: &Buffers<'_>) -> bool { + let Ok(plen) = self.config.rgba_len() else { + return false; + }; + let Some(wlen) = work_len(self.config.width(), self.config.height(), self.animated) else { + return false; + }; + let groups = crate::work::group_cap(self.config.width(), self.config.height()) + .and_then(|c| c.checked_mul(crate::work::GROUP_BYTES)) + .unwrap_or(0); + buffers.pixels.len() == plen && buffers.work.len() == wlen && self.state_inv(groups) + } +} + +/// Any state of a decoder of a 2 by 2 canvas in `phase`, which is not the +/// lossless payload (that is `vp8l.rs`'s harnesses), every other field +/// symbolic: the harness assumes `inv` on top. One phase per harness: with +/// the phase symbolic too, CBMC ran out of memory (over 4 GB). +#[cfg(kani)] +pub(crate) fn any_webp(phase: Phase) -> Option { + let frames = if kani::any() { + FrameCount::Unknown + } else { + FrameCount::Known(1) + }; + let config = Config::new(Format::Webp, 2, 2, frames).ok()?; + let mut w = Webp::start(&config).ok()?; + w.phase = phase; + w.hold = kani::any(); + w.have = kani::any(); + w.riff_left = kani::any(); + w.chunk_left = kani::any(); + w.anmf_left = kani::any(); + w.padded = kani::any(); + w.in_anmf = kani::any(); + w.first_chunk = kani::any(); + w.seen_anim = kani::any(); + w.have_image = kani::any(); + w.has_frame = kani::any(); + w.frames = kani::any(); + let any_frame = || FrameInfo { + left: kani::any(), + top: kani::any(), + width: kani::any(), + height: kani::any(), + duration_ms: kani::any(), + blend: if kani::any() { + Blend::Alpha + } else { + Blend::Overwrite + }, + dispose: if kani::any() { + Dispose::Background + } else { + Dispose::None + }, + }; + w.cur = any_frame(); + w.last = any_frame(); + Some(w) +} + +#[cfg(kani)] +impl Webp { + /// The file is an animation (for the harnesses). + pub(crate) fn kani_animated(&self) -> bool { + self.animated + } + + /// Sets the phase (for the harnesses: set after `any_webp`'s `Option` is + /// unwrapped, so that symbolic execution keeps it a constant). + pub(crate) fn kani_set_phase(&mut self, phase: Phase) { + self.phase = phase; + } + + /// The invariant of the state (for the harnesses). + pub(crate) fn kani_inv(&self, groups: usize) -> bool { + self.state_inv(groups) + } +} diff --git a/crates/clazel-webp/src/distance.rs b/crates/clazel-webp/src/distance.rs new file mode 100644 index 000000000..0a645d8b2 --- /dev/null +++ b/crates/clazel-webp/src/distance.rs @@ -0,0 +1,215 @@ +//! LZ77 prefix coding of lengths and distances, and the distance map +//! (RFC 9649 sections 3.6.2.2 and 3.6.2.2.1). + +/// The offsets `(xi, yi)` of distance codes 1 to 120, from Figure 20 of +/// RFC 9649 (section 3.6.2.2.1, "Distance Code to Neighboring Pixel Offset +/// Mapping"). Code `c` is `DISTANCE_MAP[c - 1]`. +pub(crate) const DISTANCE_MAP: [(i8, i8); 120] = [ + (0, 1), + (1, 0), + (1, 1), + (-1, 1), + (0, 2), + (2, 0), + (1, 2), + (-1, 2), + (2, 1), + (-2, 1), + (2, 2), + (-2, 2), + (0, 3), + (3, 0), + (1, 3), + (-1, 3), + (3, 1), + (-3, 1), + (2, 3), + (-2, 3), + (3, 2), + (-3, 2), + (0, 4), + (4, 0), + (1, 4), + (-1, 4), + (4, 1), + (-4, 1), + (3, 3), + (-3, 3), + (2, 4), + (-2, 4), + (4, 2), + (-4, 2), + (0, 5), + (3, 4), + (-3, 4), + (4, 3), + (-4, 3), + (5, 0), + (1, 5), + (-1, 5), + (5, 1), + (-5, 1), + (2, 5), + (-2, 5), + (5, 2), + (-5, 2), + (4, 4), + (-4, 4), + (3, 5), + (-3, 5), + (5, 3), + (-5, 3), + (0, 6), + (6, 0), + (1, 6), + (-1, 6), + (6, 1), + (-6, 1), + (2, 6), + (-2, 6), + (6, 2), + (-6, 2), + (4, 5), + (-4, 5), + (5, 4), + (-5, 4), + (3, 6), + (-3, 6), + (6, 3), + (-6, 3), + (0, 7), + (7, 0), + (1, 7), + (-1, 7), + (5, 5), + (-5, 5), + (7, 1), + (-7, 1), + (4, 6), + (-4, 6), + (6, 4), + (-6, 4), + (2, 7), + (-2, 7), + (7, 2), + (-7, 2), + (3, 7), + (-3, 7), + (7, 3), + (-7, 3), + (5, 6), + (-5, 6), + (6, 5), + (-6, 5), + (8, 0), + (4, 7), + (-4, 7), + (7, 4), + (-7, 4), + (8, 1), + (8, 2), + (6, 6), + (-6, 6), + (8, 3), + (5, 7), + (-5, 7), + (7, 5), + (-7, 5), + (8, 4), + (6, 7), + (-6, 7), + (7, 6), + (-7, 6), + (8, 5), + (7, 7), + (-7, 7), + (8, 6), + (8, 7), +]; + +/// The extra bits that follow a length or distance prefix code +/// (section 3.6.2.2): `(prefix_code - 2) >> 1` for codes of 4 and over, +/// none below. +pub(crate) fn extra_bits(prefix: u32) -> u8 { + if prefix < 4 { + 0 + } else { + u8::try_from(prefix.saturating_sub(2) >> 1).unwrap_or(0) + } +} + +/// The value (a length, or a distance code) of a prefix code with its extra +/// bits read (section 3.6.2.2's pseudocode): +/// +/// ```text +/// if (prefix_code < 4) return prefix_code + 1; +/// int extra_bits = (prefix_code - 2) >> 1; +/// int offset = (2 + (prefix_code & 1)) << extra_bits; +/// return offset + ReadBits(extra_bits) + 1; +/// ``` +pub(crate) fn value(prefix: u32, extra: u32) -> u32 { + if prefix < 4 { + return prefix.saturating_add(1); + } + let bits = u32::from(extra_bits(prefix)); + let offset = 2u32 + .saturating_add(prefix & 1) + .checked_shl(bits) + .unwrap_or(0); + offset.saturating_add(extra).saturating_add(1) +} + +/// The distance, in pixels back in scan-line order, of distance code `code` +/// (at least 1) in an image `xsize` pixels wide (section 3.6.2.2.1): +/// codes over 120 are the distance plus 120; the others go through the map +/// as `xi + yi * xsize`, and a result under 1 is 1. +pub(crate) fn distance(code: u32, xsize: u32) -> u32 { + if code > 120 { + return code.saturating_sub(120); + } + let at = usize::try_from(code.saturating_sub(1)).unwrap_or(0); + let (xi, yi) = DISTANCE_MAP.get(at).copied().unwrap_or((0, 1)); + let d = i64::from(yi) + .saturating_mul(i64::from(xsize)) + .saturating_add(i64::from(xi)); + u32::try_from(d.max(1)).unwrap_or(1) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn values() { + assert_eq!(value(0, 0), 1); + assert_eq!(value(3, 0), 4); + // Table 4: 4 is 5..6 with 1 extra bit, 5 is 7..8, 6 is 9..12. + assert_eq!((value(4, 0), value(4, 1)), (5, 6)); + assert_eq!((value(5, 0), value(5, 1)), (7, 8)); + assert_eq!((value(6, 0), value(6, 3)), (9, 12)); + assert_eq!((value(23, 0), value(23, 1023)), (3073, 4096)); + assert_eq!((value(38, 0), value(38, (1 << 18) - 1)), (524_289, 786_432)); + assert_eq!( + (value(39, 0), value(39, (1 << 18) - 1)), + (786_433, 1_048_576) + ); + assert_eq!(extra_bits(39), 18); + assert_eq!(extra_bits(23), 10); + } + + #[test] + fn map() { + // Code 1 is the pixel above, 2 the one on the left, 3 the top left, + // 4 the top right. + assert_eq!(distance(1, 10), 10); + assert_eq!(distance(2, 10), 1); + assert_eq!(distance(3, 10), 11); + assert_eq!(distance(4, 10), 9); + // (-1, 1) at width 1 is 0, so 1. + assert_eq!(distance(4, 1), 1); + assert_eq!(distance(120, 100), 7 * 100 + 8); + assert_eq!(distance(121, 100), 1); + assert_eq!(distance(1000, 100), 880); + assert_eq!(DISTANCE_MAP.len(), 120); + } +} diff --git a/crates/clazel-webp/src/huffman.rs b/crates/clazel-webp/src/huffman.rs new file mode 100644 index 000000000..c64b763df --- /dev/null +++ b/crates/clazel-webp/src/huffman.rs @@ -0,0 +1,229 @@ +//! Canonical prefix codes: construction from code lengths and decoding one +//! symbol at a time, without tables (RFC 9649 section 3.7). +//! +//! RFC 9649 section 3.7.1: "Most of the data is coded using a canonical +//! prefix code". The code of a symbol is its position among the symbols of +//! its length, sorted by symbol value, after all shorter codes; the first +//! bit read from the stream is the **most** significant bit of the code (the +//! DEFLATE convention, which libwebp's tables implement by reversing the +//! code). The decoder here is the classic one over per-length counts (as +//! zlib's `puff.c`): no table, so a code costs `2 * (alphabet + 16)` bytes +//! and decoding takes at most 15 steps. +//! +//! Listed choices, from libwebp's `VP8LBuildHuffmanTable` where the RFC says +//! less: +//! - a code with **all lengths zero** is an error (section 3.7.2.1.1's note +//! says an empty code is coded as a single symbol 0, so a zero-symbol code +//! is never valid); +//! - a code with **exactly one symbol** takes zero bits, whatever length +//! (1 to 15) its entry has (section 3.7.2.1: "even when no bits are +//! consumed when that single leaf node tree is used"); +//! - every other code must be **complete**: the lengths fill the code space +//! exactly (section 3.7.2.1: "The described tree must be a complete +//! binary tree"); over-subscribed and incomplete codes are errors. +//! +//! A code is stored as `counts` (16 little-endian `u16`: the number of +//! symbols of each length 1 to 15, and entry 0 is 1 for the one-symbol code +//! and 0 otherwise) and `symbols` (little-endian `u16`, sorted by length and +//! then symbol value). + +/// The longest code length: RFC 9649 section 3.7.2.1.2 ("Values [1..15]"). +pub(crate) const MAX_LEN: u8 = 15; + +/// Reads the `i`th little-endian `u16` of `b` (0 if out of range). +pub(crate) fn get16(b: &[u8], i: usize) -> u16 { + i.checked_mul(2) + .and_then(|o| b.get(o..o.checked_add(2)?)) + .and_then(|s| <[u8; 2]>::try_from(s).ok()) + .map_or(0, u16::from_le_bytes) +} + +/// Writes the `i`th little-endian `u16` of `b`; false if out of range. +pub(crate) fn set16(b: &mut [u8], i: usize, v: u16) -> bool { + let slot = i + .checked_mul(2) + .and_then(|o| b.get_mut(o..o.checked_add(2)?)); + match slot { + Some(s) => { + s.copy_from_slice(&v.to_le_bytes()); + true + } + None => false, + } +} + +/// Builds the code for `lens` (one length per symbol of the alphabet, each +/// 0 to 15) into `counts` (at least 32 bytes) and `symbols` (at least +/// `2 * lens.len()` bytes). +pub(crate) fn build( + lens: &[u8], + counts: &mut [u8], + symbols: &mut [u8], +) -> Result<(), &'static str> { + // The histogram of lengths. + let mut hist = [0u16; 16]; + for &l in lens { + match hist.get_mut(usize::from(l)) { + Some(h) if l <= MAX_LEN => *h = h.saturating_add(1), + _ => return Err("a prefix code length over 15"), + } + } + let total = u16::try_from(lens.len()).map_err(|_| "an alphabet over 65535 symbols")?; + let unused = hist.first().copied().unwrap_or(0); + let used = total.saturating_sub(unused); + if used == 0 { + return Err("a prefix code with no symbols"); + } + if used == 1 { + // One symbol: zero bits (RFC 9649 section 3.7.2.1). + let sym = lens.iter().position(|&l| l != 0).unwrap_or(0); + for i in 0..16 { + let v = u16::from(i == 0); + if !set16(counts, i, v) { + return Err("a counts array that is too small"); + } + } + if !set16(symbols, 0, u16::try_from(sym).unwrap_or(0)) { + return Err("a symbol array that is too small"); + } + return Ok(()); + } + // Complete and not over-subscribed: the code space left after each + // length, doubled at each step, ends at exactly 0. + let mut left: i32 = 1; + for &h in hist.iter().skip(1) { + left = left.wrapping_mul(2).wrapping_sub(i32::from(h)); + if left < 0 { + return Err("an over-subscribed prefix code"); + } + } + if left != 0 { + return Err("an incomplete prefix code"); + } + // The first slot of each length in `symbols`. + let mut offset = [0u16; 16]; + let mut run = 0u16; + for (len, &h) in hist.iter().enumerate().skip(1) { + if let Some(o) = offset.get_mut(len) { + *o = run; + } + run = run.saturating_add(h); + } + for (sym, &l) in lens.iter().enumerate() { + if l == 0 { + continue; + } + let Some(o) = offset.get_mut(usize::from(l)) else { + return Err("a prefix code length over 15"); + }; + let at = usize::from(*o); + *o = o.saturating_add(1); + if !set16(symbols, at, u16::try_from(sym).unwrap_or(0)) { + return Err("a symbol array that is too small"); + } + } + for (i, &h) in hist.iter().enumerate() { + let v = if i == 0 { 0 } else { h }; + if !set16(counts, i, v) { + return Err("a counts array that is too small"); + } + } + Ok(()) +} + +/// What one try at reading a symbol gave. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Decoded { + /// The symbol and the bits its code took. + Symbol(u16, u8), + /// The code is longer than the `avail` bits given: more are needed. + More, + /// The bits match no code: the tables are not a built code. + Invalid, +} + +/// Decodes one symbol. `bits` holds the next bits of the stream, the next one +/// in bit 0, and `avail` of them (at most 32) are real. +pub(crate) fn decode(counts: &[u8], symbols: &[u8], bits: u32, avail: u8) -> Decoded { + if get16(counts, 0) == 1 { + return Decoded::Symbol(get16(symbols, 0), 0); + } + let mut code: u32 = 0; + let mut first: u32 = 0; + let mut index: u32 = 0; + for len in 1..=MAX_LEN { + if len > avail { + return Decoded::More; + } + let bit = bits + .checked_shr(u32::from(len).wrapping_sub(1)) + .unwrap_or(0) + & 1; + code |= bit; + let count = u32::from(get16(counts, usize::from(len))); + if code < first.wrapping_add(count) { + let at = index.wrapping_add(code.wrapping_sub(first)); + let sym = usize::try_from(at).map_or(0, |a| get16(symbols, a)); + return Decoded::Symbol(sym, len); + } + index = index.wrapping_add(count); + first = first.wrapping_add(count).wrapping_shl(1); + code = code.wrapping_shl(1); + } + Decoded::Invalid +} + +#[cfg(test)] +mod tests { + use super::*; + + fn build_vec(lens: &[u8]) -> Result<(Vec, Vec), &'static str> { + let mut counts = vec![0u8; 32]; + let mut symbols = vec![0u8; lens.len().max(1).saturating_mul(2)]; + build(lens, &mut counts, &mut symbols)?; + Ok((counts, symbols)) + } + + #[test] + fn rfc_style_example() { + // Lengths 2, 1, 3, 3 (symbols 0 to 3): codes B=0, A=10, C=110, D=111 + // (symbol 1 has the shortest code, then 0, then 2 and 3). + let (c, s) = build_vec(&[2, 1, 3, 3]).unwrap_or_default(); + // Bits are read first bit first: "0" is symbol 1. + assert_eq!(decode(&c, &s, 0b0, 1), Decoded::Symbol(1, 1)); + // "10" first bit 1, second bit 0: bits value 0b01. + assert_eq!(decode(&c, &s, 0b01, 2), Decoded::Symbol(0, 2)); + // "110": first, second, third bits 1,1,0: value 0b011. + assert_eq!(decode(&c, &s, 0b011, 3), Decoded::Symbol(2, 3)); + // "111". + assert_eq!(decode(&c, &s, 0b111, 3), Decoded::Symbol(3, 3)); + // Not enough bits. + assert_eq!(decode(&c, &s, 0b1, 1), Decoded::More); + assert_eq!(decode(&c, &s, 0b11, 2), Decoded::More); + } + + #[test] + fn single_symbol_takes_no_bits() { + let (c, s) = build_vec(&[0, 0, 5, 0]).unwrap_or_default(); + assert_eq!(decode(&c, &s, 0, 0), Decoded::Symbol(2, 0)); + let (c, s) = build_vec(&[0, 0, 0, 15]).unwrap_or_default(); + assert_eq!(decode(&c, &s, 0xFFFF, 15), Decoded::Symbol(3, 0)); + } + + #[test] + fn refuses_bad_codes() { + assert!(build_vec(&[0, 0, 0]).is_err()); + assert!(build_vec(&[1, 1, 1]).is_err()); + assert!(build_vec(&[2, 2, 2]).is_err()); + assert!(build_vec(&[1, 2]).is_err()); + assert!(build_vec(&[16, 1]).is_err()); + assert!(build_vec(&[1, 1]).is_ok()); + } + + #[test] + fn two_symbols_lower_first() { + let (c, s) = build_vec(&[0, 1, 0, 1]).unwrap_or_default(); + assert_eq!(decode(&c, &s, 0, 1), Decoded::Symbol(1, 1)); + assert_eq!(decode(&c, &s, 1, 1), Decoded::Symbol(3, 1)); + } +} diff --git a/crates/clazel-webp/src/lib.rs b/crates/clazel-webp/src/lib.rs new file mode 100644 index 000000000..69b7f96ad --- /dev/null +++ b/crates/clazel-webp/src/lib.rs @@ -0,0 +1,124 @@ +//! A WebP decoder (phase-6.md item 6.8): the RIFF container, animation +//! frames composited onto a canvas, and VP8L lossless decoding, as a +//! resumable step function over caller-owned buffers implementing +//! [`clazel_image_shape::Decoder`]. Lossy `VP8 ` data is item 6.9; here it is +//! recognised and refused. +//! +//! # The standard +//! +//! *RFC 9649: WebP Image Format*, Zern, Massimino and Alakuijala, November +//! 2024, (2,418 lines, fetched +//! 2026-11-18, SHA-256 +//! `f4d7cd92b015c0b340f10a932bb03e564a758a17df50cbb992c16d8459d60e58`). Its +//! section 2 is the container ("WebP Container Specification"), section 3 the +//! lossless bitstream ("Specification for WebP Lossless Bitstream"). Section +//! numbers in the comments are that document's: 2.3 RIFF File Format, 2.4 +//! WebP File Header, 2.5 Simple File Format (Lossy), 2.6 Simple File Format +//! (Lossless), 2.7 Extended File Format (2.7.1.1 Animation, 2.7.1.2 Alpha, +//! 2.7.1.6 Unknown Chunks, 2.7.2 Canvas Assembly from Frames), 3.4 RIFF +//! Header, 3.5 Transforms (3.5.1 Predictor, 3.5.2 Color, 3.5.3 Subtract +//! Green, 3.5.4 Color Indexing), 3.6 Image Data (3.6.2.2 LZ77 Backward +//! Reference and 3.6.2.2.1 Distance Mapping, 3.6.2.3 Color Cache Coding), +//! 3.7 Entropy Code (3.7.2.1 Decoding and Building the Prefix Codes, 3.7.2.2 +//! Decoding of Meta Prefix Codes, 3.7.2.3 Decoding Entropy-Coded Image +//! Data), 3.8 Overall Structure of the Format. Where the RFC is silent or +//! loose, the decoder follows libwebp, and each such place is a listed +//! choice below, commented where it takes effect. +//! +//! # What it decodes +//! +//! - **Container**: `RIFF` size `WEBP`, then the simple lossless format +//! (`VP8L`) or the extended one (`VP8X`, flags and canvas size, with +//! `ICCP`, `EXIF`, `XMP ` and unknown chunks skipped, `ALPH` skipped). +//! `VP8 ` (lossy) is recognised, its size read by `decode_config`, and +//! `step` says [`Error::Unsupported`](clazel_image_shape::Error::Unsupported). +//! - **Animation**: `ANIM` and `ANMF`. **Every frame** is decoded into the +//! frame buffer and composited onto the canvas (the pixel buffer, straight +//! RGBA8) with the previous frame's disposal method applied first; +//! [`Status::FrameDone`](clazel_image_shape::Status::FrameDone) means the +//! pixel buffer holds the canvas as displayed with that frame and +//! [`Webp::frame`] has the frame's duration, blending, disposal and +//! rectangle. The first `FrameDone` is the first composited frame that +//! decision 6.0(d)'s option A shows. +//! - **VP8L** (`vp8l.rs`): the four transforms (predictor with its fourteen +//! modes, colour, subtract green, colour indexing with pixel bundling), the +//! colour cache, meta prefix codes (entropy image), prefix codes (simple +//! and normal code length codes), LZ77 backward references with the +//! distance map. +//! +//! # Listed choices +//! +//! 1. **The canvas of an animation starts transparent black and the +//! background colour of `ANIM` is ignored** (section 2.7.1.1 calls it a +//! hint: "Viewer applications SHOULD treat the background color value as +//! a hint and are not required to use it"), and disposal method 1 +//! ("Dispose to the background color") clears the frame's rectangle to +//! transparent black: what browsers do. The loop count is not read: +//! looping is policy for 6.19. +//! 2. **Alpha blending** (blending method 0) is done on the stored values, +//! not in linear light (section 2.7.1.1 says "SHOULD be done in linear +//! color space"; libwebp and browsers do not), with libwebp's integer +//! arithmetic (`compose.rs`), except that a destination with alpha 0 takes +//! the source exactly. +//! 3. **A still image** is decoded straight into the pixel buffer. The +//! canvas of a `VP8X` file, or the size in the `VP8L` header, is the +//! image's size, and the two must agree. `alpha_is_used` is "a hint only" +//! (section 3.4) and ignored: the alpha channel is used as decoded. +//! 4. **`ANMF` chunks in a still image** are skipped, `ANIM` too ("If the +//! Animation flag is not set and this chunk is present, it MUST be +//! ignored"). In an animation, image data outside a frame is an error, an +//! `ANMF` before the `ANIM` is an error, and each frame needs exactly one +//! bitstream, which must be the size of the frame; a frame must lie +//! inside the canvas (section 2.7.2). +//! 5. **`ALPH`** is skipped: it belongs with lossy data (item 6.9), and "A +//! frame containing a 'VP8L' Chunk SHOULD NOT contain this chunk". +//! 6. **The RIFF size** ends the stream: trailing data is ignored (section +//! 2.4: "Readers MAY parse such files, ignoring the trailing data"), a +//! chunk that does not fit in what contains it is an error, and the +//! padding byte of an odd chunk is read and ignored. +//! 7. **Prefix codes** (`huffman.rs`): all-zero lengths and over-subscribed or +//! incomplete codes are errors, a single symbol takes zero bits; +//! `max_symbol` and the alphabet sizes as section 3.7.2.1.2; a symbol of a +//! simple code past the alphabet is ignored (libwebp's behaviour). +//! 8. **Predictor modes 14 and 15**, which section 3.5.1 does not define, +//! predict like mode 0 (libwebp's). +//! 9. **Prefix code groups**: at most `MAX_GROUPS` = 1,024 groups, and no more +//! than the number of blocks of 4 by 4 pixels the image has; an entropy +//! image that names more is +//! [`Error::Unsupported`](clazel_image_shape::Error::Unsupported) +//! (libwebp's encoder numbers groups densely, so this is not met by files +//! it makes). Section 3.7.2.2.2 reads "num_prefix_groups = max(entropy +//! image) + 1", all of them present in the stream. +//! 10. **Reading past the data**: a chunk that ends before the image is +//! complete is an error (`Malformed`), also in the middle of a code (the +//! decoder works on exactly the chunk's bytes). +//! 11. **A backward reference** whose distance reaches before the first pixel +//! or whose length passes the last is an error (libwebp's), and copies +//! proceed pixel by pixel, so an overlapping reference repeats. +//! 12. **Work space**: [`work_len`](clazel_image_shape::Decoder::work_len) is +//! about 11.6 kB (colour cache, code lengths, colour table) plus 12 bytes +//! per block of 4 by 4 pixels (the three sub-images), plus 6.4 kB per +//! prefix code group held (at most 1,024), plus, for an animation, a +//! frame buffer as large as the canvas (`work.rs` has the table). +//! +//! # Verification +//! +//! Tier 1 target (`Cargo.toml`): the Kani harnesses of `proofs.rs`, listed +//! with their bounds in the README. Every index and every arithmetic step in +//! the crate is checked (`get`, `checked_*`, `wrapping_*`: the workspace lints +//! deny the unchecked forms), so the harnesses prove the absence of panics +//! and the stated properties of the parts, and `inv` is about the ranges of +//! the counters. + +mod bits; +mod compose; +mod decoder; +mod distance; +mod huffman; +#[cfg(kani)] +mod proofs; +mod transform; +mod vp8l; +mod work; + +pub use decoder::{Animation, Blend, Dispose, FrameInfo, Webp}; diff --git a/crates/clazel-webp/src/proofs.rs b/crates/clazel-webp/src/proofs.rs new file mode 100644 index 000000000..16391577d --- /dev/null +++ b/crates/clazel-webp/src/proofs.rs @@ -0,0 +1,620 @@ +//! Kani harnesses (Tier 1). Compiled only under `cargo kani`. +//! +//! Sixty-nine harnesses, bounded as the README lists. Under Kani the colour +//! cache is 8 entries (`MAX_CACHE_BITS` 3 in `work.rs`) so that a green +//! alphabet is 288 symbols and a prefix code group 2,352 bytes; the code is +//! the same code with other constants. +//! +//! **Parts of the lossless decoder** +//! - [`huffman_build_any`]: prefix code construction over **any code lengths** +//! (6 symbols, every `u8` length): no panic; an error if any length is over +//! 15; on `Ok` the code is complete (Kraft sum exactly 1) or a single +//! symbol, and the symbol table holds each used symbol exactly once. +//! - [`huffman_decode_any`]: decoding any bits with any number of them +//! available, from any built code: no panic, a symbol of the alphabet whose +//! length is the code's, never more bits than are available, and `More` only +//! when fewer than 15 are. +//! - [`distance_value_any`], [`distance_map_any`]: the LZ77 prefix coding +//! (every prefix code 0 to 39 with any extra bits) and the distance map +//! (every distance code, every width up to 2^14): never 0, in range. +//! - [`predict_any`]: the predictor of every mode over any four pixels. +//! - [`predictor_row_any`], [`color_row_any`], [`green_row_any`]: each inverse +//! transform over **any row** of up to 3 pixels, any block size bits, any +//! transform data and any pixels: no panic, the right pixels written. +//! - [`palette_row_any`], [`palette_in_place_any`]: colour indexing with pixel +//! bundling, every bundling width, 4 pixels in 2 rows, expanded in place +//! from the last row to the first, equal to the expansion from a copy. +//! - [`blend_px_any`]: alpha blending of any two pixels. +//! +//! **The step function, in parts** (the whole `step` does not fit the model +//! checker: README). One harness per phase, role or transform, mostly made by +//! the macros below: with the phase symbolic, CBMC ran out of memory. +//! - `act_harness!`: one atomic action of the lossless decoder in one phase +//! (40 harnesses, `act_signature` to `act_dist_extra_palette`; the two +//! distance phases copy into the image of the role, so they have one per +//! role), over any state satisfying `Vp8l::inv`, any bits and **any +//! contents** of every scratch region: no panic, `inv` kept on `Ok`. +//! `act_build_code` bounds the alphabet to 8 symbols (`huffman_build_any` +//! covers `build` itself). +//! - `finish_harness!` and [`finish_main_predictor`]: finishing an image of +//! each role (9 harnesses: colour table, entropy image, the two transform +//! images, and the main image with no transform or one of each kind; the +//! loop that inverts several depends only on each transform and the +//! pixels). `finish_main_predictor` is at exactly 2 by 2 pixels +//! (`predictor_row_any` covers rows of 3): the same. +//! - `container_harness!`: one byte of the container in each phase but the +//! lossless payload (9 harnesses), on a 2 by 2 canvas: the same for +//! `Webp::state_inv`. +//! +//! **Bounds that keep them in memory** (measured under a 4 GB cap): +//! - The pixel, finish and container harnesses use MiniSat: CaDiCaL and +//! kissat each ran out of 4 GB on `act_red`, which MiniSat proves in 30 +//! seconds. The others use kissat. +//! - The finish and container harnesses unwind 6 (images of at most 2 rows of +//! 4 pixels; at 40, loops with symbolic bounds unrolled past memory). +//! - Where the colour table is written or indexed at a symbolic position (the +//! palette role's distance phases, the main image's colour indexing), it is +//! 4 colours (`with_scratch`), the bound on `ncolors`. + +use crate::compose::blend_px; +use crate::decoder::{Phase, any_webp}; +use crate::distance; +use crate::huffman::{self, Decoded, get16}; +use crate::transform::{self, BLACK, px}; +use crate::vp8l::{P, Role, any_vp8l}; +use crate::work::{ + CACHE_ENTRIES, CL_ALPHABET, COUNTS_BYTES, GROUP_BYTES, MAX_GREEN, PALETTE_BYTES, Parts, Scratch, +}; + +/// Prefix code construction over any six code lengths. +#[kani::proof] +#[kani::unwind(18)] +fn huffman_build_any() { + let lens: [u8; 6] = kani::any(); + let mut counts = [0u8; COUNTS_BYTES]; + let mut symbols: [u8; 12] = kani::any(); + let result = huffman::build(&lens, &mut counts, &mut symbols); + if lens.iter().any(|&l| l > 15) { + assert!(result.is_err()); + } + if result.is_ok() { + let used = lens.iter().filter(|&&l| l != 0).count(); + assert!(used >= 1); + if get16(&counts, 0) == 1 { + assert!(used == 1); + let s = usize::from(get16(&symbols, 0)); + assert!(s < 6 && lens.get(s).copied().unwrap_or(0) != 0); + } else { + let mut kraft = 0u32; + let mut total = 0usize; + for len in 1..16usize { + let c = u32::from(get16(&counts, len)); + kraft += c << (15 - len); + total += usize::from(get16(&counts, len)); + } + assert!(kraft == 1 << 15); + assert!(total == used); + // Each used symbol appears once among the first `used` slots. + for (s, &l) in lens.iter().enumerate() { + if l != 0 { + let times = (0..used) + .filter(|&i| usize::from(get16(&symbols, i)) == s) + .count(); + assert!(times == 1); + } + } + } + } +} + +/// Decoding one symbol from any built code, any bits, any number available. +#[kani::proof] +#[kani::unwind(18)] +fn huffman_decode_any() { + let lens: [u8; 6] = kani::any(); + let mut counts = [0u8; COUNTS_BYTES]; + let mut symbols = [0u8; 12]; + if huffman::build(&lens, &mut counts, &mut symbols).is_ok() { + let bits: u32 = kani::any(); + let avail: u8 = kani::any(); + kani::assume(avail <= 32); + let lone = get16(&counts, 0) == 1; + match huffman::decode(&counts, &symbols, bits, avail) { + Decoded::Symbol(s, len) => { + let s = usize::from(s); + assert!(s < 6); + assert!(len <= avail || lone); + if lone { + assert!(len == 0); + } else { + assert!(len >= 1 && len <= 15); + assert!(lens.get(s).copied() == Some(len)); + } + } + Decoded::More => assert!(!lone && avail < 15), + Decoded::Invalid => assert!(false), + } + } +} + +/// The LZ77 prefix coding of every prefix code with any extra bits. +#[kani::proof] +fn distance_value_any() { + let prefix: u32 = kani::any(); + kani::assume(prefix < 40); + let bits = distance::extra_bits(prefix); + assert!(bits <= 18); + let extra: u32 = kani::any(); + kani::assume(u64::from(extra) < (1u64 << bits)); + let v = distance::value(prefix, extra); + // Section 3.6.2.2, Table 4: values 1 to 1,048,576. + assert!(v >= 1 && v <= 1_048_576); + if prefix < 4 { + assert!(v == prefix + 1); + } +} + +/// The distance of every distance code at every width. +#[kani::proof] +fn distance_map_any() { + let code: u32 = kani::any(); + kani::assume(code >= 1); + let xsize: u32 = kani::any(); + kani::assume(xsize >= 1 && xsize <= 16384); + let d = distance::distance(code, xsize); + assert!(d >= 1); + if code > 120 { + assert!(d == code - 120); + } else { + // At most 7 rows and 8 columns away. + assert!(u64::from(d) <= 7 * u64::from(xsize) + 8); + } +} + +/// The predictor of every mode over any four pixels. +#[kani::proof] +fn predict_any() { + let mode: u8 = kani::any(); + let l: [u8; 4] = kani::any(); + let t: [u8; 4] = kani::any(); + let tr: [u8; 4] = kani::any(); + let tl: [u8; 4] = kani::any(); + let p = transform::predict(mode, l, t, tr, tl); + match mode & 15 { + 0 | 14 | 15 => assert!(p == BLACK), + 1 => assert!(p == l), + 2 => assert!(p == t), + 3 => assert!(p == tr), + 4 => assert!(p == tl), + 7 => { + for i in 0..4 { + let want = (u16::from(l[i]) + u16::from(t[i])) / 2; + assert!(u16::from(p[i]) == want); + } + } + 11 => assert!(p == l || p == t), + _ => {} + } +} + +/// The inverse predictor transform of any row of 3 pixels. +#[kani::proof] +#[kani::unwind(13)] +fn predictor_row_any() { + let mut buf: [u8; 24] = kani::any(); + let before = buf; + let y: usize = kani::any(); + kani::assume(y < 2); + let bits: u8 = kani::any(); + kani::assume(bits >= 2 && bits <= 9); + let modes: [u8; 4] = kani::any(); + // 3 pixels are at most one block of 4 wide and, in 2 rows, one high. + assert!(transform::predictor_row(&mut buf, 3, y, bits, 1, &modes)); + let other = 1 - y; + assert!(buf[other * 12..other * 12 + 12] == before[other * 12..other * 12 + 12]); + if y == 0 { + // The top-left pixel is predicted black. + let r = px(&before, 0); + let want = [r[0], r[1], r[2], r[3].wrapping_add(255)]; + assert!(px(&buf, 0) == want); + } +} + +/// The inverse colour transform of any row of 3 pixels. +#[kani::proof] +#[kani::unwind(6)] +fn color_row_any() { + let mut buf: [u8; 12] = kani::any(); + let before = buf; + let bits: u8 = kani::any(); + kani::assume(bits >= 2 && bits <= 9); + let elems: [u8; 4] = kani::any(); + assert!(transform::color_row(&mut buf, 3, 0, bits, 1, &elems)); + for i in 0..3 { + let (a, b) = (px(&before, i), px(&buf, i)); + assert!(a[1] == b[1] && a[3] == b[3]); + } +} + +/// The inverse subtract green transform of any row of 3 pixels. +#[kani::proof] +#[kani::unwind(6)] +fn green_row_any() { + let mut buf: [u8; 12] = kani::any(); + let before = buf; + assert!(transform::green_row(&mut buf, 3, 0)); + for i in 0..3 { + let (a, b) = (px(&before, i), px(&buf, i)); + assert!(b == [a[0].wrapping_add(a[1]), a[1], a[2].wrapping_add(a[1]), a[3]]); + } +} + +/// Colour indexing of one row of up to 8 pixels, every bundling width. +#[kani::proof] +#[kani::unwind(10)] +fn palette_row_any() { + let bits: u8 = kani::any(); + kani::assume(bits <= 3); + let n: usize = kani::any(); + kani::assume(n <= 4); + let w: usize = kani::any(); + kani::assume(w >= 1 && w <= 8); + let table: [u8; 16] = kani::any(); + let mut buf: [u8; 32] = kani::any(); + let orig = buf; + assert!(transform::palette_row(&mut buf, 0, 0, w, bits, &table, n)); + for x in 0..w { + let g = usize::from(px(&orig, x >> bits)[1]); + let per = 1usize << bits; + let width = 8 >> bits; + let idx = (g >> (width * (x % per))) & ((1 << width) - 1); + let want = if idx < n { px(&table, idx) } else { [0; 4] }; + // Pixels are written last to first, each reading only packed pixels at + // or before its own position, which are not yet overwritten. + assert!(px(&buf, x) == want); + } +} + +/// Colour indexing of two rows of 4 in place, last row first. +#[kani::proof] +#[kani::unwind(10)] +fn palette_in_place_any() { + let bits: u8 = kani::any(); + kani::assume(bits <= 3); + let n: usize = kani::any(); + kani::assume(n <= 4); + let table: [u8; 16] = kani::any(); + let mut buf: [u8; 32] = kani::any(); + let orig = buf; + let (w, h) = (4usize, 2usize); + let pw = (w + (1 << bits) - 1) >> bits; + for y in (0..h).rev() { + assert!(transform::palette_row( + &mut buf, + y * pw, + y * w, + w, + bits, + &table, + n + )); + } + for y in 0..h { + for x in 0..w { + let g = usize::from(px(&orig, y * pw + (x >> bits))[1]); + let per = 1usize << bits; + let width = 8 >> bits; + let idx = (g >> (width * (x % per))) & ((1 << width) - 1); + let want = if idx < n { px(&table, idx) } else { [0; 4] }; + assert!(px(&buf, y * w + x) == want); + } + } +} + +/// Alpha blending of any two pixels. +#[kani::proof] +fn blend_px_any() { + let src: [u8; 4] = kani::any(); + let dst: [u8; 4] = kani::any(); + let out = blend_px(src, dst); + if src[3] == 255 { + assert!(out == src); + } + if src[3] == 0 { + assert!(out == dst); + } + if src[3] != 0 && src[3] != 255 && dst[3] != 0 { + assert!(out[3] >= src[3]); + } +} + +/// The scratch regions as separate arrays (the real buffer is one, cut by +/// `Parts::new`, whose lengths are in `inv` and in a unit test; the model +/// checker follows every access into one big object). +/// +/// With `small_palette`, the colour table is 4 colours (16 bytes), the +/// harness bound on `ncolors`: a symbolic index into the full table ran out +/// of 4 GB where the table is written or read at a symbolic position. +fn with_scratch(small_palette: bool, f: impl FnOnce(&mut Scratch<'_>, &mut [u8; 16])) { + let mut cache: [u8; 4 * CACHE_ENTRIES] = kani::any(); + let mut lens: [u8; MAX_GREEN] = kani::any(); + let mut cl_lens: [u8; CL_ALPHABET] = kani::any(); + let mut cl_counts: [u8; COUNTS_BYTES] = kani::any(); + let mut cl_syms: [u8; 2 * CL_ALPHABET] = kani::any(); + let mut palette_full: [u8; PALETTE_BYTES] = kani::any(); + let mut palette_small: [u8; 16] = kani::any(); + let palette: &mut [u8] = if small_palette { + &mut palette_small + } else { + &mut palette_full + }; + let mut pred: [u8; 4] = kani::any(); + let mut color: [u8; 4] = kani::any(); + let mut meta: [u8; 4] = kani::any(); + let mut groups: [u8; GROUP_BYTES] = kani::any(); + let mut target: [u8; 16] = kani::any(); + let mut s = Scratch { + cache: &mut cache, + lens: &mut lens, + cl_lens: &mut cl_lens, + cl_counts: &mut cl_counts, + cl_syms: &mut cl_syms, + palette, + pred: &mut pred, + color: &mut color, + meta: &mut meta, + groups: &mut groups, + }; + f(&mut s, &mut target); +} + +/// One action of the lossless decoder in `$phase`, over any state satisfying +/// `inv`, any bits and any scratch contents: no panic, `inv` kept on `Ok`. +/// One harness per phase (`any_vp8l` says why). +macro_rules! act_harness { + ($name:ident, $phase:ident, $unwind:literal, $alphabet:expr, $solver:ident) => { + act_harness!(@go $name, $phase, None, $unwind, $alphabet, $solver); + }; + ($name:ident, $phase:ident, $role:ident, $unwind:literal, $alphabet:expr, $solver:ident) => { + act_harness!(@go $name, $phase, Some(Role::$role), $unwind, $alphabet, $solver); + }; + (@go $name:ident, $phase:ident, $role:expr, $unwind:literal, $alphabet:expr, $solver:ident) => { + #[kani::proof] + #[kani::solver($solver)] + #[kani::unwind($unwind)] + fn $name() { + let mut v = any_vp8l(P::$phase, $role, GROUP_BYTES, $alphabet); + with_scratch(matches!($role, Some(Role::Palette)), |s, target| { + if v.kani_act(s, target).is_ok() { + assert!(v.kani_inv(GROUP_BYTES)); + } + }); + } + }; +} + +/// No bound on the alphabet beyond `inv` (at most `MAX_GREEN`). +const ANY_ALPHABET: u16 = u16::MAX; + +// The header, transform, colour cache and meta prefix reads. +act_harness!(act_signature, Signature, 20, ANY_ALPHABET, kissat); +act_harness!(act_width, Width, 20, ANY_ALPHABET, kissat); +act_harness!(act_height, Height, 20, ANY_ALPHABET, kissat); +act_harness!(act_version, Version, 20, ANY_ALPHABET, kissat); +act_harness!(act_transform_flag, TransformFlag, 20, ANY_ALPHABET, kissat); +act_harness!(act_transform_type, TransformType, 20, ANY_ALPHABET, kissat); +act_harness!(act_transform_bits, TransformBits, 20, ANY_ALPHABET, kissat); +act_harness!(act_palette_size, PaletteSize, 20, ANY_ALPHABET, kissat); +act_harness!(act_cache_flag, CacheFlag, 20, ANY_ALPHABET, kissat); +act_harness!(act_cache_bits, CacheBits, 20, ANY_ALPHABET, kissat); +act_harness!(act_meta_flag, MetaFlag, 20, ANY_ALPHABET, kissat); +act_harness!(act_meta_bits, MetaBits, 20, ANY_ALPHABET, kissat); + +// Reading a prefix code. `CodeKind` sets the alphabet itself (up to 288 +// symbols, whose lengths it clears). +act_harness!(act_code_kind, CodeKind, 300, ANY_ALPHABET, kissat); +act_harness!(act_simple_count, SimpleCount, 20, ANY_ALPHABET, kissat); +act_harness!(act_simple_first, SimpleFirst, 20, ANY_ALPHABET, kissat); +act_harness!(act_simple_sym0, SimpleSym0, 20, ANY_ALPHABET, kissat); +act_harness!(act_simple_sym1, SimpleSym1, 20, ANY_ALPHABET, kissat); +act_harness!(act_normal_count, NormalCount, 21, ANY_ALPHABET, kissat); +act_harness!(act_normal_cl, NormalCl, 21, ANY_ALPHABET, kissat); +act_harness!(act_normal_flag, NormalFlag, 20, ANY_ALPHABET, kissat); +act_harness!(act_normal_nbits, NormalNbits, 20, ANY_ALPHABET, kissat); +act_harness!(act_normal_max, NormalMax, 20, ANY_ALPHABET, kissat); +act_harness!(act_lengths, Lengths, 20, ANY_ALPHABET, kissat); +act_harness!(act_length_rep, LengthRep, 300, ANY_ALPHABET, kissat); +// Building a code over any lengths of 288 symbols does not fit in memory: +// this harness bounds the alphabet to 8 symbols. `huffman_build_any` covers +// `huffman::build` itself over any lengths. +act_harness!(act_build_code, BuildCode, 20, 8, kissat); + +// Decoding pixels (a literal's symbol, a length, a distance, a cache index, +// a backward reference of at most 4 pixels). +act_harness!(act_green, Green, 20, ANY_ALPHABET, minisat); +act_harness!(act_red, Red, 20, ANY_ALPHABET, minisat); +act_harness!(act_blue, Blue, 20, ANY_ALPHABET, minisat); +act_harness!(act_alpha, Alpha, 20, ANY_ALPHABET, minisat); +act_harness!(act_len_extra, LenExtra, 20, ANY_ALPHABET, minisat); +// The distance phases copy a backward reference into the image of the role, +// one harness per role (with the role symbolic too, MiniSat ran out of 4 GB). +act_harness!(act_dist_sym_main, DistSym, Main, 20, ANY_ALPHABET, minisat); +act_harness!(act_dist_sym_meta, DistSym, Meta, 20, ANY_ALPHABET, minisat); +act_harness!( + act_dist_sym_predictor, + DistSym, + Predictor, + 20, + ANY_ALPHABET, + minisat +); +act_harness!( + act_dist_sym_colour, + DistSym, + Colour, + 20, + ANY_ALPHABET, + minisat +); +act_harness!( + act_dist_sym_palette, + DistSym, + Palette, + 20, + ANY_ALPHABET, + minisat +); +act_harness!( + act_dist_extra_main, + DistExtra, + Main, + 20, + ANY_ALPHABET, + minisat +); +act_harness!( + act_dist_extra_meta, + DistExtra, + Meta, + 20, + ANY_ALPHABET, + minisat +); +act_harness!( + act_dist_extra_predictor, + DistExtra, + Predictor, + 20, + ANY_ALPHABET, + minisat +); +act_harness!( + act_dist_extra_colour, + DistExtra, + Colour, + 20, + ANY_ALPHABET, + minisat +); +act_harness!( + act_dist_extra_palette, + DistExtra, + Palette, + 20, + ANY_ALPHABET, + minisat +); + +/// Finishing an image of `$role`: no panic, `inv` kept on `Ok`. +macro_rules! finish_harness { + ($name:ident, $role:ident) => { + finish_harness!($name, $role, None); + }; + ($name:ident, $role:ident, $transform:expr) => { + #[kani::proof] + #[kani::solver(minisat)] + #[kani::unwind(6)] + fn $name() { + let mut v = any_vp8l(P::Done, Some(Role::$role), GROUP_BYTES, ANY_ALPHABET); + if let Some(k) = $transform { + v.kani_one_transform(k); + } + with_scratch($transform == Some(3), |s, target| { + if v.kani_finish(s, target).is_ok() { + assert!(v.kani_inv(GROUP_BYTES)); + } + }); + } + }; +} + +// The main image inverts its transforms in a loop whose body depends only on +// that transform and the pixels (`invert` takes `&self`); with every order of +// four transforms at once MiniSat ran out of 4 GB, so one harness per kind, +// each with that one transform over any pixels, and one with none. +finish_harness!(finish_main_none, Main, Some(4)); +finish_harness!(finish_main_colour, Main, Some(1)); +finish_harness!(finish_main_green, Main, Some(2)); +finish_harness!(finish_main_index, Main, Some(3)); +finish_harness!(finish_meta, Meta); +finish_harness!(finish_predictor, Predictor); +finish_harness!(finish_colour, Colour); +finish_harness!(finish_palette, Palette); + +/// Finishing the main image with one predictor transform, on an image of +/// exactly 2 by 2 pixels: with the size symbolic (even bounded to 1 row of 2 +/// pixels) no solver fitted in 4 GB, as every loop unrolled along every path. +/// `predictor_row_any` covers rows of 3 pixels with any block bits. +#[kani::proof] +#[kani::solver(minisat)] +#[kani::unwind(6)] +fn finish_main_predictor() { + let mut v = any_vp8l(P::Done, Some(Role::Main), GROUP_BYTES, ANY_ALPHABET); + v.kani_one_transform(0); + v.kani_set_first_size(2, 2); + with_scratch(false, |s, target| { + if v.kani_finish(s, target).is_ok() { + assert!(v.kani_inv(GROUP_BYTES)); + } + }); +} + +/// One byte of the container in `$phase`, on a 2 by 2 canvas: no panic, +/// `state_inv` kept on `Ok`. One harness per phase (`any_webp` says why). +macro_rules! container_harness { + ($name:ident, $phase:ident) => { + #[kani::proof] + #[kani::solver(minisat)] + #[kani::unwind(6)] + fn $name() { + container_byte(Phase::$phase); + } + }; +} + +container_harness!(container_riff, Riff); +container_harness!(container_chunk_hdr, ChunkHdr); +container_harness!(container_vp8x, Vp8x); +container_harness!(container_anim, Anim); +container_harness!(container_anmf, Anmf); +container_harness!(container_skip, Skip); +container_harness!(container_pad, Pad); +container_harness!(container_ended, Ended); +container_harness!(container_spent, Spent); + +fn container_byte(phase: Phase) { + let Some(mut webp) = any_webp(phase) else { + return; + }; + webp.kani_set_phase(phase); + let groups = GROUP_BYTES; + kani::assume(webp.kani_inv(groups)); + let b: u8 = kani::any(); + let mut pixels: [u8; 16] = kani::any(); + let mut frame_array: [u8; 16] = kani::any(); + let frame_len = if webp.kani_animated() { 16 } else { 0 }; + with_scratch(false, |s, _| { + let Some(frame) = frame_array.get_mut(..frame_len) else { + return; + }; + let mut parts = Parts { + pixels: &mut pixels, + scratch: Scratch { + cache: &mut *s.cache, + lens: &mut *s.lens, + cl_lens: &mut *s.cl_lens, + cl_counts: &mut *s.cl_counts, + cl_syms: &mut *s.cl_syms, + palette: &mut *s.palette, + pred: &mut *s.pred, + color: &mut *s.color, + meta: &mut *s.meta, + groups: &mut *s.groups, + }, + frame, + }; + if webp.byte(b, &mut parts).is_ok() { + assert!(webp.kani_inv(groups)); + } + }); +} diff --git a/crates/clazel-webp/src/transform.rs b/crates/clazel-webp/src/transform.rs new file mode 100644 index 000000000..0af3375ee --- /dev/null +++ b/crates/clazel-webp/src/transform.rs @@ -0,0 +1,431 @@ +//! The inverse transforms of VP8L (RFC 9649 section 3.5), one row at a time, +//! in place on a pixel buffer of RGBA bytes. +//! +//! A pixel is 4 bytes in the order red, green, blue, alpha (the order of the +//! output, so no reordering at the end); the RFC's ARGB value of a pixel is +//! alpha, red, green, blue from the high byte down, and a channel is 8 bits +//! whatever the order, so every transform here is written per channel. +//! +//! Every function indexes with checked operations only; a position outside +//! the buffer leaves the buffer alone and the function returns `false` +//! (callers size the buffers so that does not happen; the Kani harnesses +//! prove it does not panic for any inputs). + +/// A pixel: red, green, blue, alpha. +pub(crate) type Px = [u8; 4]; + +/// Opaque black: RFC 9649 section 3.5.1, mode 0's "0xff000000". +pub(crate) const BLACK: Px = [0, 0, 0, 255]; + +/// Pixel `i` of `buf` (transparent black if it is not there). +pub(crate) fn px(buf: &[u8], i: usize) -> Px { + i.checked_mul(4) + .and_then(|o| buf.get(o..o.checked_add(4)?)) + .and_then(|s| Px::try_from(s).ok()) + .unwrap_or([0; 4]) +} + +/// Sets pixel `i` of `buf`; false if it is not there. +pub(crate) fn put(buf: &mut [u8], i: usize, p: Px) -> bool { + let slot = i + .checked_mul(4) + .and_then(|o| buf.get_mut(o..o.checked_add(4)?)); + match slot { + Some(d) => { + d.copy_from_slice(&p); + true + } + None => false, + } +} + +/// The channels of `a` plus those of `b`, each modulo 256. +fn add_px(a: Px, b: Px) -> Px { + let [a0, a1, a2, a3] = a; + let [b0, b1, b2, b3] = b; + [ + a0.wrapping_add(b0), + a1.wrapping_add(b1), + a2.wrapping_add(b2), + a3.wrapping_add(b3), + ] +} + +/// Section 3.5.1's `Average2` of one channel: `(a + b) / 2`, without the +/// intermediate overflow. +fn avg(a: u8, b: u8) -> u8 { + (a & b).wrapping_add((a ^ b) >> 1) +} + +/// `Average2` on every channel. +pub(crate) fn avg2(a: Px, b: Px) -> Px { + let [a0, a1, a2, a3] = a; + let [b0, b1, b2, b3] = b; + [avg(a0, b0), avg(a1, b1), avg(a2, b2), avg(a3, b3)] +} + +/// Section 3.5.1's `Clamp`. +fn clamp(v: i32) -> u8 { + u8::try_from(v.clamp(0, 255)).unwrap_or(0) +} + +/// The channels of two pixels, paired. +fn zip(a: Px, b: Px) -> impl Iterator { + a.into_iter() + .zip(b) + .map(|(x, y)| (i32::from(x), i32::from(y))) +} + +/// Section 3.5.1's `Select(L, T, TL)`: the one of L and T nearer, in the +/// sum of the channels' differences, to `L + T - TL`; T on a tie. +pub(crate) fn select(l: Px, t: Px, tl: Px) -> Px { + let mut p_l: i32 = 0; + let mut p_t: i32 = 0; + for ((lc, tc), tlc) in zip(l, t).zip(tl.into_iter().map(i32::from)) { + let p = lc.wrapping_add(tc).wrapping_sub(tlc); + p_l = p_l.wrapping_add(p.wrapping_sub(lc).abs()); + p_t = p_t.wrapping_add(p.wrapping_sub(tc).abs()); + } + if p_l < p_t { l } else { t } +} + +/// `ClampAddSubtractFull(a, b, c)` on every channel: `Clamp(a + b - c)`. +fn clamp_add_sub_full(a: Px, b: Px, c: Px) -> Px { + let mut out = [0u8; 4]; + for (o, ((x, y), z)) in out + .iter_mut() + .zip(zip(a, b).zip(c.into_iter().map(i32::from))) + { + *o = clamp(x.wrapping_add(y).wrapping_sub(z)); + } + out +} + +/// `ClampAddSubtractHalf(a, b)` on every channel: `Clamp(a + (a - b) / 2)`, +/// the division truncating toward zero as in C. +fn clamp_add_sub_half(a: Px, b: Px) -> Px { + let mut out = [0u8; 4]; + for (o, (x, y)) in out.iter_mut().zip(zip(a, b)) { + *o = clamp(x.wrapping_add(x.wrapping_sub(y) / 2)); + } + out +} + +/// The predicted value of mode `mode` (section 3.5.1, Table 2) from the +/// left, top, top-right and top-left pixels. Modes 14 and 15, which the RFC +/// does not define, predict like mode 0 (a listed choice: libwebp's). +pub(crate) fn predict(mode: u8, l: Px, t: Px, tr: Px, tl: Px) -> Px { + match mode & 15 { + 1 => l, + 2 => t, + 3 => tr, + 4 => tl, + 5 => avg2(avg2(l, tr), t), + 6 => avg2(l, tl), + 7 => avg2(l, t), + 8 => avg2(tl, t), + 9 => avg2(t, tr), + 10 => avg2(avg2(l, tl), avg2(t, tr)), + 11 => select(l, t, tl), + 12 => clamp_add_sub_full(l, t, tl), + 13 => clamp_add_sub_half(avg2(l, t), tl), + _ => BLACK, + } +} + +/// `x >> bits`, 0 if `bits` is not a shift. +fn shr(x: usize, bits: u8) -> usize { + x.checked_shr(u32::from(bits)).unwrap_or(0) +} + +/// The inverse predictor transform of row `y` of an image `w` pixels wide +/// (section 3.5.1). `buf` holds the image, rows before `y` already inverted; +/// `modes` is the predictor image, `bw` blocks of `1 << bits` pixels per row, +/// the mode of a block in its pixel's green byte. +/// +/// The border rules (section 3.5.1): the top-left pixel is predicted +/// 0xff000000, the rest of the top row by L, and the rest of the left column +/// by T. The rightmost pixel's top-right neighbour is the leftmost pixel of +/// its own row, which is where the linear position `i - w + 1` falls. +pub(crate) fn predictor_row( + buf: &mut [u8], + w: usize, + y: usize, + bits: u8, + bw: usize, + modes: &[u8], +) -> bool { + predictor_row_checked(buf, w, y, bits, bw, modes).is_some() +} + +fn predictor_row_checked( + buf: &mut [u8], + w: usize, + y: usize, + bits: u8, + bw: usize, + modes: &[u8], +) -> Option<()> { + let row = y.checked_mul(w)?; + for x in 0..w { + let i = row.checked_add(x)?; + let pred = if y == 0 { + if x == 0 { + BLACK + } else { + px(buf, i.checked_sub(1)?) + } + } else if x == 0 { + px(buf, i.checked_sub(w)?) + } else { + let block = shr(y, bits).checked_mul(bw)?.checked_add(shr(x, bits))?; + let mode = px(modes, block).get(1).copied().unwrap_or(0); + let up = i.checked_sub(w)?; + predict( + mode, + px(buf, i.checked_sub(1)?), + px(buf, up), + px(buf, up.checked_add(1)?), + px(buf, up.checked_sub(1)?), + ) + }; + let sum = add_px(px(buf, i), pred); + if !put(buf, i, sum) { + return None; + } + } + Some(()) +} + +/// Section 3.5.2's `ColorTransformDelta(t, c)`: `(t * c) >> 5` for the +/// signed 8-bit values, modulo 256. +fn delta(t: u8, c: u8) -> u8 { + let p = i32::from(i8::from_ne_bytes([t])).wrapping_mul(i32::from(i8::from_ne_bytes([c]))); + u8::try_from((p >> 5) & 0xFF).unwrap_or(0) +} + +/// The inverse colour transform of row `y` of an image `w` pixels wide +/// (section 3.5.2): red gains the green-to-red delta, blue the green-to-blue +/// delta of the green and the red-to-blue delta of the **new** red. `elems` +/// is the colour transform image: per block, red is `red_to_blue`, green +/// `green_to_blue`, blue `green_to_red`. +pub(crate) fn color_row( + buf: &mut [u8], + w: usize, + y: usize, + bits: u8, + bw: usize, + elems: &[u8], +) -> bool { + color_row_checked(buf, w, y, bits, bw, elems).is_some() +} + +fn color_row_checked( + buf: &mut [u8], + w: usize, + y: usize, + bits: u8, + bw: usize, + elems: &[u8], +) -> Option<()> { + let row = y.checked_mul(w)?; + for x in 0..w { + let i = row.checked_add(x)?; + let block = shr(y, bits).checked_mul(bw)?.checked_add(shr(x, bits))?; + let [red_to_blue, green_to_blue, green_to_red, _] = px(elems, block); + let [r, g, b, a] = px(buf, i); + let new_r = r.wrapping_add(delta(green_to_red, g)); + let new_b = b + .wrapping_add(delta(green_to_blue, g)) + .wrapping_add(delta(red_to_blue, new_r)); + if !put(buf, i, [new_r, g, new_b, a]) { + return None; + } + } + Some(()) +} + +/// The inverse subtract green transform of row `y` of an image `w` wide +/// (section 3.5.3): red and blue gain the green. +pub(crate) fn green_row(buf: &mut [u8], w: usize, y: usize) -> bool { + let Some(row) = y.checked_mul(w) else { + return false; + }; + for x in 0..w { + let Some(i) = row.checked_add(x) else { + return false; + }; + let [r, g, b, a] = px(buf, i); + if !put(buf, i, [r.wrapping_add(g), g, b.wrapping_add(g), a]) { + return false; + } + } + true +} + +/// The pixel bundling width of a colour table of `n` entries (section +/// 3.5.4, Table 3): `width_bits` 3, 2, 1 or 0. +pub(crate) fn bundle_bits(n: usize) -> u8 { + match n { + 0..=2 => 3, + 3..=4 => 2, + 5..=16 => 1, + _ => 0, + } +} + +/// Expands one row of the colour indexing transform (section 3.5.4): the +/// `w` pixels of the row come from the packed row that starts at pixel +/// `src`, written to the row that starts at pixel `dst`; `table` holds +/// `n` colours (4 bytes each), and an index of `n` or more gives +/// transparent black. Pixels are produced from the last to the first and +/// each reads only packed pixels at or before its own position, so with +/// `dst >= src` the row can be expanded in place (rows from the last to the +/// first). +pub(crate) fn palette_row( + buf: &mut [u8], + src: usize, + dst: usize, + w: usize, + bits: u8, + table: &[u8], + n: usize, +) -> bool { + palette_row_checked(buf, src, dst, w, bits, table, n).is_some() +} + +fn palette_row_checked( + buf: &mut [u8], + src: usize, + dst: usize, + w: usize, + bits: u8, + table: &[u8], + n: usize, +) -> Option<()> { + // `1 << bits` pixels per packed pixel, each `8 >> bits` bits wide. + let per = 1usize.checked_shl(u32::from(bits))?; + let width = 8usize.checked_shr(u32::from(bits))?; + let mask = 1usize + .checked_shl(u32::try_from(width).ok()?)? + .checked_sub(1)?; + for x in (0..w).rev() { + let packed = px(buf, src.checked_add(shr(x, bits))?); + let g = usize::from(packed.get(1).copied().unwrap_or(0)); + let slot = x.checked_rem(per)?.checked_mul(width)?; + let index = g.checked_shr(u32::try_from(slot).ok()?)? & mask; + let colour = if index < n { px(table, index) } else { [0; 4] }; + if !put(buf, dst.checked_add(x)?, colour) { + return None; + } + } + Some(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn averages() { + assert_eq!(avg(255, 254), 254); + assert_eq!(avg(1, 2), 1); + assert_eq!(avg2([0, 10, 20, 255], [2, 11, 20, 255]), [1, 10, 20, 255]); + } + + #[test] + fn select_ties_go_to_top() { + // L == T: both distances equal, T. + assert_eq!( + select([1, 2, 3, 4], [1, 2, 3, 4], [0, 0, 0, 0]), + [1, 2, 3, 4] + ); + // T == TL: p == L, so L is at distance 0 and T is not. + assert_eq!( + select([9, 9, 9, 9], [5, 5, 5, 5], [5, 5, 5, 5]), + [9, 9, 9, 9] + ); + // L == TL: p == T: T closer. + assert_eq!( + select([5, 5, 5, 5], [9, 9, 9, 9], [5, 5, 5, 5]), + [9, 9, 9, 9] + ); + } + + #[test] + fn clamps() { + assert_eq!(clamp_add_sub_full([250; 4], [250; 4], [0; 4]), [255; 4]); + assert_eq!(clamp_add_sub_full([0; 4], [0; 4], [9; 4]), [0; 4]); + // a + (a - b) / 2 with a - b = -3 truncates to -1. + assert_eq!(clamp_add_sub_half([10; 4], [13; 4]), [9; 4]); + assert_eq!(clamp_add_sub_half([10; 4], [14; 4]), [8; 4]); + } + + #[test] + fn colour_delta() { + // 3.5 fixed point: t = 32 is 1.0, so delta(32, c) = c for a signed c. + assert_eq!(delta(32, 7), 7); + assert_eq!(delta(32, 0xFF), 0xFF); + assert_eq!(delta(0xE0, 8), 0xF8); + } + + #[test] + fn predictor_borders() { + // 3 wide, 2 high, all residuals 1; mode 0 everywhere (black). + let mut buf = vec![1u8; 24]; + let modes = [0u8; 4]; + assert!(predictor_row(&mut buf, 3, 0, 2, 1, &modes)); + assert!(predictor_row(&mut buf, 3, 1, 2, 1, &modes)); + // Top-left: 1 + black = [1, 1, 1, 0] (alpha 255 + 1 wraps). + assert_eq!(px(&buf, 0), [1, 1, 1, 0]); + // The rest of the top row: L + residual. + assert_eq!(px(&buf, 1), [2, 2, 2, 1]); + // Left column: T + residual. + assert_eq!(px(&buf, 3), [2, 2, 2, 1]); + } + + #[test] + fn rightmost_top_right_is_the_row_start() { + // Mode 3 (TR) on row 1 of a 2-wide image: the pixel at x = 1 takes + // the pixel at (0, 1), the first of its own row. + let mut buf = vec![0u8; 16]; + put(&mut buf, 2, [10, 20, 30, 40]); + put(&mut buf, 3, [0, 0, 0, 0]); + let modes = [0u8, 3, 0, 0]; + assert!(predictor_row(&mut buf, 2, 1, 2, 1, &modes)); + assert_eq!(px(&buf, 3), [10, 20, 30, 40]); + } + + #[test] + fn colour_and_green() { + let mut buf = vec![0u8; 4]; + put(&mut buf, 0, [10, 20, 30, 40]); + assert!(green_row(&mut buf, 1, 0)); + assert_eq!(px(&buf, 0), [30, 20, 50, 40]); + // green_to_red = 32 (1.0), green_to_blue = 0, red_to_blue = 32. + let elems = [32u8, 0, 32, 255]; + put(&mut buf, 0, [10, 20, 30, 40]); + assert!(color_row(&mut buf, 1, 0, 2, 1, &elems)); + // red 10 + 20 = 30; blue 30 + 0 + delta(32, 30) = 60. + assert_eq!(px(&buf, 0), [30, 20, 60, 40]); + } + + #[test] + fn palette_expansion() { + let table: Vec = (0..4u8).flat_map(|i| [i, i, i, 255]).collect(); + // 2 bits per index (4 colours, bundle bits 2): packed green 0b11_10_01_00. + let mut buf = vec![0u8; 16]; + put(&mut buf, 0, [0, 0b1110_0100, 0, 255]); + assert!(palette_row(&mut buf, 0, 0, 4, 2, &table, 4)); + for x in 0..4u8 { + assert_eq!(px(&buf, usize::from(x)), [x, x, x, 255]); + } + // An index past the table is transparent black. + let mut buf = vec![0u8; 16]; + put(&mut buf, 0, [0, 0b1110_0100, 0, 255]); + assert!(palette_row(&mut buf, 0, 0, 4, 2, &table, 3)); + assert_eq!(px(&buf, 3), [0; 4]); + assert_eq!(bundle_bits(2), 3); + assert_eq!(bundle_bits(17), 0); + } +} diff --git a/crates/clazel-webp/src/vp8l.rs b/crates/clazel-webp/src/vp8l.rs new file mode 100644 index 000000000..8dd58442f --- /dev/null +++ b/crates/clazel-webp/src/vp8l.rs @@ -0,0 +1,1259 @@ +//! The VP8L bitstream (RFC 9649 section 3), as a state machine fed one byte +//! at a time. +//! +//! Citations are to RFC 9649 by section; the listed choices are in `lib.rs`. +//! +//! # How it runs +//! +//! [`Vp8l::push`] appends a byte to the bit reader and [`Vp8l::advance`] +//! then runs **atomic actions** until one needs bits that have not come: an +//! action is one fixed-width read (at most 18 bits), or one prefix-coded +//! symbol (at most 15 bits), plus what follows from it. Each action is a +//! phase of the machine; what an action holds between bytes is in the +//! fields of [`Vp8l`], so the state is a fixed-size value. When the last +//! byte of the chunk has been pushed (`Bits::end`) an action that wants bits +//! that are not there is an error. +//! +//! # What it decodes into +//! +//! The image is decoded into `target` as RGBA bytes. The transforms' +//! images, the entropy image, the colour table, the colour cache and the +//! prefix codes live in the regions of [`Scratch`]. When the last pixel has +//! come, the inverse transforms run over `target` in place, last transform +//! first (section 3.5: "The inverse transforms are applied in the reverse +//! order that they are read from the bitstream"). + +use clazel_image_shape::Error; + +use crate::bits::Bits; +use crate::distance; +use crate::huffman::{self, Decoded}; +use crate::transform::{self, Px, put, px}; +use crate::work::{ + CL_ALPHABET, COUNTS_BYTES, DIST_ALPHABET, GREEN_BASE, GROUP_BYTES, MAX_CACHE_BITS, SYMBOL_BASE, + Scratch, +}; + +/// The message for a chunk that ends inside the image. +const EOS: &str = "the VP8L data ends before the image is complete"; + +fn bad(why: &'static str) -> Error { + Error::Malformed(why) +} + +/// `kCodeLengthCodeOrder` of section 3.7.2.1.2. +const CL_ORDER: [usize; CL_ALPHABET] = [ + 17, 18, 0, 1, 2, 3, 4, 5, 16, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, +]; + +/// What an image is for (section 3.6.1, "Roles of Image Data"). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Role { + /// The ARGB image. + Main, + /// The entropy image. + Meta, + /// The predictor image. + Predictor, + /// The colour transform image. + Colour, + /// The colour table. + Palette, +} + +/// The four transforms (section 3.5, Table 1, in that order). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Kind { + Predictor, + Colour, + Green, + Index, +} + +/// A transform read from the stream. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Tf { + kind: Kind, + /// Block size bits (predictor, colour), or pixel bundling bits (index). + bits: u8, + /// The width of the image the transform applies to when it is inverted + /// (for the index transform, the width **after** expanding). + xsize: u32, +} + +/// The phases of the machine. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum P { + /// Section 3.4: the signature byte 0x2f. + Signature, + /// 14 bits: width minus one. + Width, + /// 14 bits: height minus one. + Height, + /// 1 bit alpha_is_used, 3 bits version. + Version, + /// Section 3.5: 1 bit, another transform? + TransformFlag, + /// 2 bits: the type. + TransformType, + /// 3 bits: size bits of the predictor or colour transform. + TransformBits, + /// 8 bits: colour table size minus one. + PaletteSize, + /// Section 3.6.2.3: 1 bit, a colour cache? + CacheFlag, + /// 4 bits: the cache size bits. + CacheBits, + /// Section 3.7.2.2: 1 bit, meta prefix codes? + MetaFlag, + /// 3 bits: prefix bits. + MetaBits, + /// Section 3.7.2.1: 1 bit, simple code length code? + CodeKind, + /// 1 bit: number of symbols minus one. + SimpleCount, + /// 1 bit: is_first_8bits. + SimpleFirst, + /// 1 or 8 bits: the first symbol. + SimpleSym0, + /// 8 bits: the second symbol. + SimpleSym1, + /// 4 bits: num_code_lengths minus 4. + NormalCount, + /// 3 bits each: the code length code's lengths. + NormalCl, + /// 1 bit: a max_symbol follows? + NormalFlag, + /// 3 bits: length_nbits. + NormalNbits, + /// length_nbits bits: max_symbol minus 2. + NormalMax, + /// A symbol of the code length code. + Lengths, + /// The extra bits of a repeat code. + LengthRep, + /// The code is complete: build it. + BuildCode, + /// Section 3.7.2.3: a symbol of the green code. + Green, + /// A red symbol. + Red, + /// A blue symbol. + Blue, + /// An alpha symbol. + Alpha, + /// The extra bits of a length. + LenExtra, + /// A distance prefix symbol. + DistSym, + /// The extra bits of a distance. + DistExtra, + /// The image is decoded and its transforms inverted. + Done, +} + +/// What one action did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Act { + /// Run the next action. + Again, + /// More bits are needed. + Wait, + /// An image (of whatever role) has all its pixels. + ImageDone, + /// The main image is complete. + Finished, +} + +/// The lossless decoder's state. +#[derive(Debug, Clone, Copy)] +pub(crate) struct Vp8l { + phase: P, + bits: Bits, + /// The size the container says the image has. + want_w: u32, + want_h: u32, + /// The size in the header, and the width after colour indexing. + width: u32, + height: u32, + xsize: u32, + /// The transforms read so far, and which kinds (bit per kind). + tf: [Tf; 4], + ntf: u8, + used: u8, + /// The image being decoded: its role, width, pixel count, position. + role: Role, + ixs: u32, + total: u32, + pos: u32, + /// The colour cache's size bits (0: none) and the main image's, saved + /// while its entropy image is read. + cache_bits: u8, + main_cache_bits: u8, + /// The main image has meta prefix codes: bits, width of the entropy + /// image, and the number of groups. + use_meta: bool, + meta_bits: u8, + meta_w: u32, + ngroups: u32, + /// What the transform being read is, and its size bits or colour count. + pending: Kind, + tbits: u8, + ncolors: u16, + /// The prefix code being read: group, code (0 to 4), alphabet size. + g: u32, + c: u8, + alphabet: u16, + simple_n: u8, + first8: bool, + cl_n: u8, + cl_i: u8, + nbits: u8, + max_left: u32, + sym: u32, + prev: u8, + rep: u8, + /// A pixel being read: green, red, blue; a length; the prefix code and + /// extra bit count of a length or distance. + lit_g: u8, + lit_r: u8, + lit_b: u8, + length: u32, + dprefix: u32, + extra_n: u8, +} + +/// `ceil(v / 2^bits)` (section 3.5.1's `DIV_ROUND_UP`). +fn ceil_shr(v: u32, bits: u8) -> Option { + let add = 1u32.checked_shl(u32::from(bits))?.checked_sub(1)?; + v.checked_add(add)?.checked_shr(u32::from(bits)) +} + +/// The cache slot of a pixel: `(0x1e35a7bd * color) >> (32 - bits)` +/// (section 3.6.2.3), where `color` is the ARGB value. +fn cache_key(p: Px, bits: u8) -> usize { + let [r, g, b, a] = p; + let argb = u32::from(a).wrapping_shl(24) + | u32::from(r).wrapping_shl(16) + | u32::from(g).wrapping_shl(8) + | u32::from(b); + let h = 0x1e35_a7bd_u32.wrapping_mul(argb); + let shift = 32u32.saturating_sub(u32::from(bits)); + usize::try_from(h.checked_shr(shift).unwrap_or(0)).unwrap_or(0) +} + +/// The counts and symbols slices of code `c` of group `g`. +fn group_ref(groups: &[u8], g: u32, c: u8) -> Option<(&[u8], &[u8])> { + let start = usize::try_from(g).ok()?.checked_mul(GROUP_BYTES)?; + let grp = groups.get(start..start.checked_add(GROUP_BYTES)?)?; + let (counts, syms) = grp.split_at_checked(5 * COUNTS_BYTES)?; + let c = usize::from(c); + let at = c.checked_mul(COUNTS_BYTES)?; + let counts = counts.get(at..at.checked_add(COUNTS_BYTES)?)?; + let base = SYMBOL_BASE.get(c)?.checked_mul(2)?; + Some((counts, syms.get(base..)?)) +} + +/// The same, writable. +fn group_mut(groups: &mut [u8], g: u32, c: u8) -> Option<(&mut [u8], &mut [u8])> { + let start = usize::try_from(g).ok()?.checked_mul(GROUP_BYTES)?; + let grp = groups.get_mut(start..start.checked_add(GROUP_BYTES)?)?; + let (counts, syms) = grp.split_at_mut_checked(5 * COUNTS_BYTES)?; + let c = usize::from(c); + let at = c.checked_mul(COUNTS_BYTES)?; + let counts = counts.get_mut(at..at.checked_add(COUNTS_BYTES)?)?; + let base = SYMBOL_BASE.get(c)?.checked_mul(2)?; + Some((counts, syms.get_mut(base..)?)) +} + +impl Vp8l { + /// A decoder for an image the container says is `want_w` by `want_h`. + pub(crate) fn new(want_w: u32, want_h: u32) -> Vp8l { + Vp8l { + phase: P::Signature, + bits: Bits::new(), + want_w, + want_h, + width: 0, + height: 0, + xsize: 0, + tf: [Tf { + kind: Kind::Green, + bits: 0, + xsize: 0, + }; 4], + ntf: 0, + used: 0, + role: Role::Main, + ixs: 0, + total: 0, + pos: 0, + cache_bits: 0, + main_cache_bits: 0, + use_meta: false, + meta_bits: 0, + meta_w: 0, + ngroups: 1, + pending: Kind::Green, + tbits: 0, + ncolors: 0, + g: 0, + c: 0, + alphabet: 0, + simple_n: 0, + first8: false, + cl_n: 0, + cl_i: 0, + nbits: 0, + max_left: 0, + sym: 0, + prev: 8, + rep: 0, + lit_g: 0, + lit_r: 0, + lit_b: 0, + length: 0, + dprefix: 0, + extra_n: 0, + } + } + + /// The invariant: every counter is in its range, so that the indexes the + /// actions compute are in the regions (`groups` is the byte length of the + /// groups region). + pub(crate) fn inv(&self, groups: usize) -> bool { + let cap = groups / GROUP_BYTES; + self.bits.inv() + && self.bits.len() <= 40 + && self.ntf <= 4 + && self.used < 16 + && self.pos <= self.total + && self.ixs <= 16384 + && self.total <= 1 << 26 + && self.width <= 16384 + && self.height <= 16384 + && self.xsize <= 16384 + && self.cache_bits <= MAX_CACHE_BITS + && self.main_cache_bits <= MAX_CACHE_BITS + && self.ncolors <= 256 + && self.tbits <= 9 + && self.meta_bits <= 9 + && usize::try_from(self.ngroups).is_ok_and(|n| n >= 1 && n <= cap.max(1)) + && usize::try_from(self.g).is_ok_and(|g| g <= cap) + && self.c <= 5 + && usize::from(self.alphabet) <= crate::work::MAX_GREEN + && self.cl_n <= 19 + && self.cl_i <= 19 + && (self.phase != P::NormalCl || self.cl_i < self.cl_n) + && self.nbits <= 16 + && self.rep <= 2 + && self.extra_n <= 18 + && (self.phase != P::LenExtra + || (self.dprefix < 24 && self.extra_n == distance::extra_bits(self.dprefix))) + && (self.phase != P::DistExtra + || (self.dprefix < 40 && self.extra_n == distance::extra_bits(self.dprefix))) + && self.sym <= u32::from(self.alphabet) + && self.length <= 4096 + } + + /// Appends the next byte of the chunk; `last` is true for the chunk's + /// last byte. + pub(crate) fn push(&mut self, b: u8, last: bool) { + self.bits.push(b); + self.bits.end = last; + } + + /// Runs actions until bits are needed (`Ok(false)`) or the image is + /// complete (`Ok(true)`). + pub(crate) fn advance( + &mut self, + s: &mut Scratch<'_>, + target: &mut [u8], + ) -> Result { + loop { + if self.phase == P::Done { + return Ok(true); + } + match self.act(s, target)? { + Act::Again => {} + Act::Wait => return Ok(false), + Act::ImageDone => { + if self.finish_image(s, target)? == Act::Finished { + return Ok(true); + } + } + Act::Finished => return Ok(true), + } + } + } + + /// Reads `k` bits: `Some` if they are there, `None` to wait, an error if + /// the chunk has ended. + fn read(&mut self, k: u8) -> Result, Error> { + match self.bits.take(k) { + Some(v) => Ok(Some(v)), + None if self.bits.end => Err(bad(EOS)), + None => Ok(None), + } + } + + /// Starts decoding an image of `w` by `h` pixels for `role` into a + /// region of `region` bytes, at its colour cache information. + fn begin_image(&mut self, role: Role, w: u32, h: u32, region: usize) -> Result<(), Error> { + let total = w + .checked_mul(h) + .ok_or(bad("an image with too many pixels"))?; + let bytes = usize::try_from(total).ok().and_then(|t| t.checked_mul(4)); + if w == 0 || h == 0 || bytes.is_none_or(|b| b > region) { + return Err(bad("an image larger than the space for it")); + } + self.role = role; + self.ixs = w; + self.total = total; + self.pos = 0; + self.cache_bits = 0; + self.use_meta = false; + self.ngroups = 1; + self.phase = P::CacheFlag; + Ok(()) + } + + /// Adds a transform (each kind at most once: section 3.5, "Each + /// transform is allowed to be used only once"). + fn push_tf(&mut self, kind: Kind, bits: u8, xsize: u32) -> Result<(), Error> { + let slot = self.tf.get_mut(usize::from(self.ntf)); + let Some(slot) = slot else { + return Err(bad("more than four transforms")); + }; + *slot = Tf { kind, bits, xsize }; + self.ntf = self.ntf.saturating_add(1); + Ok(()) + } + + /// The alphabet size of code `c` of a group. + fn alphabet_of(&self, c: u8) -> u16 { + match c { + 0 => { + let cache = if self.cache_bits == 0 { + 0 + } else { + 1usize.checked_shl(u32::from(self.cache_bits)).unwrap_or(0) + }; + u16::try_from(GREEN_BASE.saturating_add(cache)).unwrap_or(0) + } + 1..=3 => 256, + _ => u16::try_from(DIST_ALPHABET).unwrap_or(0), + } + } + + /// The colour cache information is read: the main image goes on to its + /// meta prefix codes, the others to their prefix codes. + fn after_cache(&mut self, s: &mut Scratch<'_>) { + if self.role == Role::Main { + self.phase = P::MetaFlag; + } else { + self.start_codes(s); + } + } + + /// Starts reading the prefix codes of every group, and clears the cache. + fn start_codes(&mut self, s: &mut Scratch<'_>) { + if self.cache_bits > 0 { + s.cache.fill(0); + } + self.g = 0; + self.c = 0; + self.phase = P::CodeKind; + } + + /// The group that codes the pixel at (`x`, `y`): section 3.7.2.2.2, + /// `meta_prefix_code = (entropy_image[position] >> 8) & 0xffff`, the red + /// and green bytes. + fn group_at(&self, x: u32, y: u32, meta: &[u8]) -> Result { + if !self.use_meta { + return Ok(0); + } + let bx = x.checked_shr(u32::from(self.meta_bits)).unwrap_or(0); + let by = y.checked_shr(u32::from(self.meta_bits)).unwrap_or(0); + let at = by + .checked_mul(self.meta_w) + .and_then(|v| v.checked_add(bx)) + .and_then(|v| usize::try_from(v).ok()) + .ok_or(bad("an entropy image position out of range"))?; + let [r, g, _, _] = px(meta, at); + let group = u32::from(r).wrapping_shl(8) | u32::from(g); + if group >= self.ngroups { + return Err(bad("a meta prefix code past the last group")); + } + Ok(group) + } + + /// Reads one symbol of code `c` of group `g`. + fn symbol(&mut self, groups: &[u8], g: u32, c: u8) -> Result, Error> { + let (counts, syms) = + group_ref(groups, g, c).ok_or(bad("a prefix code group out of range"))?; + match huffman::decode(counts, syms, self.bits.peek32(), self.bits.len()) { + Decoded::Symbol(sym, len) => { + self.bits.skip(len); + Ok(Some(sym)) + } + Decoded::More if self.bits.end => Err(bad(EOS)), + Decoded::More => Ok(None), + Decoded::Invalid => Err(bad("a prefix code that does not decode")), + } + } + + /// One atomic action. + fn act(&mut self, s: &mut Scratch<'_>, target: &mut [u8]) -> Result { + macro_rules! take { + ($k:expr) => { + match self.read($k)? { + Some(v) => v, + None => return Ok(Act::Wait), + } + }; + } + match self.phase { + // Section 3.4, item 6: "1-byte signature 0x2f". + P::Signature => { + if take!(8) != 0x2F { + return Err(bad("the VP8L signature byte is not 0x2f")); + } + self.phase = P::Width; + } + // Section 3.4: `ReadBits(14) + 1`, twice. + P::Width => { + self.width = take!(14).saturating_add(1); + self.phase = P::Height; + } + P::Height => { + self.height = take!(14).saturating_add(1); + self.phase = P::Version; + } + // Section 3.4: alpha_is_used (a hint only), then version_number, + // which "MUST be set to 0". + P::Version => { + let v = take!(4); + if v >> 1 != 0 { + return Err(bad("a VP8L version other than 0")); + } + if self.width != self.want_w || self.height != self.want_h { + return Err(bad("the VP8L image is not the size its container says")); + } + self.xsize = self.width; + self.phase = P::TransformFlag; + } + // Section 3.5: `while (ReadBits(1))`. + P::TransformFlag => { + if take!(1) == 0 { + let region = target.len(); + self.begin_image(Role::Main, self.xsize, self.height, region)?; + } else { + self.phase = P::TransformType; + } + } + P::TransformType => { + let kind = match take!(2) { + 0 => Kind::Predictor, + 1 => Kind::Colour, + 2 => Kind::Green, + _ => Kind::Index, + }; + let bit = match kind { + Kind::Predictor => 1, + Kind::Colour => 2, + Kind::Green => 4, + Kind::Index => 8, + }; + if self.used & bit != 0 { + return Err(bad("a transform used twice")); + } + self.used |= bit; + self.pending = kind; + match kind { + Kind::Predictor | Kind::Colour => self.phase = P::TransformBits, + Kind::Green => { + self.push_tf(Kind::Green, 0, self.xsize)?; + self.phase = P::TransformFlag; + } + Kind::Index => self.phase = P::PaletteSize, + } + } + // Sections 3.5.1 and 3.5.2: `size_bits = ReadBits(3) + 2`, then + // the sub-resolution image of + // `DIV_ROUND_UP(image_width, 1 << size_bits)` by the same of the + // height. + P::TransformBits => { + let bits = u8::try_from(take!(3).saturating_add(2)).unwrap_or(2); + self.tbits = bits; + let w = ceil_shr(self.xsize, bits).ok_or(bad("a block count overflow"))?; + let h = ceil_shr(self.height, bits).ok_or(bad("a block count overflow"))?; + let (role, region) = if self.pending == Kind::Predictor { + (Role::Predictor, s.pred.len()) + } else { + (Role::Colour, s.color.len()) + }; + self.begin_image(role, w, h, region)?; + } + // Section 3.5.4: `color_table_size = ReadBits(8) + 1`, read as an + // image of that width and a height of 1. + P::PaletteSize => { + let n = take!(8).saturating_add(1); + self.ncolors = u16::try_from(n).unwrap_or(256); + let region = s.palette.len(); + self.begin_image(Role::Palette, n, 1, region)?; + } + // Section 3.6.2.3: a 1-bit flag, then `ReadBits(4)`, "[1..11]". + P::CacheFlag => { + if take!(1) == 0 { + self.cache_bits = 0; + self.after_cache(s); + } else { + self.phase = P::CacheBits; + } + } + P::CacheBits => { + let v = u8::try_from(take!(4)).unwrap_or(0); + if v == 0 || v > MAX_CACHE_BITS { + return Err(bad("a colour cache size outside 1 to 11 bits")); + } + self.cache_bits = v; + self.after_cache(s); + } + // Section 3.7.2.2: a 1-bit flag, `prefix_bits = ReadBits(3) + 2` + // and the entropy image of `DIV_ROUND_UP(width, 1 << prefix_bits)` + // by the same of the height. + P::MetaFlag => { + if take!(1) == 0 { + self.use_meta = false; + self.ngroups = 1; + self.start_codes(s); + } else { + self.phase = P::MetaBits; + } + } + P::MetaBits => { + let bits = u8::try_from(take!(3).saturating_add(2)).unwrap_or(2); + let w = ceil_shr(self.xsize, bits).ok_or(bad("a block count overflow"))?; + let h = ceil_shr(self.height, bits).ok_or(bad("a block count overflow"))?; + self.main_cache_bits = self.cache_bits; + self.meta_bits = bits; + self.meta_w = w; + let region = s.meta.len(); + self.begin_image(Role::Meta, w, h, region)?; + } + // Section 3.7.2.1: 1 means a simple code length code. + P::CodeKind => { + self.alphabet = self.alphabet_of(self.c); + self.sym = 0; + let a = usize::from(self.alphabet); + s.lens + .get_mut(..a) + .ok_or(bad("an alphabet larger than its space"))? + .fill(0); + self.phase = if take!(1) == 1 { + P::SimpleCount + } else { + P::NormalCount + }; + } + // Section 3.7.2.1.1: `num_symbols = ReadBits(1) + 1`, + // `is_first_8bits = ReadBits(1)`, `symbol0 = ReadBits(1 + 7 * + // is_first_8bits)`, and a second 8-bit symbol if there are two. + P::SimpleCount => { + self.simple_n = u8::try_from(take!(1).saturating_add(1)).unwrap_or(1); + self.phase = P::SimpleFirst; + } + P::SimpleFirst => { + self.first8 = take!(1) == 1; + self.phase = P::SimpleSym0; + } + P::SimpleSym0 => { + let k = if self.first8 { 8 } else { 1 }; + let sym = take!(k); + self.set_length_one(s, sym); + self.phase = if self.simple_n == 2 { + P::SimpleSym1 + } else { + P::BuildCode + }; + } + P::SimpleSym1 => { + let sym = take!(8); + self.set_length_one(s, sym); + self.phase = P::BuildCode; + } + // Section 3.7.2.1.2: `num_code_lengths = 4 + ReadBits(4)`, then + // that many 3-bit lengths in `kCodeLengthCodeOrder`. + P::NormalCount => { + self.cl_n = u8::try_from(take!(4).saturating_add(4)).unwrap_or(19); + self.cl_i = 0; + s.cl_lens.fill(0); + self.phase = P::NormalCl; + } + P::NormalCl => { + let v = u8::try_from(take!(3)).unwrap_or(0); + let at = CL_ORDER.get(usize::from(self.cl_i)).copied(); + if let Some(slot) = at.and_then(|a| s.cl_lens.get_mut(a)) { + *slot = v; + } + self.cl_i = self.cl_i.saturating_add(1); + if self.cl_i >= self.cl_n { + huffman::build(s.cl_lens, s.cl_counts, s.cl_syms) + .map_err(|_| bad("an invalid code length code"))?; + self.phase = P::NormalFlag; + } + } + // `if (ReadBits(1) == 0) max_symbol = alphabet size; else + // length_nbits = 2 + 2 * ReadBits(3); max_symbol = 2 + + // ReadBits(length_nbits)`, an error if over the alphabet. + P::NormalFlag => { + self.sym = 0; + self.prev = 8; + if take!(1) == 0 { + self.max_left = u32::from(self.alphabet); + self.phase = P::Lengths; + } else { + self.phase = P::NormalNbits; + } + } + P::NormalNbits => { + let v = take!(3); + self.nbits = u8::try_from(v.saturating_mul(2).saturating_add(2)).unwrap_or(2); + self.phase = P::NormalMax; + } + P::NormalMax => { + let max = take!(self.nbits).saturating_add(2); + if max > u32::from(self.alphabet) { + return Err(bad("a max_symbol over the alphabet")); + } + self.max_left = max; + self.phase = P::Lengths; + } + // The code lengths: a symbol of the code length code per step, + // at most max_symbol of them, 0 to 15 a length, 16 to 18 + // repeats. + P::Lengths => { + if self.sym >= u32::from(self.alphabet) || self.max_left == 0 { + self.phase = P::BuildCode; + return Ok(Act::Again); + } + match huffman::decode(s.cl_counts, s.cl_syms, self.bits.peek32(), self.bits.len()) { + Decoded::Symbol(sym, len) => { + self.bits.skip(len); + self.max_left = self.max_left.saturating_sub(1); + if sym < 16 { + let v = u8::try_from(sym).unwrap_or(0); + let at = usize::try_from(self.sym).unwrap_or(usize::MAX); + if let Some(slot) = s.lens.get_mut(at) { + *slot = v; + } + self.sym = self.sym.saturating_add(1); + if v != 0 { + self.prev = v; + } + } else if sym <= 18 { + self.rep = u8::try_from(sym.saturating_sub(16)).unwrap_or(0); + self.phase = P::LengthRep; + } else { + // The code length code has 19 symbols (section + // 3.7.2.1.2), so a built code never yields more. + return Err(bad("a code length symbol over 18")); + } + } + Decoded::More if self.bits.end => return Err(bad(EOS)), + Decoded::More => return Ok(Act::Wait), + Decoded::Invalid => return Err(bad("a code length that does not decode")), + } + } + // Code 16 repeats the previous nonzero length 3 + ReadBits(2) + // times (8 if there is none yet), 17 is 3 + ReadBits(3) zeros, + // 18 is 11 + ReadBits(7) zeros. + P::LengthRep => { + let (extra, offset) = match self.rep { + 0 => (2, 3), + 1 => (3, 3), + _ => (7, 11), + }; + let n = take!(extra).saturating_add(offset); + let end = self.sym.saturating_add(n); + if end > u32::from(self.alphabet) { + return Err(bad("a repeat past the end of the alphabet")); + } + let value = if self.rep == 0 { self.prev } else { 0 }; + let from = usize::try_from(self.sym).unwrap_or(usize::MAX); + let to = usize::try_from(end).unwrap_or(usize::MAX); + if let Some(run) = s.lens.get_mut(from..to) { + run.fill(value); + } + self.sym = end; + self.phase = P::Lengths; + } + // Build the code of this group's `c`th slot, then go to the + // next code, group, or the pixels. + P::BuildCode => { + let a = usize::from(self.alphabet); + let lens = s + .lens + .get(..a) + .ok_or(bad("an alphabet larger than its space"))?; + let (counts, syms) = group_mut(s.groups, self.g, self.c) + .ok_or(bad("a prefix code group out of range"))?; + huffman::build(lens, counts, syms).map_err(bad)?; + self.c = self.c.saturating_add(1); + if self.c >= 5 { + self.c = 0; + self.g = self.g.saturating_add(1); + } + self.phase = if self.g >= self.ngroups { + P::Green + } else { + P::CodeKind + }; + } + P::Green | P::Red | P::Blue | P::Alpha | P::LenExtra | P::DistSym | P::DistExtra => { + return self.act_pixel(s, target); + } + P::Done => return Ok(Act::Finished), + } + Ok(Act::Again) + } + + /// Sets the length of `sym` to 1 if the alphabet has it (libwebp ignores + /// a symbol past the alphabet in a simple code: a distance code's + /// 8-bit symbol, say). + fn set_length_one(&self, s: &mut Scratch<'_>, sym: u32) { + if sym < u32::from(self.alphabet) + && let Some(slot) = usize::try_from(sym).ok().and_then(|at| s.lens.get_mut(at)) + { + *slot = 1; + } + } + + /// One atomic action on the pixels of the image (section 3.7.2.3). + fn act_pixel(&mut self, s: &mut Scratch<'_>, target: &mut [u8]) -> Result { + let (dest, meta): (&mut [u8], &[u8]) = match self.role { + Role::Main => (target, &*s.meta), + Role::Meta => (&mut *s.meta, &[]), + Role::Predictor => (&mut *s.pred, &[]), + Role::Colour => (&mut *s.color, &[]), + Role::Palette => (&mut *s.palette, &[]), + }; + let groups: &[u8] = &*s.groups; + let cache: &mut [u8] = &mut *s.cache; + let x = self.pos.checked_rem(self.ixs).unwrap_or(0); + let y = self.pos.checked_div(self.ixs).unwrap_or(0); + let group = self.group_at(x, y, meta)?; + match self.phase { + P::Green => { + let Some(sym) = self.symbol(groups, group, 0)? else { + return Ok(Act::Wait); + }; + let sym = usize::from(sym); + if sym < 256 { + self.lit_g = u8::try_from(sym).unwrap_or(0); + self.phase = P::Red; + } else if sym < GREEN_BASE { + // A length prefix code (section 3.7.2.3, 2.i). + let prefix = u32::try_from(sym.saturating_sub(256)).unwrap_or(0); + let extra = distance::extra_bits(prefix); + self.dprefix = prefix; + if extra == 0 { + self.length = distance::value(prefix, 0); + self.phase = P::DistSym; + } else { + self.extra_n = extra; + self.phase = P::LenExtra; + } + } else { + // A colour cache code (3). + let index = sym.saturating_sub(GREEN_BASE); + let size = 1usize.checked_shl(u32::from(self.cache_bits)).unwrap_or(0); + if self.cache_bits == 0 || index >= size { + return Err(bad("a colour cache index past the cache")); + } + let p = px(cache, index); + return self.emit(p, dest, cache); + } + } + P::Red => { + let Some(sym) = self.symbol(groups, group, 1)? else { + return Ok(Act::Wait); + }; + self.lit_r = u8::try_from(sym).unwrap_or(0); + self.phase = P::Blue; + } + P::Blue => { + let Some(sym) = self.symbol(groups, group, 2)? else { + return Ok(Act::Wait); + }; + self.lit_b = u8::try_from(sym).unwrap_or(0); + self.phase = P::Alpha; + } + P::Alpha => { + let Some(sym) = self.symbol(groups, group, 3)? else { + return Ok(Act::Wait); + }; + let a = u8::try_from(sym).unwrap_or(0); + let p = [self.lit_r, self.lit_g, self.lit_b, a]; + return self.emit(p, dest, cache); + } + P::LenExtra => { + let v = match self.read(self.extra_n)? { + Some(v) => v, + None => return Ok(Act::Wait), + }; + self.length = distance::value(self.dprefix, v); + self.phase = P::DistSym; + } + P::DistSym => { + let Some(sym) = self.symbol(groups, group, 4)? else { + return Ok(Act::Wait); + }; + let prefix = u32::from(sym); + // The distance code has 40 symbols (section 3.7.2.3), so a built + // code never yields more. + if prefix >= 40 { + return Err(bad("a distance code over 39")); + } + let extra = distance::extra_bits(prefix); + if extra == 0 { + return self.copy(distance::value(prefix, 0), dest, cache); + } + self.dprefix = prefix; + self.extra_n = extra; + self.phase = P::DistExtra; + } + P::DistExtra => { + let v = match self.read(self.extra_n)? { + Some(v) => v, + None => return Ok(Act::Wait), + }; + return self.copy(distance::value(self.dprefix, v), dest, cache); + } + _ => return Err(bad("a pixel action outside the pixels")), + } + Ok(Act::Again) + } + + /// Stores a pixel and moves on. + fn emit(&mut self, p: Px, dest: &mut [u8], cache: &mut [u8]) -> Result { + // The pixel phases end when `pos` reaches `total` (`after_pixels`). + if self.pos >= self.total { + return Err(bad("a pixel past the end of the image")); + } + if !put(dest, usize::try_from(self.pos).unwrap_or(usize::MAX), p) { + return Err(bad("a pixel outside the image")); + } + if self.cache_bits > 0 { + put(cache, cache_key(p, self.cache_bits), p); + } + self.pos = self.pos.saturating_add(1); + Ok(self.after_pixels()) + } + + /// A backward reference of `self.length` pixels at distance code `code` + /// (section 3.7.2.3, 2.vii: "Copy L pixels (in scan-line order) from the + /// sequence of pixels starting at the current position minus D pixels"). + /// The copy goes pixel by pixel, so a distance shorter than the length + /// repeats what was just written. A distance before the start of the + /// image or a length past its end is an error (as in libwebp). + fn copy(&mut self, code: u32, dest: &mut [u8], cache: &mut [u8]) -> Result { + let dist = distance::distance(code, self.ixs); + if dist > self.pos { + return Err(bad("a backward reference before the start of the image")); + } + if self.length > self.total.saturating_sub(self.pos) { + return Err(bad("a backward reference past the end of the image")); + } + let pos = usize::try_from(self.pos).unwrap_or(usize::MAX); + let back = usize::try_from(dist).unwrap_or(usize::MAX); + for k in 0..usize::try_from(self.length).unwrap_or(0) { + let to = pos.saturating_add(k); + let p = px(dest, to.saturating_sub(back)); + if !put(dest, to, p) { + return Err(bad("a pixel outside the image")); + } + if self.cache_bits > 0 { + put(cache, cache_key(p, self.cache_bits), p); + } + } + self.pos = self.pos.saturating_add(self.length); + Ok(self.after_pixels()) + } + + /// After a pixel or a copy: the next pixel, or the end of the image. + fn after_pixels(&mut self) -> Act { + if self.pos >= self.total { + Act::ImageDone + } else { + self.phase = P::Green; + Act::Again + } + } + + /// An image of any role has all its pixels. + fn finish_image(&mut self, s: &mut Scratch<'_>, target: &mut [u8]) -> Result { + match self.role { + Role::Predictor => { + self.push_tf(Kind::Predictor, self.tbits, self.xsize)?; + self.phase = P::TransformFlag; + } + Role::Colour => { + self.push_tf(Kind::Colour, self.tbits, self.xsize)?; + self.phase = P::TransformFlag; + } + // Section 3.5.4: "The color table is always subtraction-coded": + // each entry adds the previous one, per channel, modulo 256. + Role::Palette => { + let n = usize::from(self.ncolors); + let mut prev: Px = [0; 4]; + for i in 0..n { + let [pr, pg, pb, pa] = prev; + let [r, g, b, a] = px(s.palette, i); + prev = [ + r.wrapping_add(pr), + g.wrapping_add(pg), + b.wrapping_add(pb), + a.wrapping_add(pa), + ]; + put(s.palette, i, prev); + } + // Pixel bundling (Table 3), and the width it leaves. + let bits = transform::bundle_bits(n); + self.push_tf(Kind::Index, bits, self.xsize)?; + self.xsize = ceil_shr(self.xsize, bits).ok_or(bad("a width overflow"))?; + self.phase = P::TransformFlag; + } + // Section 3.7.2.2.2: `num_prefix_groups = max(entropy image) + + // 1`, from each pixel's red and green bytes. + Role::Meta => { + let mut max = 0u32; + for i in 0..usize::try_from(self.total).unwrap_or(0) { + let [r, g, _, _] = px(s.meta, i); + max = max.max(u32::from(r).wrapping_shl(8) | u32::from(g)); + } + let groups = max.saturating_add(1); + let cap = s.groups.len().checked_div(GROUP_BYTES).unwrap_or(0); + if usize::try_from(groups).is_ok_and(|g| g > cap) { + return Err(Error::Unsupported( + "more prefix code groups than this decoder holds", + )); + } + self.ngroups = groups; + // Back to the main image. + self.role = Role::Main; + self.cache_bits = self.main_cache_bits; + self.use_meta = true; + self.ixs = self.xsize; + self.total = self.xsize.saturating_mul(self.height); + self.pos = 0; + self.start_codes(s); + } + Role::Main => { + self.invert(s, target)?; + self.phase = P::Done; + return Ok(Act::Finished); + } + } + Ok(Act::Again) + } + + /// Inverts the transforms, last first (section 3.5), in place. + fn invert(&self, s: &mut Scratch<'_>, target: &mut [u8]) -> Result<(), Error> { + let fail = bad("a transform ran off the buffer"); + let h = usize::try_from(self.height).map_err(|_| fail)?; + let mut i = self.ntf; + while i > 0 { + i = i.saturating_sub(1); + let Some(t) = self.tf.get(usize::from(i)).copied() else { + return Err(fail); + }; + let w = usize::try_from(t.xsize).map_err(|_| fail)?; + let ok = match t.kind { + Kind::Predictor => { + let bw = ceil_shr(t.xsize, t.bits) + .and_then(|v| usize::try_from(v).ok()) + .ok_or(fail)?; + (0..h).all(|y| transform::predictor_row(target, w, y, t.bits, bw, s.pred)) + } + Kind::Colour => { + let bw = ceil_shr(t.xsize, t.bits) + .and_then(|v| usize::try_from(v).ok()) + .ok_or(fail)?; + (0..h).all(|y| transform::color_row(target, w, y, t.bits, bw, s.color)) + } + Kind::Green => (0..h).all(|y| transform::green_row(target, w, y)), + Kind::Index => { + let pw = ceil_shr(t.xsize, t.bits) + .and_then(|v| usize::try_from(v).ok()) + .ok_or(fail)?; + let n = usize::from(self.ncolors); + (0..h).rev().all(|y| { + let src = y.checked_mul(pw); + let dst = y.checked_mul(w); + match (src, dst) { + (Some(src), Some(dst)) => { + transform::palette_row(target, src, dst, w, t.bits, s.palette, n) + } + _ => false, + } + }) + } + }; + if !ok { + return Err(fail); + } + } + Ok(()) + } +} + +#[cfg(kani)] +fn any_kind() -> Kind { + match kani::any::() & 3 { + 0 => Kind::Predictor, + 1 => Kind::Colour, + 2 => Kind::Green, + _ => Kind::Index, + } +} + +#[cfg(kani)] +fn any_role() -> Role { + match kani::any::() % 5 { + 0 => Role::Main, + 1 => Role::Meta, + 2 => Role::Predictor, + 3 => Role::Colour, + _ => Role::Palette, + } +} + +/// Any state in `phase`, every field symbolic, with the invariant and with +/// the bounds a harness can unwind: images of at most 4 pixels, 2 rows, 4 +/// colours, references of at most 4 pixels, and an alphabet of at most +/// `max_alphabet` symbols. The role is `role`, or any role if `None`. +/// +/// One phase per harness: with the phase symbolic too, CBMC ran out of +/// memory (over 4 GB) or time (over 15 minutes) on every class of phases. +#[cfg(kani)] +pub(crate) fn any_vp8l(phase: P, role: Option, groups: usize, max_alphabet: u16) -> Vp8l { + let mut v = Vp8l::new(kani::any(), kani::any()); + v.phase = phase; + v.bits = Bits::any(); + v.width = kani::any(); + v.height = kani::any(); + v.xsize = kani::any(); + for t in &mut v.tf { + *t = Tf { + kind: any_kind(), + bits: kani::any(), + xsize: kani::any(), + }; + } + v.ntf = kani::any(); + v.used = kani::any(); + v.role = role.unwrap_or_else(any_role); + v.ixs = kani::any(); + v.total = kani::any(); + v.pos = kani::any(); + v.cache_bits = kani::any(); + v.main_cache_bits = kani::any(); + v.use_meta = kani::any(); + v.meta_bits = kani::any(); + v.meta_w = kani::any(); + v.ngroups = kani::any(); + v.pending = any_kind(); + v.tbits = kani::any(); + v.ncolors = kani::any(); + v.g = kani::any(); + v.c = kani::any(); + v.alphabet = kani::any(); + v.simple_n = kani::any(); + v.first8 = kani::any(); + v.cl_n = kani::any(); + v.cl_i = kani::any(); + v.nbits = kani::any(); + v.max_left = kani::any(); + v.sym = kani::any(); + v.prev = kani::any(); + v.rep = kani::any(); + v.lit_g = kani::any(); + v.lit_r = kani::any(); + v.lit_b = kani::any(); + v.length = kani::any(); + v.dprefix = kani::any(); + v.extra_n = kani::any(); + kani::assume(v.inv(groups)); + kani::assume(v.tf.iter().all(|t| t.xsize <= 4 && t.bits <= 9)); + kani::assume(v.ixs <= 4 && v.total <= 4 && v.xsize <= 4 && v.height <= 2); + kani::assume(v.width <= 4 && v.ncolors <= 4 && v.length <= 4 && v.meta_w <= 4); + kani::assume(v.alphabet <= max_alphabet); + v +} + +#[cfg(kani)] +impl Vp8l { + /// One action (for the harnesses). + pub(crate) fn kani_act(&mut self, s: &mut Scratch<'_>, target: &mut [u8]) -> Result<(), Error> { + self.act(s, target).map(|_| ()) + } + + /// Finishing an image of any role (for the harnesses). + pub(crate) fn kani_finish( + &mut self, + s: &mut Scratch<'_>, + target: &mut [u8], + ) -> Result<(), Error> { + self.finish_image(s, target).map(|_| ()) + } + + /// Keeps only the first transform, of kind `k` (0 predictor, 1 colour, 2 + /// subtract green, 3 colour indexing), or none if `k` is over 3 (for the + /// harnesses: `invert` is a loop whose body depends only on that + /// transform and the pixels, so one transform of each kind covers it). + pub(crate) fn kani_one_transform(&mut self, k: u8) { + let kind = match k { + 0 => Kind::Predictor, + 1 => Kind::Colour, + 2 => Kind::Green, + 3 => Kind::Index, + _ => { + self.ntf = 0; + return; + } + }; + self.ntf = 1; + if let Some(t) = self.tf.get_mut(0) { + t.kind = kind; + } + } + + /// Sets the image to `h` rows and the first transform to `w` pixels wide + /// (for the harnesses: constants keep symbolic execution small). + pub(crate) fn kani_set_first_size(&mut self, w: u32, h: u32) { + self.height = h; + if let Some(t) = self.tf.get_mut(0) { + t.xsize = w; + } + } + + /// The invariant, for the harnesses. + pub(crate) fn kani_inv(&self, groups: usize) -> bool { + self.inv(groups) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fresh_state_satisfies_inv() { + let v = Vp8l::new(0, 0); + assert!(v.bits.inv()); + assert!(v.inv(GROUP_BYTES)); + assert!(v.inv(GROUP_BYTES * 5)); + } +} diff --git a/crates/clazel-webp/src/work.rs b/crates/clazel-webp/src/work.rs new file mode 100644 index 000000000..baa882024 --- /dev/null +++ b/crates/clazel-webp/src/work.rs @@ -0,0 +1,214 @@ +//! The layout of the work buffer, and the bounds that fix it. +//! +//! `work_len(width, height, animated)` is the sum of these parts, in this +//! order: +//! +//! | part | bytes | holds | +//! |------|-------|-------| +//! | `cache` | `4 * CACHE_ENTRIES` | the colour cache, RGBA | +//! | `lens` | `MAX_GREEN` | the code lengths of the prefix code being read | +//! | `cl_lens` | 19 | the code length code's lengths | +//! | `cl_counts` | 32 | the code length code's counts per length, `u16` little-endian | +//! | `cl_syms` | 38 | its symbols sorted by length, `u16` | +//! | `palette` | 1024 | the colour table, up to 256 RGBA entries | +//! | `pred` | `sub` | the predictor transform's image | +//! | `color` | `sub` | the colour transform's image | +//! | `meta` | `sub` | the entropy image (meta prefix codes) | +//! | `groups` | `cap * GROUP_BYTES` | the prefix code groups | +//! | `frame` | the canvas, if animated, else 0 | a frame before it is composited | +//! +//! `sub` is 4 bytes for each block of 4 by 4 pixels of the image (the +//! sub-resolution images' blocks are at least 4 by 4: RFC 9649 sections +//! 3.5.1 and 3.7.2.2.1, "ReadBits(3) + 2"). `cap` is the number of prefix +//! code groups held: the blocks of 4 by 4, at most [`MAX_GROUPS`] (a listed +//! choice: an entropy image naming more groups is refused as unsupported). + +/// The largest colour cache, in bits: RFC 9649 section 3.6.2.3, "[1..11]". +/// (Under Kani the cache, and so every alphabet, is smaller: README.) +pub(crate) const MAX_CACHE_BITS: u8 = if cfg!(kani) { 3 } else { 11 }; + +/// The colour cache's entries at that size. +pub(crate) const CACHE_ENTRIES: usize = if cfg!(kani) { 8 } else { 2048 }; + +/// The green alphabet without a cache: 256 literals and 24 length prefix +/// codes (section 3.7.2.3). +pub(crate) const GREEN_BASE: usize = 280; + +/// The green alphabet at its largest: it holds the colour cache's indexes. +pub(crate) const MAX_GREEN: usize = GREEN_BASE + CACHE_ENTRIES; + +/// The distance alphabet: 40 prefix codes (section 3.6.2.2). +pub(crate) const DIST_ALPHABET: usize = 40; + +/// The code length code's alphabet (section 3.7.2.1.2, `kCodeLengthCodes`). +pub(crate) const CL_ALPHABET: usize = 19; + +/// The bytes of one prefix code's counts: 16 `u16`. +pub(crate) const COUNTS_BYTES: usize = 32; + +/// Where each of a group's five codes' symbols start, in symbols: green, +/// red, blue, alpha, distance (the order of section 3.7.2). +pub(crate) const SYMBOL_BASE: [usize; 5] = [ + 0, + MAX_GREEN, + MAX_GREEN + 256, + MAX_GREEN + 512, + MAX_GREEN + 768, +]; + +/// Bytes of one group: five counts arrays and the symbols of the five codes. +pub(crate) const GROUP_BYTES: usize = 5 * COUNTS_BYTES + 2 * (MAX_GREEN + 768 + DIST_ALPHABET); + +/// The most prefix code groups held (a listed choice). +pub(crate) const MAX_GROUPS: usize = 1024; + +/// The bytes of the colour table. +pub(crate) const PALETTE_BYTES: usize = 1024; + +/// The blocks of 4 by 4 pixels of a `width` by `height` image. +fn blocks(width: u32, height: u32) -> Option { + let bw = usize::try_from(width.checked_add(3)? / 4).ok()?; + let bh = usize::try_from(height.checked_add(3)? / 4).ok()?; + bw.checked_mul(bh) +} + +/// The bytes of each sub-resolution image's region. +pub(crate) fn sub_len(width: u32, height: u32) -> Option { + blocks(width, height)?.checked_mul(4) +} + +/// The prefix code groups held for an image of this size. +pub(crate) fn group_cap(width: u32, height: u32) -> Option { + Some(blocks(width, height)?.clamp(1, MAX_GROUPS)) +} + +/// The bytes of the work buffer for a canvas of `width` by `height`. +pub(crate) fn work_len(width: u32, height: u32, animated: bool) -> Option { + let sub = sub_len(width, height)?; + let canvas = if animated { + usize::try_from(width) + .ok()? + .checked_mul(usize::try_from(height).ok()?)? + .checked_mul(4)? + } else { + 0 + }; + CACHE_ENTRIES + .checked_mul(4)? + .checked_add(MAX_GREEN)? + .checked_add(CL_ALPHABET)? + .checked_add(COUNTS_BYTES)? + .checked_add(CL_ALPHABET.checked_mul(2)?)? + .checked_add(PALETTE_BYTES)? + .checked_add(sub.checked_mul(3)?)? + .checked_add(group_cap(width, height)?.checked_mul(GROUP_BYTES)?)? + .checked_add(canvas) +} + +/// The scratch regions of the lossless decoder, cut from the work buffer. +#[derive(Debug)] +pub(crate) struct Scratch<'a> { + /// The colour cache. + pub cache: &'a mut [u8], + /// Code lengths being read. + pub lens: &'a mut [u8], + /// The code length code's lengths. + pub cl_lens: &'a mut [u8], + /// The code length code's counts. + pub cl_counts: &'a mut [u8], + /// The code length code's symbols. + pub cl_syms: &'a mut [u8], + /// The colour table. + pub palette: &'a mut [u8], + /// The predictor image. + pub pred: &'a mut [u8], + /// The colour transform image. + pub color: &'a mut [u8], + /// The entropy image. + pub meta: &'a mut [u8], + /// The prefix code groups. + pub groups: &'a mut [u8], +} + +/// The pixel buffer, the scratch and the frame buffer. +#[derive(Debug)] +pub(crate) struct Parts<'a> { + /// The canvas. + pub pixels: &'a mut [u8], + /// The lossless decoder's regions. + pub scratch: Scratch<'a>, + /// A frame as decoded, before compositing (empty unless animated). + pub frame: &'a mut [u8], +} + +impl<'a> Parts<'a> { + /// Cuts `work` (exactly `work_len(width, height, animated)` bytes) into + /// its parts; `None` if it has another length. + pub(crate) fn new( + pixels: &'a mut [u8], + work: &'a mut [u8], + width: u32, + height: u32, + animated: bool, + ) -> Option> { + if work.len() != work_len(width, height, animated)? { + return None; + } + let sub = sub_len(width, height)?; + let (cache, rest) = work.split_at_mut_checked(CACHE_ENTRIES.checked_mul(4)?)?; + let (lens, rest) = rest.split_at_mut_checked(MAX_GREEN)?; + let (cl_lens, rest) = rest.split_at_mut_checked(CL_ALPHABET)?; + let (cl_counts, rest) = rest.split_at_mut_checked(COUNTS_BYTES)?; + let (cl_syms, rest) = rest.split_at_mut_checked(CL_ALPHABET.checked_mul(2)?)?; + let (palette, rest) = rest.split_at_mut_checked(PALETTE_BYTES)?; + let (pred, rest) = rest.split_at_mut_checked(sub)?; + let (color, rest) = rest.split_at_mut_checked(sub)?; + let (meta, rest) = rest.split_at_mut_checked(sub)?; + let (groups, frame) = + rest.split_at_mut_checked(group_cap(width, height)?.checked_mul(GROUP_BYTES)?)?; + Some(Parts { + pixels, + scratch: Scratch { + cache, + lens, + cl_lens, + cl_counts, + cl_syms, + palette, + pred, + color, + meta, + groups, + }, + frame, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cuts_exactly() { + for (w, h, animated) in [(1u32, 1u32, false), (5, 3, true), (64, 64, false)] { + let wl = work_len(w, h, animated).unwrap_or(0); + assert!(wl > 0); + let mut pixels = vec![0u8; usize::try_from(w * h * 4).unwrap_or(0)]; + let mut work = vec![0u8; wl]; + let parts = Parts::new(&mut pixels, &mut work, w, h, animated); + assert!( + parts.is_some_and(|p| p.frame.len() == if animated { p.pixels.len() } else { 0 }) + ); + let mut work = vec![0u8; wl.saturating_add(1)]; + assert!(Parts::new(&mut pixels, &mut work, w, h, animated).is_none()); + } + } + + #[test] + fn caps() { + assert_eq!(group_cap(1, 1), Some(1)); + assert_eq!(group_cap(16384, 16384), Some(MAX_GROUPS)); + assert_eq!(sub_len(5, 5), Some(16)); + } +} diff --git a/crates/clazel-webp/tests/common/mod.rs b/crates/clazel-webp/tests/common/mod.rs new file mode 100644 index 000000000..6e5ff62e5 --- /dev/null +++ b/crates/clazel-webp/tests/common/mod.rs @@ -0,0 +1,239 @@ +//! Shared by the WebP test targets: driving the decoder through the shared +//! interface, a deterministic generator, and byte mutations. + +use clazel_image_shape::{Buffers, ConfigStatus, Decoder, Error, Status}; +use clazel_webp::{FrameInfo, Webp}; + +/// How a decode ended. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum End { + /// No more frames (the RIFF data ended, or a still image's one frame). + Trailer, + /// An error (the frames before it were completed). + Error(String), + /// The input ran out. + Truncated, +} + +/// What the decoder gave for a file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Decoded { + pub width: u32, + pub height: u32, + /// Each completed frame: its facts and the canvas after it. + pub frames: Vec<(Option, Vec)>, + pub end: End, +} + +pub const CANARY: u8 = 0xEE; + +/// Decodes `bytes` through the shared interface, in chunks of `chunk` bytes +/// (with an empty chunk between each pair if `empties`), checking the +/// invariant and the contract on every step. +pub fn decode(bytes: &[u8], chunk: usize, empties: bool) -> Result { + let head = bytes.get(..bytes.len().min(64)).unwrap_or(bytes); + let config = match Webp::decode_config(head) { + Ok(ConfigStatus::Ready(c)) => c, + Ok(ConfigStatus::NeedMore) => return Err("the input ends inside the header".into()), + Err(e) => return Err(format!("{e:?}")), + }; + // A longer prefix gives the same config. + if let Ok(ConfigStatus::Ready(c2)) = Webp::decode_config(bytes) { + assert_eq!(c2, config); + } + let plen = config.rgba_len().map_err(|e| format!("{e:?}"))?; + let wlen = Webp::work_len(&config).map_err(|e| format!("{e:?}"))?; + // A mutated header may name a canvas of gigabytes: not what is tested here. + if plen > 8_000_000 || wlen > 24_000_000 { + return Err("a canvas too big for the test".into()); + } + let mut pixels = vec![CANARY; plen.saturating_add(3)]; + let mut work = vec![CANARY; wlen.saturating_add(3)]; + let mut state = Webp::start(&config).map_err(|e| format!("{e:?}"))?; + let mut frames = Vec::new(); + let mut end = End::Truncated; + { + let mut buffers = + Buffers::new(&config, wlen, &mut pixels, &mut work).map_err(|e| format!("{e:?}"))?; + assert!(state.inv(&buffers)); + let mut pos = 0usize; + let mut spent = false; + let mut empty_turn = false; + 'feed: while pos < bytes.len() { + let take = if empties && empty_turn { + 0 + } else { + chunk.min(bytes.len().saturating_sub(pos)) + }; + empty_turn = !empty_turn; + let part = bytes + .get(pos..pos.saturating_add(take)) + .ok_or("chunk out of range")?; + let result = state.step(part, &mut buffers); + assert!(state.inv(&buffers), "invariant lost at byte {pos}"); + match result { + Ok(step) => { + assert!(step.consumed <= part.len()); + match step.status { + Status::NeedMore => { + assert_eq!(step.consumed, part.len()); + pos = pos.saturating_add(take); + } + Status::FrameDone => { + frames.push((state.frame().copied(), buffers.pixels.to_vec())); + pos = pos.saturating_add(step.consumed); + } + } + } + Err(Error::NoMoreFrames) => { + end = End::Trailer; + break 'feed; + } + Err(e) => { + end = End::Error(format!("{e:?}")); + spent = true; + break 'feed; + } + } + } + if !spent && end == End::Truncated && pos >= bytes.len() { + // Everything was used; a decoder at the end of its data says so. + if let Err(Error::NoMoreFrames) = state.step(&[], &mut buffers) { + end = End::Trailer; + } + } + if spent { + assert_eq!(state.step(&[0], &mut buffers), Err(Error::Spent)); + assert!(state.inv(&buffers)); + } + } + assert!( + pixels + .get(plen..) + .is_some_and(|t| t.iter().all(|&b| b == CANARY)) + ); + assert!( + work.get(wlen..) + .is_some_and(|t| t.iter().all(|&b| b == CANARY)) + ); + Ok(Decoded { + width: config.width(), + height: config.height(), + frames, + end, + }) +} + +/// A deterministic generator (xorshift64*). +pub struct Rng(pub u64); + +impl Rng { + pub fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_F491_4F6C_DD1D) + } + + pub fn below(&mut self, n: usize) -> usize { + let n = u64::try_from(n.max(1)).unwrap_or(1); + usize::try_from(self.next().checked_rem(n).unwrap_or(0)).unwrap_or(0) + } +} + +/// One mutation of `bytes`, chosen by `rng`. +pub fn mutate(bytes: &[u8], rng: &mut Rng) -> Vec { + let mut v = bytes.to_vec(); + for _ in 0..=rng.below(3) { + let len = v.len().max(1); + let at = rng.below(len); + match rng.below(7) { + 0 => { + if let Some(b) = v.get_mut(at) { + *b = u8::try_from(rng.below(256)).unwrap_or(0); + } + } + 1 => { + if let Some(b) = v.get_mut(at) { + *b = if rng.below(2) == 0 { 0 } else { 255 }; + } + } + 2 => v.truncate(at), + 3 => { + let n = 1usize.saturating_add(rng.below(16)); + let end = at.saturating_add(n).min(v.len()); + if at < end { + v.drain(at..end); + } + } + 4 => { + let n = 1usize.saturating_add(rng.below(16)); + let end = at.saturating_add(n).min(v.len()); + if let Some(run) = v.get(at..end).map(<[u8]>::to_vec) { + for (i, b) in run.iter().enumerate() { + v.insert(at.saturating_add(i), *b); + } + } + } + 5 => { + for i in 0..4 { + if let Some(b) = v.get_mut(at.saturating_add(i)) { + *b = u8::try_from(rng.below(256)).unwrap_or(0); + } + } + } + _ => { + if let Some(b) = v.get_mut(at) { + *b ^= 1 << rng.below(8); + } + } + } + } + v +} + +/// What the reference decoder gave. +pub fn reference(bytes: &[u8]) -> Result { + let mut d = + image_webp::WebPDecoder::new(std::io::Cursor::new(bytes)).map_err(|e| format!("{e}"))?; + let (w, h) = d.dimensions(); + let size = d.output_buffer_size().ok_or("size")?; + let alpha = d.has_alpha(); + let rgba = |buf: &[u8]| -> Vec { + if alpha { + buf.to_vec() + } else { + buf.as_chunks::<3>() + .0 + .iter() + .flat_map(|c| c.iter().copied().chain([255])) + .collect() + } + }; + let mut frames = Vec::new(); + let mut buf = vec![0u8; size]; + if d.is_animated() { + let mut t = 0u32; + while d.read_frame(&mut buf).is_ok() { + t = t.saturating_add(1); + frames.push((None, rgba(&buf))); + if t > 100 { + break; + } + } + } else { + d.read_image(&mut buf).map_err(|e| format!("{e}"))?; + frames.push((None, rgba(&buf))); + } + Ok(Decoded { + width: w, + height: h, + frames: frames + .into_iter() + .map(|(_, p): (Option, Vec)| (None, p)) + .collect(), + end: End::Trailer, + }) +} diff --git a/crates/clazel-webp/tests/conformance_webp.rs b/crates/clazel-webp/tests/conformance_webp.rs new file mode 100644 index 000000000..a8589332c --- /dev/null +++ b/crates/clazel-webp/tests/conformance_webp.rs @@ -0,0 +1,313 @@ +//! Conformance of `clazel-webp` (phase-6.md item 6.8) on Wuffs' WebP test +//! data, `tests/data/wuffs-webp/` (Wuffs commit `b36a81d`, through the table +//! `cargo xtask testdata` generates), against `image-webp` 0.2.4 as the +//! reference engine. (libwebp's own test vectors have no licence, so they are +//! not vendored: phase-6.md, 6.1's inventory; `image-webp`'s output stands in +//! for libwebp's.) +//! +//! Suites, each printing a `clazel-conformance:` line +//! (`xtask/src/conformance.rs`); a unit is one file: +//! +//! - **differential**: both decoders read the file; a file passes when the +//! two agree on the size, on how many frames there are, and on **every +//! frame's canvas RGBA, exactly**. The lossy files are expected to be +//! refused as unsupported (item 6.9), each listed. +//! - **chunking**: each file through the `Decoder` interface in chunks of 1, +//! 2, 3, 7, 64, 4096 bytes, with and without empty chunks between, gives +//! exactly what one chunk of the whole file gives, with the invariant +//! checked after every step and the canary bytes past the buffers +//! untouched. +//! - **mutations**: deterministic byte mutations of each file are decoded in +//! two chunkings; none panics, the invariant holds after every step, +//! `consumed` obeys the contract, and an error leaves the decoder spent. +//! +//! The hand-made streams (every transform, mode and code path, written by a +//! small VP8L encoder of `tests/handmade`) are in `tests/handmade/mod.rs`. +//! +//! **Under Miri** (25,000 to 30,000 times slower than a native debug run, so +//! at most about 0.002 s native per test, measured with `--cfg miri` natively: +//! the 156-byte `pjw-thumbnail.lossless.webp` decodes in 0.85 ms, the +//! animation in 13 ms because its work buffer is larger) every suite is cut to +//! a stride: that one file (and the lossy `pjw-thumbnail.lossy.webp` for the +//! refusal), chunk size 1 only, 2 mutants. + +mod files { + include!("generated/wuffs_webp.rs"); +} +mod common; +mod handmade; + +use clazel_image_shape::{Buffers, ConfigStatus, Decoder, Error, Status}; +use clazel_webp::Webp; +use common::{End, Rng, decode, mutate, reference}; + +/// Files, by name and bytes (under Miri, the small ones only). +fn all_files() -> Vec<(&'static str, &'static [u8])> { + let mut v: Vec<(&str, &[u8])> = Vec::new(); + for c in files::FILES { + assert_eq!(c.sha256.len(), 64); + v.push((c.file, c.bytes)); + } + if cfg!(miri) { + v.retain(|(f, _)| *f == "pjw-thumbnail.lossless.webp"); + } + v +} + +/// Files whose outcome legitimately differs from the reference's, with the +/// reason. Each is checked to differ (a file listed here that starts to agree +/// fails the suite, so the list stays true). +const XFAIL: &[(&str, &str)] = &[ + ( + "animated-red-blue.webp", + "alpha-blended frames: image-webp rounds the blend more precisely than libwebp's integer formula, which we follow (listed choice 2); checked to agree within 1 per channel, and frame 0 exactly", + ), + ( + "pjw-thumbnail.lossy.webp", + "lossy (VP8): refused as unsupported, decoding it is item 6.9", + ), + ( + "mona-lisa.21x32.q0.lossy.webp", + "lossy (VP8): refused as unsupported, decoding it is item 6.9", + ), +]; + +fn xfail_reason(file: &str) -> Option<&'static str> { + XFAIL.iter().find(|(f, _)| *f == file).map(|(_, r)| *r) +} + +/// What differs between the two decoders on a file, if anything. +fn differences(file: &str, bytes: &[u8]) -> Option { + let ours = match decode(bytes, usize::MAX, false) { + Ok(d) => d, + Err(e) => return Some(format!("{file}: ours failed: {e}")), + }; + let theirs = match reference(bytes) { + Ok(r) => r, + Err(e) => return Some(format!("{file}: reference failed: {e}")), + }; + if (ours.width, ours.height) != (theirs.width, theirs.height) { + return Some(format!( + "{file}: size {}x{} vs {}x{}", + ours.width, ours.height, theirs.width, theirs.height + )); + } + if ours.frames.len() != theirs.frames.len() { + return Some(format!( + "{file}: {} frames vs the reference's {} (ours ended {:?})", + ours.frames.len(), + theirs.frames.len(), + ours.end + )); + } + for (i, (a, b)) in ours.frames.iter().zip(&theirs.frames).enumerate() { + if a.1 != b.1 { + let at = a.1.iter().zip(&b.1).position(|(x, y)| x != y); + return Some(format!("{file}: frame {i} differs, first at byte {at:?}")); + } + } + None +} + +#[test] +fn differential_wuffs_webp() { + let files = all_files(); + let (mut pass, mut xfail, mut fail) = (0usize, 0usize, 0usize); + let mut notes = Vec::new(); + for (file, bytes) in &files { + match (differences(file, bytes), xfail_reason(file)) { + (None, None) => { + pass += 1; + if let Ok(d) = decode(bytes, usize::MAX, false) { + println!( + " {file}: agrees ({}x{}, {} frames)", + d.width, + d.height, + d.frames.len() + ); + } + } + (Some(d), Some(why)) => { + if !near_agreement(file, bytes) { + fail += 1; + notes.push(format!( + "{file}: differs by more than the listed rounding: {d}" + )); + continue; + } + xfail += 1; + println!(" {file}: expected difference ({why}): {d}"); + } + (Some(d), None) => { + fail += 1; + notes.push(d); + } + (None, Some(why)) => { + fail += 1; + notes.push(format!("{file}: listed as different ({why}) but agrees")); + } + } + } + println!( + "clazel-conformance: suite=\"Wuffs WebP files vs image-webp 0.2.4\" pass={pass} xfail={xfail} fail={fail} total={}", + files.len() + ); + assert!(notes.is_empty(), "{}", notes.join("\n")); +} + +#[test] +fn lossy_is_refused_not_misread() -> Result<(), String> { + for c in files::FILES { + let (file, bytes) = (c.file, c.bytes); + let wanted = if cfg!(miri) { + file == "pjw-thumbnail.lossy.webp" + } else { + file.contains("lossy") + }; + if !wanted { + continue; + } + // The config is read (the size of the VP8 frame)... + let head = bytes.get(..bytes.len().min(64)).unwrap_or(bytes); + let Ok(ConfigStatus::Ready(config)) = Webp::decode_config(head) else { + return Err(format!("{file}: no config")); + }; + assert!(config.width() > 0 && config.height() > 0); + // ... and decoding says unsupported. + match decode(bytes, usize::MAX, false) { + Ok(d) => assert!( + matches!(&d.end, End::Error(e) if e.contains("Unsupported")), + "{file}: {:?}", + d.end + ), + Err(e) => return Err(format!("{file}: {e}")), + } + } + Ok(()) +} + +#[test] +fn chunking_changes_nothing() { + let sizes: &[usize] = if cfg!(miri) { + &[1] + } else { + &[1, 2, 3, 7, 64, 4096] + }; + let files = all_files(); + let (mut pass, mut fail) = (0usize, 0usize); + for (file, bytes) in &files { + let whole = decode(bytes, usize::MAX, false); + let mut ok = whole.is_ok(); + for &size in sizes { + for empties in [false, true] { + if cfg!(miri) && empties { + continue; + } + if !cfg!(miri) && bytes.len() > 40_000 && (size < 64 || empties) { + continue; + } + let got = decode(bytes, size, empties); + if got != whole { + println!(" {file}: chunk {size} (empties {empties}) differs from the whole"); + ok = false; + } + } + } + if ok { + pass += 1; + } else { + fail += 1; + } + } + println!( + "clazel-conformance: suite=\"WebP chunking: every file at chunk sizes 1 to 4096\" pass={pass} xfail=0 fail={fail} total={}", + files.len() + ); + assert_eq!(fail, 0); +} + +#[test] +fn mutants_never_panic() { + let per_file = if cfg!(miri) { 2 } else { 60 }; + let (mut total, mut errors) = (0usize, 0usize); + for (file, bytes) in all_files() { + // Mutating a megabyte file hundreds of times is not what is tested. + if !cfg!(miri) && bytes.len() > 30_000 { + continue; + } + let mut rng = Rng(0x9E37_79B9_7F4A_7C15 ^ u64::try_from(bytes.len()).unwrap_or(0)); + for n in 0..per_file { + let m = mutate(bytes, &mut rng); + for chunk in [usize::MAX, 5] { + if cfg!(miri) && chunk == 5 { + continue; + } + match decode(&m, chunk, false) { + Ok(d) => { + if matches!(d.end, End::Error(_)) { + errors += 1; + } + } + Err(_) => errors += 1, + } + total += 1; + } + let _ = (file, n); + } + } + println!("mutants decoded: {total}, ended in an error: {errors}"); + assert!(total > 0); +} + +#[test] +fn spent_after_error_and_no_more_frames() -> Result<(), String> { + let (_, bytes) = all_files() + .into_iter() + .find(|(f, _)| *f == "pjw-thumbnail.lossless.webp") + .ok_or("the file is missing")?; + let head = bytes.get(..64).unwrap_or(bytes); + let Ok(ConfigStatus::Ready(config)) = Webp::decode_config(head) else { + return Err("no config".into()); + }; + let plen = config.rgba_len().map_err(|e| format!("{e:?}"))?; + let wlen = Webp::work_len(&config).map_err(|e| format!("{e:?}"))?; + let mut pixels = vec![0u8; plen]; + let mut work = vec![0u8; wlen]; + let mut buffers = + Buffers::new(&config, wlen, &mut pixels, &mut work).map_err(|e| format!("{e:?}"))?; + let mut state = Webp::start(&config).map_err(|e| format!("{e:?}"))?; + let step = state + .step(bytes, &mut buffers) + .map_err(|e| format!("{e:?}"))?; + assert_eq!(step.status, Status::FrameDone); + assert_eq!(state.step(&[0], &mut buffers), Err(Error::NoMoreFrames)); + let mut state = Webp::start(&config).map_err(|e| format!("{e:?}"))?; + assert!(state.step(b"RIFX\0\0\0\0WEBP", &mut buffers).is_err()); + assert_eq!(state.step(&[0], &mut buffers), Err(Error::Spent)); + Ok(()) +} + +/// For the files listed as differing in blend rounding only: the same frames +/// and sizes, the first frame exactly, every channel of later frames within 1. +fn near_agreement(file: &str, bytes: &[u8]) -> bool { + if file.contains("lossy") { + return true; + } + let (Ok(ours), Ok(theirs)) = (decode(bytes, usize::MAX, false), reference(bytes)) else { + return false; + }; + ours.frames.len() == theirs.frames.len() + && ours + .frames + .iter() + .zip(&theirs.frames) + .enumerate() + .all(|(i, (a, b))| { + a.1.len() == b.1.len() + && a.1.iter().zip(&b.1).all( + |(x, y)| { + if i == 0 { x == y } else { x.abs_diff(*y) <= 1 } + }, + ) + }) +} diff --git a/crates/clazel-webp/tests/data/wuffs-webp/LICENSE.md b/crates/clazel-webp/tests/data/wuffs-webp/LICENSE.md new file mode 100644 index 000000000..87cf6995c --- /dev/null +++ b/crates/clazel-webp/tests/data/wuffs-webp/LICENSE.md @@ -0,0 +1,25 @@ +Copyright 2023 The Wuffs Authors + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/crates/clazel-webp/tests/data/wuffs-webp/SOURCE.md b/crates/clazel-webp/tests/data/wuffs-webp/SOURCE.md new file mode 100644 index 000000000..8771fab85 --- /dev/null +++ b/crates/clazel-webp/tests/data/wuffs-webp/SOURCE.md @@ -0,0 +1,28 @@ +# Wuffs WebP test data (`wuffs-webp`), for `clazel-webp` + +Upstream: https://github.com/google/wuffs +Commit: b36a81d3352604fee1fee3661f39909706b1b966 +License: MIT or Apache-2.0, our choice (Wuffs' `LICENSE-MIT` at that commit, unchanged, as `LICENSE.md`; `LICENSE-APACHE` is the alternative) + +The directory `test/data/` of that commit (`main` on 2026-11-16), the `*.webp` files item 6.8 uses: every `*.lossless.webp` is in; the lossy ones are two small files (`pjw-thumbnail.lossy.webp`, `mona-lisa.21x32.q0.lossy.webp`) kept to test that a lossy file is refused as unsupported (the rest of the lossy files are item 6.9's); `animated-red-blue.webp` is the one animation. Each file is copied byte for byte, unmodified, from `https://raw.githubusercontent.com/google/wuffs/b36a81d3352604fee1fee3661f39909706b1b966/test/data/`. + +These stand in for libwebp's test vectors (`chromium.googlesource.com/webm/libwebp-test-data`), which have no licence stated and so cannot be vendored (phase-6.md, 6.1's inventory); the reference outputs are `image-webp`'s (a dev-dependency of `clazel-webp`), not libwebp's `dwebp`. + +Provenance, from `test/data/README.md` at that commit: the `*.webp` files were generated by the `cwebp` command line tool from the same images as the GIF and PNG files of the same name: `bricks-*` are encodings of an original photo by Nigel Tao; `harvesters`, `hat`, `hibiscus.regular` and `hippopotamus` are encodings of artworks the Metropolitan Museum of Art lists as public domain; `hibiscus.primitive` is that Tiffany image run through `fogleman/primitive`; `mona-lisa.*` is derived from a Wikimedia Commons photo of the Mona Lisa; `pjw-thumbnail.*` is derived from a photo by Rob Pike; `animated-red-blue.webp` is an original animation by Nigel Tao. Wuffs' README says some files derive from third-party images; these are the ones used here. + +Files and their SHA-256 (`xtask/src/testdata/wuffs_webp.rs` checks each, and that it starts with `RIFF`, a size and `WEBP`): + +- `animated-red-blue.webp`: sha256 `f79c8ba9a32f850d9a7d311a349bccdb0a0bc902b2ac12dc475f2435f76eb993` +- `bricks-color.lossless.webp`: sha256 `70f665cbec2a1a7b5517a073100c079cadfec51c0f484b763197eeeb97a23ac6` +- `bricks-dither.lossless.webp`: sha256 `1bc1775f477cd9fdc971ae82ca89a03c9ff1239b8e5ebda449a1bef0eb01acfd` +- `bricks-gray.lossless.webp`: sha256 `44f07a8e6634ff1aa6abcc3acc59e34ece2b1417bae1a91e77c8157879500917` +- `bricks-nodither.lossless.webp`: sha256 `326fee51784d8d3362a81aeeb270fd9e15b2fd5dceacb900d4e3fa54df71bf5f` +- `harvesters.lossless.webp`: sha256 `12c135327e961d0fbc4efc994d9ad7f5bb79eadbdc2686356078ed95812d1c8c` +- `hat.lossless.webp`: sha256 `d91429191b532107311b3c16a251a65ecc1142375132c3c27ac2f1dc78f9ac80` +- `hibiscus.primitive.lossless.webp`: sha256 `f50695a2afa7f5ead0d38b2b0f2be081d0a9f642f65cb14e2b7b69689bff3098` +- `hibiscus.regular.lossless.webp`: sha256 `5ca1345294d22b9db016078f872316ec47811697f918c4427a537d9cd0814726` +- `hippopotamus.lossless.webp`: sha256 `3fdf2cae4fa5b661edb77e8526f8223e230a0ad2404098b2db4192886d328964` +- `mona-lisa.21x32.lossless.webp`: sha256 `8aeca425ffc30a355d50e9651ffe90dc434fdea9e6628ba7f2ba92e2646c586a` +- `mona-lisa.21x32.q0.lossy.webp`: sha256 `8f03717a413fabb1ea615774dbe87dce2f09a13b8bbaf53dd484d0db272621e1` +- `pjw-thumbnail.lossless.webp`: sha256 `51833d9ffda40d1b6517ad006346bae82851ccbfeddb9eb3dd5fb754f5935561` +- `pjw-thumbnail.lossy.webp`: sha256 `a28d570729953f18fe1e7dceaec3cad0c39bee0efde2cabde98025b80b225f50` diff --git a/crates/clazel-webp/tests/data/wuffs-webp/animated-red-blue.webp b/crates/clazel-webp/tests/data/wuffs-webp/animated-red-blue.webp new file mode 100644 index 000000000..6895f971f Binary files /dev/null and b/crates/clazel-webp/tests/data/wuffs-webp/animated-red-blue.webp differ diff --git a/crates/clazel-webp/tests/data/wuffs-webp/bricks-color.lossless.webp b/crates/clazel-webp/tests/data/wuffs-webp/bricks-color.lossless.webp new file mode 100644 index 000000000..4724ec6e2 Binary files /dev/null and b/crates/clazel-webp/tests/data/wuffs-webp/bricks-color.lossless.webp differ diff --git a/crates/clazel-webp/tests/data/wuffs-webp/bricks-dither.lossless.webp b/crates/clazel-webp/tests/data/wuffs-webp/bricks-dither.lossless.webp new file mode 100644 index 000000000..d498fbcb3 Binary files /dev/null and b/crates/clazel-webp/tests/data/wuffs-webp/bricks-dither.lossless.webp differ diff --git a/crates/clazel-webp/tests/data/wuffs-webp/bricks-gray.lossless.webp b/crates/clazel-webp/tests/data/wuffs-webp/bricks-gray.lossless.webp new file mode 100644 index 000000000..ddfa8c480 Binary files /dev/null and b/crates/clazel-webp/tests/data/wuffs-webp/bricks-gray.lossless.webp differ diff --git a/crates/clazel-webp/tests/data/wuffs-webp/bricks-nodither.lossless.webp b/crates/clazel-webp/tests/data/wuffs-webp/bricks-nodither.lossless.webp new file mode 100644 index 000000000..bd4674771 Binary files /dev/null and b/crates/clazel-webp/tests/data/wuffs-webp/bricks-nodither.lossless.webp differ diff --git a/crates/clazel-webp/tests/data/wuffs-webp/harvesters.lossless.webp b/crates/clazel-webp/tests/data/wuffs-webp/harvesters.lossless.webp new file mode 100644 index 000000000..b473e90b4 Binary files /dev/null and b/crates/clazel-webp/tests/data/wuffs-webp/harvesters.lossless.webp differ diff --git a/crates/clazel-webp/tests/data/wuffs-webp/hat.lossless.webp b/crates/clazel-webp/tests/data/wuffs-webp/hat.lossless.webp new file mode 100644 index 000000000..006ce7c40 Binary files /dev/null and b/crates/clazel-webp/tests/data/wuffs-webp/hat.lossless.webp differ diff --git a/crates/clazel-webp/tests/data/wuffs-webp/hibiscus.primitive.lossless.webp b/crates/clazel-webp/tests/data/wuffs-webp/hibiscus.primitive.lossless.webp new file mode 100644 index 000000000..702400d22 Binary files /dev/null and b/crates/clazel-webp/tests/data/wuffs-webp/hibiscus.primitive.lossless.webp differ diff --git a/crates/clazel-webp/tests/data/wuffs-webp/hibiscus.regular.lossless.webp b/crates/clazel-webp/tests/data/wuffs-webp/hibiscus.regular.lossless.webp new file mode 100644 index 000000000..ec53cb9d5 Binary files /dev/null and b/crates/clazel-webp/tests/data/wuffs-webp/hibiscus.regular.lossless.webp differ diff --git a/crates/clazel-webp/tests/data/wuffs-webp/hippopotamus.lossless.webp b/crates/clazel-webp/tests/data/wuffs-webp/hippopotamus.lossless.webp new file mode 100644 index 000000000..6746e303c Binary files /dev/null and b/crates/clazel-webp/tests/data/wuffs-webp/hippopotamus.lossless.webp differ diff --git a/crates/clazel-webp/tests/data/wuffs-webp/mona-lisa.21x32.lossless.webp b/crates/clazel-webp/tests/data/wuffs-webp/mona-lisa.21x32.lossless.webp new file mode 100644 index 000000000..e43fcc2c2 Binary files /dev/null and b/crates/clazel-webp/tests/data/wuffs-webp/mona-lisa.21x32.lossless.webp differ diff --git a/crates/clazel-webp/tests/data/wuffs-webp/mona-lisa.21x32.q0.lossy.webp b/crates/clazel-webp/tests/data/wuffs-webp/mona-lisa.21x32.q0.lossy.webp new file mode 100644 index 000000000..1c017e3ad Binary files /dev/null and b/crates/clazel-webp/tests/data/wuffs-webp/mona-lisa.21x32.q0.lossy.webp differ diff --git a/crates/clazel-webp/tests/data/wuffs-webp/pjw-thumbnail.lossless.webp b/crates/clazel-webp/tests/data/wuffs-webp/pjw-thumbnail.lossless.webp new file mode 100644 index 000000000..db66aa1a6 Binary files /dev/null and b/crates/clazel-webp/tests/data/wuffs-webp/pjw-thumbnail.lossless.webp differ diff --git a/crates/clazel-webp/tests/data/wuffs-webp/pjw-thumbnail.lossy.webp b/crates/clazel-webp/tests/data/wuffs-webp/pjw-thumbnail.lossy.webp new file mode 100644 index 000000000..21f22bfeb Binary files /dev/null and b/crates/clazel-webp/tests/data/wuffs-webp/pjw-thumbnail.lossy.webp differ diff --git a/crates/clazel-webp/tests/generated/wuffs_webp.rs b/crates/clazel-webp/tests/generated/wuffs_webp.rs new file mode 100644 index 000000000..a313ef43f --- /dev/null +++ b/crates/clazel-webp/tests/generated/wuffs_webp.rs @@ -0,0 +1,85 @@ +// Generated by `cargo xtask testdata` from `tests/data/wuffs-webp/` (xtask/src/testdata/wuffs_webp.rs). Do not edit. + +/// One vendored WebP file, checked against `SOURCE.md` when generated. +pub struct Case { + /// The file's name in `tests/data/wuffs-webp/`. + pub file: &'static str, + /// Its SHA-256, as `SOURCE.md` lists it. + pub sha256: &'static str, + /// Its bytes. + pub bytes: &'static [u8], +} + +/// Every file of the suite. +pub const FILES: &[Case] = &[ + Case { + file: "animated-red-blue.webp", + sha256: "f79c8ba9a32f850d9a7d311a349bccdb0a0bc902b2ac12dc475f2435f76eb993", + bytes: include_bytes!("../data/wuffs-webp/animated-red-blue.webp"), + }, + Case { + file: "bricks-color.lossless.webp", + sha256: "70f665cbec2a1a7b5517a073100c079cadfec51c0f484b763197eeeb97a23ac6", + bytes: include_bytes!("../data/wuffs-webp/bricks-color.lossless.webp"), + }, + Case { + file: "bricks-dither.lossless.webp", + sha256: "1bc1775f477cd9fdc971ae82ca89a03c9ff1239b8e5ebda449a1bef0eb01acfd", + bytes: include_bytes!("../data/wuffs-webp/bricks-dither.lossless.webp"), + }, + Case { + file: "bricks-gray.lossless.webp", + sha256: "44f07a8e6634ff1aa6abcc3acc59e34ece2b1417bae1a91e77c8157879500917", + bytes: include_bytes!("../data/wuffs-webp/bricks-gray.lossless.webp"), + }, + Case { + file: "bricks-nodither.lossless.webp", + sha256: "326fee51784d8d3362a81aeeb270fd9e15b2fd5dceacb900d4e3fa54df71bf5f", + bytes: include_bytes!("../data/wuffs-webp/bricks-nodither.lossless.webp"), + }, + Case { + file: "harvesters.lossless.webp", + sha256: "12c135327e961d0fbc4efc994d9ad7f5bb79eadbdc2686356078ed95812d1c8c", + bytes: include_bytes!("../data/wuffs-webp/harvesters.lossless.webp"), + }, + Case { + file: "hat.lossless.webp", + sha256: "d91429191b532107311b3c16a251a65ecc1142375132c3c27ac2f1dc78f9ac80", + bytes: include_bytes!("../data/wuffs-webp/hat.lossless.webp"), + }, + Case { + file: "hibiscus.primitive.lossless.webp", + sha256: "f50695a2afa7f5ead0d38b2b0f2be081d0a9f642f65cb14e2b7b69689bff3098", + bytes: include_bytes!("../data/wuffs-webp/hibiscus.primitive.lossless.webp"), + }, + Case { + file: "hibiscus.regular.lossless.webp", + sha256: "5ca1345294d22b9db016078f872316ec47811697f918c4427a537d9cd0814726", + bytes: include_bytes!("../data/wuffs-webp/hibiscus.regular.lossless.webp"), + }, + Case { + file: "hippopotamus.lossless.webp", + sha256: "3fdf2cae4fa5b661edb77e8526f8223e230a0ad2404098b2db4192886d328964", + bytes: include_bytes!("../data/wuffs-webp/hippopotamus.lossless.webp"), + }, + Case { + file: "mona-lisa.21x32.lossless.webp", + sha256: "8aeca425ffc30a355d50e9651ffe90dc434fdea9e6628ba7f2ba92e2646c586a", + bytes: include_bytes!("../data/wuffs-webp/mona-lisa.21x32.lossless.webp"), + }, + Case { + file: "mona-lisa.21x32.q0.lossy.webp", + sha256: "8f03717a413fabb1ea615774dbe87dce2f09a13b8bbaf53dd484d0db272621e1", + bytes: include_bytes!("../data/wuffs-webp/mona-lisa.21x32.q0.lossy.webp"), + }, + Case { + file: "pjw-thumbnail.lossless.webp", + sha256: "51833d9ffda40d1b6517ad006346bae82851ccbfeddb9eb3dd5fb754f5935561", + bytes: include_bytes!("../data/wuffs-webp/pjw-thumbnail.lossless.webp"), + }, + Case { + file: "pjw-thumbnail.lossy.webp", + sha256: "a28d570729953f18fe1e7dceaec3cad0c39bee0efde2cabde98025b80b225f50", + bytes: include_bytes!("../data/wuffs-webp/pjw-thumbnail.lossy.webp"), + }, +]; diff --git a/crates/clazel-webp/tests/handmade/mod.rs b/crates/clazel-webp/tests/handmade/mod.rs new file mode 100644 index 000000000..3500b88a2 --- /dev/null +++ b/crates/clazel-webp/tests/handmade/mod.rs @@ -0,0 +1,1556 @@ +//! WebP streams made by hand: a small VP8L **encoder** (written here, from +//! RFC 9649, sharing no code with the decoder) makes images that use every +//! transform, predictor mode, colour cache size, meta prefix codes, simple and +//! normal code length codes and backward references; each is decoded and must +//! equal the image it was made from, exactly, and `image-webp` must agree. +//! Container cases (still `VP8X`, chunks to skip, animations with blending and +//! disposal, and malformed files) are built the same way. +//! +//! The workspace lints apply to tests (no indexing, no unchecked arithmetic), +//! so the encoder is written with the small helpers below. +//! +//! Under Miri (25,000 to 30,000 times slower than a native debug run: about +//! 0.002 s native per test) each test runs one or two images of at most 6 by 6 +//! pixels, and the loops over seeds, sizes and cache bits run once. + +use crate::common::{End, Rng, decode, reference}; + +const CL_ORDER: [usize; 19] = [ + 17, 18, 0, 1, 2, 3, 4, 5, 16, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, +]; + +/// Figure 20 of RFC 9649, again, as `(xi, yi)`. +const MAP: [(i32, i32); 120] = [ + (0, 1), + (1, 0), + (1, 1), + (-1, 1), + (0, 2), + (2, 0), + (1, 2), + (-1, 2), + (2, 1), + (-2, 1), + (2, 2), + (-2, 2), + (0, 3), + (3, 0), + (1, 3), + (-1, 3), + (3, 1), + (-3, 1), + (2, 3), + (-2, 3), + (3, 2), + (-3, 2), + (0, 4), + (4, 0), + (1, 4), + (-1, 4), + (4, 1), + (-4, 1), + (3, 3), + (-3, 3), + (2, 4), + (-2, 4), + (4, 2), + (-4, 2), + (0, 5), + (3, 4), + (-3, 4), + (4, 3), + (-4, 3), + (5, 0), + (1, 5), + (-1, 5), + (5, 1), + (-5, 1), + (2, 5), + (-2, 5), + (5, 2), + (-5, 2), + (4, 4), + (-4, 4), + (3, 5), + (-3, 5), + (5, 3), + (-5, 3), + (0, 6), + (6, 0), + (1, 6), + (-1, 6), + (6, 1), + (-6, 1), + (2, 6), + (-2, 6), + (6, 2), + (-6, 2), + (4, 5), + (-4, 5), + (5, 4), + (-5, 4), + (3, 6), + (-3, 6), + (6, 3), + (-6, 3), + (0, 7), + (7, 0), + (1, 7), + (-1, 7), + (5, 5), + (-5, 5), + (7, 1), + (-7, 1), + (4, 6), + (-4, 6), + (6, 4), + (-6, 4), + (2, 7), + (-2, 7), + (7, 2), + (-7, 2), + (3, 7), + (-3, 7), + (7, 3), + (-7, 3), + (5, 6), + (-5, 6), + (6, 5), + (-6, 5), + (8, 0), + (4, 7), + (-4, 7), + (7, 4), + (-7, 4), + (8, 1), + (8, 2), + (6, 6), + (-6, 6), + (8, 3), + (5, 7), + (-5, 7), + (7, 5), + (-7, 5), + (8, 4), + (6, 7), + (-6, 7), + (7, 6), + (-7, 6), + (8, 5), + (7, 7), + (-7, 7), + (8, 6), + (8, 7), +]; + +// -------------------------------------------------------------- helpers + +fn add(a: usize, b: usize) -> usize { + a.wrapping_add(b) +} +fn sub(a: usize, b: usize) -> usize { + a.wrapping_sub(b) +} +fn mul(a: usize, b: usize) -> usize { + a.wrapping_mul(b) +} +/// `v[i]`, or the default. +fn at(v: &[T], i: usize) -> T { + v.get(i).copied().unwrap_or_default() +} +fn set(v: &mut [T], i: usize, x: T) { + if let Some(s) = v.get_mut(i) { + *s = x; + } +} +/// `ceil(a / 2^bits)`. +fn ceil_shr(a: usize, bits: u32) -> usize { + add(a, sub(1usize.wrapping_shl(bits), 1)).wrapping_shr(bits) +} +fn ch(p: u32, shift: u32) -> u8 { + p.wrapping_shr(shift) as u8 +} +fn px(a: u8, r: u8, g: u8, b: u8) -> u32 { + (u32::from(a) << 24) | (u32::from(r) << 16) | (u32::from(g) << 8) | u32::from(b) +} +const SHIFTS: [u32; 4] = [0, 8, 16, 24]; + +// ---------------------------------------------------------------- bits + +#[derive(Default)] +pub struct Writer { + bytes: Vec, + acc: u64, + n: u32, +} + +impl Writer { + pub fn put(&mut self, v: u32, k: u32) { + for i in 0..k { + let bit = u64::from(v.wrapping_shr(i) & 1); + self.acc |= bit.wrapping_shl(self.n); + self.n = self.n.wrapping_add(1); + if self.n == 8 { + self.bytes.push(self.acc as u8); + self.acc = 0; + self.n = 0; + } + } + } + + /// A code, first bit most significant. + fn put_code(&mut self, code: u32, len: u32) { + for i in (0..len).rev() { + self.put(code.wrapping_shr(i) & 1, 1); + } + } + + pub fn finish(mut self) -> Vec { + if self.n > 0 { + self.bytes.push(self.acc as u8); + } + self.bytes + } +} + +// ---------------------------------------------------------------- codes + +/// Lengths of a complete code over `n >= 2` symbols. +fn complete_lengths(n: usize) -> Vec { + let mut k = 1u32; + while 1usize.wrapping_shl(k) < n { + k = k.wrapping_add(1); + } + let short = sub(1usize.wrapping_shl(k), n); + (0..n) + .map(|i| { + if i < short { + k.wrapping_sub(1) as u8 + } else { + k as u8 + } + }) + .collect() +} + +/// Canonical codes for lengths (0 for unused). +fn canonical(lens: &[u8]) -> Vec { + let mut codes = vec![0u32; lens.len()]; + let mut code = 0u32; + for len in 1..=15u8 { + for (i, &l) in lens.iter().enumerate() { + if l == len { + set(&mut codes, i, code); + code = code.wrapping_add(1); + } + } + code = code.wrapping_shl(1); + } + codes +} + +/// How a code is written and used. +enum Code { + /// One symbol, zero bits. + One(u32), + /// Two symbols (low, high), one bit. + Two(u32, u32), + /// A full normal code. + Full(Vec, Vec), +} + +impl Code { + fn emit(&self, w: &mut Writer, sym: u32) { + match self { + Code::One(s) => assert_eq!(*s, sym), + Code::Two(a, b) => { + assert!(sym == *a || sym == *b); + w.put(u32::from(sym == *b && a != b), 1); + } + Code::Full(lens, codes) => { + let i = sym as usize; + w.put_code(at(codes, i), u32::from(at(lens, i))); + } + } + } +} + +/// Writes the code lengths of a normal code (section 3.7.2.1.2). +fn write_normal(w: &mut Writer, lens: &[u8]) { + let n_eff = lens.iter().rposition(|&l| l != 0).map_or(1, |p| add(p, 1)); + let trimmed = n_eff < lens.len(); + let body: &[u8] = lens + .get(..if trimmed { n_eff } else { lens.len() }) + .unwrap_or(lens); + // Tokens: (symbol, extra bits, extra value). + let mut toks: Vec<(usize, u32, u32)> = Vec::new(); + let mut prev = 8u8; + let mut i = 0; + while i < body.len() { + let v = at(body, i); + let run = body + .get(i..) + .unwrap_or(&[]) + .iter() + .take_while(|&&x| x == v) + .count(); + if v == 0 && run >= 11 { + let r = run.min(138); + toks.push((18, 7, sub(r, 11) as u32)); + i = add(i, r); + } else if v == 0 && run >= 3 { + let r = run.min(10); + toks.push((17, 3, sub(r, 3) as u32)); + i = add(i, r); + } else if v != 0 && v == prev && run >= 3 { + let r = run.min(6); + toks.push((16, 2, sub(r, 3) as u32)); + i = add(i, r); + } else { + toks.push((usize::from(v), 0, 0)); + if v != 0 { + prev = v; + } + i = add(i, 1); + } + } + let use_max = trimmed && toks.len() >= 2; + let cl_lens = complete_lengths(19); + let cl_codes = canonical(&cl_lens); + w.put(0, 1); // normal + w.put(15, 4); // num_code_lengths = 19 + for &o in &CL_ORDER { + w.put(u32::from(at(&cl_lens, o)), 3); + } + if use_max { + w.put(1, 1); + let rest = sub(toks.len(), 2); + let mut k = 0u32; + while rest >= 1usize.wrapping_shl(add(2, mul(2, k as usize)) as u32) { + k = k.wrapping_add(1); + } + w.put(k, 3); + w.put(rest as u32, add(2, mul(2, k as usize)) as u32); + } else { + w.put(0, 1); + // Without max_symbol every symbol of the alphabet must be covered. + assert_eq!(body.len(), lens.len()); + } + for (sym, extra, val) in toks { + w.put_code(at(&cl_codes, sym), u32::from(at(&cl_lens, sym))); + w.put(val, extra); + } +} + +/// Picks and writes the code for the symbols used in `hist`, over an +/// alphabet of `n` symbols. +fn make_code(w: &mut Writer, hist: &[u32], n: usize, force_full: bool) -> Code { + let used: Vec = (0..n).filter(|&s| at(hist, s) > 0).collect(); + if !force_full && used.len() <= 2 && used.iter().all(|&s| s < 256) { + w.put(1, 1); // simple + match used.as_slice() { + [] => { + w.put(0, 1); // one symbol + w.put(0, 1); // first is 1 bit + w.put(0, 1); // symbol 0 + Code::One(0) + } + [a] => { + w.put(0, 1); + if *a < 2 { + w.put(0, 1); + w.put(*a as u32, 1); + } else { + w.put(1, 1); + w.put(*a as u32, 8); + } + Code::One(*a as u32) + } + [a, b] => { + w.put(1, 1); // two symbols + w.put(1, 1); // first is 8 bits + w.put(*a as u32, 8); + w.put(*b as u32, 8); + Code::Two(*a as u32, *b as u32) + } + _ => Code::One(0), + } + } else { + let lens = complete_lengths(n); + let codes = canonical(&lens); + write_normal(w, &lens); + Code::Full(lens, codes) + } +} + +// ---------------------------------------------------------------- images + +/// A token of the entropy-coded image. +#[derive(Clone, Copy, Debug)] +enum Tok { + Lit(u32), + Cache(u32), + Copy { len: u32, code: u32 }, +} + +/// The LZ77 prefix coding of a value (libwebp's `VP8LPrefixEncode`). +fn prefix_encode(v: u32) -> (u32, u32, u32) { + if v <= 4 { + return (v.wrapping_sub(1), 0, 0); + } + let v1 = v.wrapping_sub(1); + let hb = 31u32.wrapping_sub(v1.leading_zeros()); + let second = v1.wrapping_shr(hb.wrapping_sub(1)) & 1; + let extra = hb.wrapping_sub(1); + let mask = 1u32.wrapping_shl(extra).wrapping_sub(1); + (hb.wrapping_mul(2).wrapping_add(second), extra, v1 & mask) +} + +fn cache_key(argb: u32, bits: u8) -> usize { + (0x1e35_a7bd_u32 + .wrapping_mul(argb) + .wrapping_shr(32u32.wrapping_sub(u32::from(bits)))) as usize +} + +/// The distance of code `code` in an image `w` wide. +fn dist_of(code: u32, w: usize) -> usize { + if code > 120 { + return code.wrapping_sub(120) as usize; + } + let (xi, yi) = MAP + .get(code.wrapping_sub(1) as usize) + .copied() + .unwrap_or((0, 1)); + let d = xi.wrapping_add(yi.wrapping_mul(w as i32)); + d.max(1) as usize +} + +/// How to code one image. +#[derive(Clone, Debug, Default)] +pub struct Opts { + pub cache_bits: u8, + /// Meta prefix codes: (prefix bits, groups). + pub meta: Option<(u8, u32)>, + /// Random backward references. + pub lz: bool, + /// Never use the simple code length code. + pub full_codes: bool, + /// Tries at a backward reference at each position. + pub copy_tries: u32, +} + +/// Counts one use of symbol `s` of code `c` of group `g`. +fn bump(hist: &mut [Vec>], g: usize, c: usize, s: usize) { + if let Some(slot) = hist + .get_mut(g) + .and_then(|h| h.get_mut(c)) + .and_then(|h| h.get_mut(s)) + { + *slot = slot.wrapping_add(1); + } +} + +/// Writes an entropy-coded image of `img` (`width` wide). `main` adds the +/// meta prefix part. +fn write_image(w: &mut Writer, rng: &mut Rng, img: &[u32], width: usize, o: &Opts, main: bool) { + let height = img.len().checked_div(width).unwrap_or(0); + if o.cache_bits > 0 { + w.put(1, 1); + w.put(u32::from(o.cache_bits), 4); + } else { + w.put(0, 1); + } + // Groups of the blocks. + let (mbits, ngroups, group_of): (u32, usize, Vec) = match (main, o.meta) { + (true, Some((bits, groups))) => { + let bits = u32::from(bits); + let bw = ceil_shr(width, bits); + let bh = ceil_shr(height, bits); + let mut gs: Vec = (0..mul(bw, bh)) + .map(|_| rng.below(groups as usize) as u32) + .collect(); + if let Some(last) = gs.last_mut() { + *last = groups.wrapping_sub(1); + } + w.put(1, 1); + w.put(bits.wrapping_sub(2), 3); + let entropy: Vec = gs + .iter() + .map(|&g| 0xFF00_0000 | ((g >> 8) << 16) | ((g & 0xFF) << 8)) + .collect(); + let sub = Opts { + full_codes: o.full_codes, + ..Opts::default() + }; + write_image(w, rng, &entropy, bw, &sub, false); + (bits, groups as usize, gs) + } + (true, None) => { + w.put(0, 1); + (0, 1, Vec::new()) + } + _ => (0, 1, Vec::new()), + }; + let group_at = |pos: usize| -> usize { + if ngroups == 1 { + return 0; + } + let x = pos.checked_rem(width).unwrap_or(0); + let y = pos.checked_div(width).unwrap_or(0); + let bw = ceil_shr(width, mbits); + let i = add(mul(y.wrapping_shr(mbits), bw), x.wrapping_shr(mbits)); + at(&group_of, i) as usize + }; + // Tokens. + let mut toks: Vec<(usize, Tok)> = Vec::new(); + let cache_len = if o.cache_bits > 0 { + 1usize.wrapping_shl(u32::from(o.cache_bits)) + } else { + 0 + }; + let mut cache = vec![0u32; cache_len]; + let mut pos = 0usize; + while pos < img.len() { + let g = group_at(pos); + let mut done = false; + if o.lz && pos > 0 && rng.below(3) == 0 { + for _ in 0..o.copy_tries.max(4) { + let code = if rng.below(2) == 0 { + add(1, rng.below(120)) as u32 + } else { + add(121, rng.below(pos)) as u32 + }; + let d = dist_of(code, width); + if d > pos { + continue; + } + let mut l = 0usize; + while add(pos, l) < img.len() + && l < 4096 + && at(img, add(pos, l)) == at(img, sub(add(pos, l), d)) + { + l = add(l, 1); + } + if l == 0 { + continue; + } + let l = add(1, rng.below(l)); + toks.push(( + g, + Tok::Copy { + len: l as u32, + code, + }, + )); + for k in 0..l { + let p = at(img, add(pos, k)); + if !cache.is_empty() { + set(&mut cache, cache_key(p, o.cache_bits), p); + } + } + pos = add(pos, l); + done = true; + break; + } + } + if done { + continue; + } + let p = at(img, pos); + if !cache.is_empty() { + let k = cache_key(p, o.cache_bits); + let hit = at(&cache, k) == p; + set(&mut cache, k, p); + if hit && rng.below(4) != 0 { + toks.push((g, Tok::Cache(k as u32))); + pos = add(pos, 1); + continue; + } + } + toks.push((g, Tok::Lit(p))); + pos = add(pos, 1); + } + // Histograms per group and code. + let green_n = add(280, cache_len); + let sizes = [green_n, 256, 256, 256, 40]; + let mut hist: Vec>> = (0..ngroups) + .map(|_| sizes.iter().map(|&n| vec![0u32; n]).collect()) + .collect(); + for &(g, t) in &toks { + match t { + Tok::Lit(p) => { + bump(&mut hist, g, 0, usize::from(ch(p, 8))); + bump(&mut hist, g, 1, usize::from(ch(p, 16))); + bump(&mut hist, g, 2, usize::from(ch(p, 0))); + bump(&mut hist, g, 3, usize::from(ch(p, 24))); + } + Tok::Cache(k) => bump(&mut hist, g, 0, add(280, k as usize)), + Tok::Copy { len, code } => { + bump(&mut hist, g, 0, add(256, prefix_encode(len).0 as usize)); + bump(&mut hist, g, 4, prefix_encode(code).0 as usize); + } + } + } + let none: Vec = Vec::new(); + let codes: Vec> = hist + .iter() + .map(|h| { + (0..5) + .map(|c| make_code(w, h.get(c).unwrap_or(&none), at(&sizes, c), o.full_codes)) + .collect() + }) + .collect(); + for (g, t) in toks { + let Some(c) = codes.get(g) else { continue }; + let emit = |w: &mut Writer, i: usize, s: u32| { + if let Some(code) = c.get(i) { + code.emit(w, s); + } + }; + match t { + Tok::Lit(p) => { + emit(w, 0, u32::from(ch(p, 8))); + emit(w, 1, u32::from(ch(p, 16))); + emit(w, 2, u32::from(ch(p, 0))); + emit(w, 3, u32::from(ch(p, 24))); + } + Tok::Cache(k) => emit(w, 0, k.wrapping_add(280)), + Tok::Copy { len, code } => { + let (lp, le, lv) = prefix_encode(len); + emit(w, 0, lp.wrapping_add(256)); + w.put(lv, le); + let (dp, de, dv) = prefix_encode(code); + emit(w, 4, dp); + w.put(dv, de); + } + } + } +} + +// ---------------------------------------------------------- transforms + +#[derive(Clone, Debug)] +pub enum T { + /// Size bits, and the modes (cycled over the blocks; empty is random). + Pred(u8, Vec), + /// Size bits. + Colour(u8), + Green, + /// The palette is taken from the image. + Index, +} + +fn avg2(a: u32, b: u32) -> u32 { + let mut out = 0u32; + for s in SHIFTS { + let v = (u32::from(ch(a, s)).wrapping_add(u32::from(ch(b, s)))) / 2; + out |= v.wrapping_shl(s); + } + out +} + +fn clamp(v: i32) -> u32 { + v.clamp(0, 255) as u32 +} + +fn per_channel(f: &dyn Fn(i32, i32, i32) -> i32, l: u32, t: u32, tl: u32) -> u32 { + let mut out = 0u32; + for s in SHIFTS { + let v = f( + i32::from(ch(l, s)), + i32::from(ch(t, s)), + i32::from(ch(tl, s)), + ); + out |= clamp(v).wrapping_shl(s); + } + out +} + +fn predict(mode: u8, l: u32, t: u32, tr: u32, tl: u32) -> u32 { + match mode { + 1 => l, + 2 => t, + 3 => tr, + 4 => tl, + 5 => avg2(avg2(l, tr), t), + 6 => avg2(l, tl), + 7 => avg2(l, t), + 8 => avg2(tl, t), + 9 => avg2(t, tr), + 10 => avg2(avg2(l, tl), avg2(t, tr)), + 11 => { + let (mut pl, mut pt) = (0i32, 0i32); + for s in SHIFTS { + let (a, b, c) = ( + i32::from(ch(l, s)), + i32::from(ch(t, s)), + i32::from(ch(tl, s)), + ); + let p = a.wrapping_add(b).wrapping_sub(c); + pl = pl.wrapping_add(p.wrapping_sub(a).abs()); + pt = pt.wrapping_add(p.wrapping_sub(b).abs()); + } + if pl < pt { l } else { t } + } + 12 => per_channel(&|a, b, c| a.wrapping_add(b).wrapping_sub(c), l, t, tl), + 13 => { + let a = avg2(l, t); + per_channel(&|x, _, y| x.wrapping_add(x.wrapping_sub(y) / 2), a, a, tl) + } + // 0, and 14 and 15 (listed choice 8). + _ => 0xFF00_0000, + } +} + +fn sub_px(a: u32, b: u32) -> u32 { + let mut out = 0u32; + for s in SHIFTS { + out |= u32::from(ch(a, s).wrapping_sub(ch(b, s))).wrapping_shl(s); + } + out +} + +fn delta(t: u8, c: u8) -> u8 { + (i32::from(t as i8).wrapping_mul(i32::from(c as i8)) >> 5) as u8 +} + +/// Applies the forward transforms in order and writes them; returns the +/// image the entropy coder gets and its width. +fn write_transforms( + w: &mut Writer, + rng: &mut Rng, + mut img: Vec, + mut width: usize, + ts: &[T], + sub_opts: &Opts, +) -> Result<(Vec, usize), String> { + let height = img.len().checked_div(width).unwrap_or(0); + for t in ts { + w.put(1, 1); + match t { + T::Pred(bits, modes) => { + let bits = u32::from(*bits); + w.put(0, 2); + w.put(bits.wrapping_sub(2), 3); + let bw = ceil_shr(width, bits); + let bh = ceil_shr(height, bits); + let ms: Vec = (0..mul(bw, bh)) + .map(|i| { + if modes.is_empty() { + rng.below(14) as u8 + } else { + at(modes, i.checked_rem(modes.len()).unwrap_or(0)) + } + }) + .collect(); + let mut out = img.clone(); + for y in 0..height { + for x in 0..width { + let i = add(mul(y, width), x); + let pred = if y == 0 && x == 0 { + 0xFF00_0000 + } else if y == 0 { + at(&img, sub(i, 1)) + } else if x == 0 { + at(&img, sub(i, width)) + } else { + let b = add(mul(y.wrapping_shr(bits), bw), x.wrapping_shr(bits)); + let up = sub(i, width); + predict( + at(&ms, b), + at(&img, sub(i, 1)), + at(&img, up), + at(&img, add(up, 1)), + at(&img, sub(up, 1)), + ) + }; + set(&mut out, i, sub_px(at(&img, i), pred)); + } + } + let modes_img: Vec = ms + .iter() + .map(|&m| 0xFF00_0000 | (u32::from(m) << 8)) + .collect(); + write_image(w, rng, &modes_img, bw, sub_opts, false); + img = out; + } + T::Colour(bits) => { + let bits = u32::from(*bits); + w.put(1, 2); + w.put(bits.wrapping_sub(2), 3); + let bw = ceil_shr(width, bits); + let bh = ceil_shr(height, bits); + // green_to_red, green_to_blue, red_to_blue + let elems: Vec<(u8, u8, u8)> = (0..mul(bw, bh)) + .map(|_| { + ( + rng.below(256) as u8, + rng.below(256) as u8, + rng.below(256) as u8, + ) + }) + .collect(); + let mut out = img.clone(); + for y in 0..height { + for x in 0..width { + let i = add(mul(y, width), x); + let (g2r, g2b, r2b) = elems + .get(add(mul(y.wrapping_shr(bits), bw), x.wrapping_shr(bits))) + .copied() + .unwrap_or_default(); + let p = at(&img, i); + let (r, g, b) = (ch(p, 16), ch(p, 8), ch(p, 0)); + let nr = r.wrapping_sub(delta(g2r, g)); + let nb = b.wrapping_sub(delta(g2b, g)).wrapping_sub(delta(r2b, r)); + set(&mut out, i, px(ch(p, 24), nr, g, nb)); + } + } + // Alpha 255, red red_to_blue, green green_to_blue, blue green_to_red. + let e: Vec = elems + .iter() + .map(|&(g2r, g2b, r2b)| px(255, r2b, g2b, g2r)) + .collect(); + write_image(w, rng, &e, bw, sub_opts, false); + img = out; + } + T::Green => { + w.put(2, 2); + for p in &mut img { + let g = ch(*p, 8); + *p = px( + ch(*p, 24), + ch(*p, 16).wrapping_sub(g), + g, + ch(*p, 0).wrapping_sub(g), + ); + } + } + T::Index => { + let mut pal: Vec = Vec::new(); + for &p in &img { + if !pal.contains(&p) { + pal.push(p); + } + } + if pal.len() > 256 { + return Err("more than 256 colours".into()); + } + w.put(3, 2); + w.put(sub(pal.len(), 1) as u32, 8); + let deltas: Vec = pal + .iter() + .enumerate() + .map(|(i, &p)| { + if i == 0 { + p + } else { + sub_px(p, at(&pal, sub(i, 1))) + } + }) + .collect(); + write_image(w, rng, &deltas, pal.len(), sub_opts, false); + let bits: u32 = match pal.len() { + 0..=2 => 3, + 3..=4 => 2, + 5..=16 => 1, + _ => 0, + }; + let pw = ceil_shr(width, bits); + let per = 1usize.wrapping_shl(bits); + let bpp = 8usize.wrapping_shr(bits); + let mut out = vec![0xFF00_0000u32; mul(pw, height)]; + for y in 0..height { + for x in 0..width { + let want = at(&img, add(mul(y, width), x)); + let idx = pal.iter().position(|&c| c == want).unwrap_or(0) as u32; + let slot = x.checked_rem(per).unwrap_or(0); + let o = add(mul(y, pw), x.wrapping_shr(bits)); + let shift = add(8, mul(bpp, slot)) as u32; + let cur = at(&out, o); + set(&mut out, o, cur | idx.wrapping_shl(shift)); + } + } + img = out; + width = pw; + } + } + } + w.put(0, 1); + Ok((img, width)) +} + +/// A complete VP8L chunk payload for `argb` (`w` by `h`). +pub fn vp8l_payload( + rng: &mut Rng, + argb: &[u32], + w: usize, + h: usize, + ts: &[T], + o: &Opts, +) -> Result, String> { + assert_eq!(argb.len(), mul(w, h)); + let mut wr = Writer::default(); + wr.put(0x2F, 8); + wr.put(sub(w, 1) as u32, 14); + wr.put(sub(h, 1) as u32, 14); + wr.put(1, 1); + wr.put(0, 3); + let subopts = Opts { + cache_bits: o.cache_bits.min(2), + lz: false, + full_codes: o.full_codes, + ..Opts::default() + }; + let (img, width) = write_transforms(&mut wr, rng, argb.to_vec(), w, ts, &subopts)?; + write_image(&mut wr, rng, &img, width, o, true); + Ok(wr.finish()) +} + +// ----------------------------------------------------------- containers + +pub fn chunk(fourcc: &[u8; 4], payload: &[u8]) -> Vec { + let mut v = fourcc.to_vec(); + v.extend((payload.len() as u32).to_le_bytes()); + v.extend(payload); + if payload.len() % 2 == 1 { + v.push(0); + } + v +} + +pub fn riff(chunks: &[Vec]) -> Vec { + let body: Vec = chunks.concat(); + let mut v = b"RIFF".to_vec(); + v.extend((body.len() as u32).wrapping_add(4).to_le_bytes()); + v.extend(b"WEBP"); + v.extend(body); + v +} + +fn u24(v: u32) -> [u8; 3] { + let [a, b, c, _] = v.to_le_bytes(); + [a, b, c] +} + +pub fn vp8x(flags: u8, w: u32, h: u32) -> Vec { + let mut p = vec![flags, 0, 0, 0]; + p.extend(u24(w.wrapping_sub(1))); + p.extend(u24(h.wrapping_sub(1))); + chunk(b"VP8X", &p) +} + +pub struct Frame { + pub x: u32, + pub y: u32, + pub w: u32, + pub h: u32, + pub ms: u32, + pub no_blend: bool, + pub dispose_bg: bool, + pub argb: Vec, +} + +pub fn anmf(f: &Frame, rng: &mut Rng) -> Result, String> { + let mut p = Vec::new(); + p.extend(u24(f.x / 2)); + p.extend(u24(f.y / 2)); + p.extend(u24(f.w.wrapping_sub(1))); + p.extend(u24(f.h.wrapping_sub(1))); + p.extend(u24(f.ms)); + p.push((u8::from(f.no_blend) << 1) | u8::from(f.dispose_bg)); + let payload = vp8l_payload( + rng, + &f.argb, + f.w as usize, + f.h as usize, + &[], + &Opts::default(), + )?; + p.extend(chunk(b"VP8L", &payload)); + Ok(chunk(b"ANMF", &p)) +} + +pub fn rgba(argb: &[u32]) -> Vec { + argb.iter() + .flat_map(|&p| [ch(p, 16), ch(p, 8), ch(p, 0), ch(p, 24)]) + .collect() +} + +/// Random image with a few "regions" so that references and caches occur. +pub fn image(rng: &mut Rng, w: usize, h: usize, colours: usize, alpha: bool) -> Vec { + let pal: Vec = (0..colours.max(1)) + .map(|_| { + let a = if alpha { rng.below(256) as u32 } else { 255 }; + (a << 24) | (rng.next() as u32 & 0x00FF_FFFF) + }) + .collect(); + let mut img: Vec = Vec::with_capacity(mul(w, h)); + for i in 0..mul(w, h) { + let p = if i > w && rng.below(3) == 0 { + at(&img, sub(i, w)) + } else if i > 0 && rng.below(3) == 0 { + at(&img, sub(i, 1)) + } else { + at(&pal, rng.below(pal.len())) + }; + img.push(p); + } + img +} + +// ---------------------------------------------------------------- checks + +/// Decodes `file` whole and in chunks of 1, and compares with `want` and +/// with `image-webp`. +fn check(name: &str, file: &[u8], want: &[u8]) -> Result<(), String> { + let whole = decode(file, usize::MAX, false).map_err(|e| format!("{name}: {e}"))?; + let frame = whole + .frames + .first() + .ok_or_else(|| format!("{name}: no frame, ended {:?}", whole.end))?; + if frame.1 != want { + let first = frame.1.iter().zip(want).position(|(a, b)| a != b); + return Err(format!("{name}: pixels differ, first at byte {first:?}")); + } + let bytewise = decode(file, 1, false).map_err(|e| format!("{name}: {e}"))?; + if bytewise != whole { + return Err(format!("{name}: byte-at-a-time differs from whole")); + } + if !cfg!(miri) { + let theirs = reference(file).map_err(|e| format!("{name}: reference: {e}"))?; + let their_frame = theirs.frames.first().ok_or("reference has no frame")?; + if their_frame.1 != want { + return Err(format!( + "{name}: image-webp disagrees with the source image" + )); + } + } + Ok(()) +} + +fn still( + rng: &mut Rng, + argb: &[u32], + w: usize, + h: usize, + ts: &[T], + o: &Opts, +) -> Result, String> { + let payload = vp8l_payload(rng, argb, w, h, ts, o)?; + Ok(riff(&[chunk(b"VP8L", &payload)])) +} + +/// A case: its name, the file made (or why it could not be), the pixels wanted. +type Case = (String, Result, String>, Vec); + +fn run(cases: Vec, suite: &str) { + let (mut pass, mut fail) = (0usize, 0usize); + let total = cases.len(); + let mut notes = Vec::new(); + for (name, file, want) in cases { + match file.and_then(|f| check(&name, &f, &want)) { + Ok(()) => pass = add(pass, 1), + Err(e) => { + fail = add(fail, 1); + notes.push(e); + } + } + } + println!("clazel-conformance: suite=\"{suite}\" pass={pass} xfail=0 fail={fail} total={total}"); + assert!(notes.is_empty(), "{}", notes.join("\n")); +} + +fn seeds() -> u64 { + if cfg!(miri) { 1 } else { 6 } +} + +#[test] +fn transforms_in_every_combination() { + let orders: Vec> = vec![ + vec![], + vec![T::Green], + vec![T::Pred(2, vec![]), T::Green], + vec![T::Colour(3)], + vec![T::Green, T::Colour(2), T::Pred(3, vec![])], + vec![T::Pred(2, vec![]), T::Colour(2), T::Green], + vec![T::Index], + vec![T::Index, T::Pred(2, vec![])], + vec![T::Index, T::Colour(2), T::Green], + vec![T::Green, T::Index], + ]; + let sizes: &[(usize, usize)] = if cfg!(miri) { + &[(3, 2)] + } else { + &[(1, 1), (2, 3), (5, 4), (9, 7), (17, 9), (33, 5)] + }; + let mut cases: Vec = Vec::new(); + for seed in 0..seeds() { + for (oi, ts) in orders.iter().enumerate() { + if cfg!(miri) && oi != 4 { + continue; + } + for &(w, h) in sizes { + let salt = seed + .wrapping_mul(7919) + .wrapping_add((oi as u64).wrapping_mul(31)); + let mut rng = Rng(0x1234_5678u64 + .wrapping_add(salt) + .wrapping_add(mul(w, h) as u64)); + // Few colours so that indexing is possible at any position. + let img = image(&mut rng, w, h, 6, seed % 2 == 0); + let name = format!("order {oi} {w}x{h} seed {seed}"); + let file = still( + &mut rng, + &img, + w, + h, + ts, + &Opts { + cache_bits: (seed % 3) as u8 * 2, + lz: seed % 2 == 1, + ..Opts::default() + }, + ); + cases.push((name, file, rgba(&img))); + } + } + } + // A combination the image does not allow (an indexing after a transform + // that makes more than 256 colours) is not a case: it is dropped. + cases.retain(|(_, f, _)| f.is_ok()); + assert!(!cases.is_empty()); + run(cases, "WebP hand-made: transforms in combination"); +} + +#[test] +fn every_predictor_mode() { + // 16 blocks of 4 by 4: the modes 0 to 13 in order, then 0 and 1 again. + let modes: Vec = (0..14).collect(); + let (w, h) = if cfg!(miri) { (8, 4) } else { (16, 16) }; + let mut cases: Vec = Vec::new(); + for seed in 0..seeds() { + let mut rng = Rng(77u64.wrapping_add(seed)); + let img = image(&mut rng, w, h, if seed % 2 == 0 { 200 } else { 5 }, true); + let name = format!("modes 0..13 seed {seed}"); + let file = still( + &mut rng, + &img, + w, + h, + &[T::Pred(2, modes.clone())], + &Opts::default(), + ); + cases.push((name, file, rgba(&img))); + } + run(cases, "WebP hand-made: every predictor mode"); +} + +#[test] +fn pixel_bundling_at_every_palette_size() { + let widths: &[usize] = if cfg!(miri) { + &[3] + } else { + &[1, 2, 3, 7, 8, 9, 15, 16, 17] + }; + let sizes: &[usize] = if cfg!(miri) { + &[3] + } else { + &[1, 2, 3, 4, 5, 16, 17, 100, 256] + }; + let mut cases: Vec = Vec::new(); + for &n in sizes { + for &w in widths { + let mut rng = Rng(5u64.wrapping_add(add(mul(n, 100), w) as u64)); + let h = 3; + // The palette is exactly n colours, and each appears. + let pal: Vec = (0..n) + .map(|i| 0xFF00_0000 | (i as u32).wrapping_mul(0x0001_0203)) + .collect(); + let mut img: Vec = (0..mul(w, h)) + .map(|i| at(&pal, add(mul(i, 7), i / 3).checked_rem(n).unwrap_or(0))) + .collect(); + for (i, &c) in pal.iter().enumerate().take(mul(w, h)) { + set(&mut img, i, c); + } + let name = format!("palette {n} width {w}"); + let file = still(&mut rng, &img, w, h, &[T::Index], &Opts::default()); + cases.push((name, file, rgba(&img))); + } + } + cases.retain(|(_, f, _)| f.is_ok()); + run(cases, "WebP hand-made: colour indexing and pixel bundling"); +} + +#[test] +fn colour_caches_and_meta_codes() { + let mut cases: Vec = Vec::new(); + let bits_list: Vec = if cfg!(miri) { + vec![2] + } else { + (0..=11).collect() + }; + for seed in 0..seeds() { + for &bits in &bits_list { + let mut rng = Rng(900u64 + .wrapping_add(seed.wrapping_mul(13)) + .wrapping_add(u64::from(bits))); + let (w, h) = if cfg!(miri) { (5, 4) } else { (23, 19) }; + let img = image(&mut rng, w, h, 40, true); + let meta = if seed % 2 == 0 { + None + } else { + Some((2 + (seed % 3) as u8, 2 + (seed % 5) as u32)) + }; + let name = format!("cache {bits} meta {meta:?} seed {seed}"); + let file = still( + &mut rng, + &img, + w, + h, + &[], + &Opts { + cache_bits: bits, + meta, + lz: true, + copy_tries: 8, + ..Opts::default() + }, + ); + cases.push((name, file, rgba(&img))); + } + } + run( + cases, + "WebP hand-made: colour cache sizes and meta prefix codes", + ); +} + +#[test] +fn backward_references_and_simple_codes() { + let mut cases: Vec = Vec::new(); + for seed in 0..seeds().wrapping_mul(2) { + let mut rng = Rng(4242u64.wrapping_add(seed)); + let (w, h) = if cfg!(miri) { + (6, 5) + } else { + (add(21, (seed % 4) as usize), 17) + }; + // Two colours and long runs: references of every kind, and simple codes. + let img = image(&mut rng, w, h, if seed % 3 == 0 { 1 } else { 2 }, false); + let name = format!("lz {w}x{h} seed {seed}"); + let file = still( + &mut rng, + &img, + w, + h, + &[], + &Opts { + lz: true, + copy_tries: 20, + full_codes: seed % 4 == 3, + ..Opts::default() + }, + ); + cases.push((name, file, rgba(&img))); + } + run( + cases, + "WebP hand-made: backward references, simple and normal codes", + ); +} + +// ------------------------------------------------------- the container + +#[test] +fn still_extended_with_chunks_to_skip() -> Result<(), String> { + let mut rng = Rng(31); + let (w, h) = (5usize, 4usize); + let img = image(&mut rng, w, h, 4, true); + let payload = vp8l_payload(&mut rng, &img, w, h, &[], &Opts::default())?; + let file = riff(&[ + vp8x(0x10 | 0x20 | 0x08, w as u32, h as u32), + chunk(b"ICCP", &[1, 2, 3]), + chunk(b"ALPH", &[0, 0]), + chunk(b"VP8L", &payload), + chunk(b"EXIF", &[9; 5]), + chunk(b"XMP ", &[]), + chunk(b"junk", &[7; 4]), + ]); + check("still VP8X", &file, &rgba(&img))?; + // The same with data after the RIFF size: ignored. + let mut tail = file.clone(); + tail.extend([1, 2, 3, 4, 5]); + check("RIFF with trailing data", &tail, &rgba(&img)) +} + +/// The compositing rule of `lib.rs`'s listed choice 2, written again. +fn blend(src: [u8; 4], dst: [u8; 4]) -> [u8; 4] { + let [sr, sg, sb, sa8] = src; + let [dr, dg, db, da8] = dst; + let (sa, da) = (u32::from(sa8), u32::from(da8)); + if sa == 0 { + return dst; + } + if sa == 255 || da == 0 { + return src; + } + let factor = da.wrapping_mul(256u32.wrapping_sub(sa)) >> 8; + let ba = sa.wrapping_add(factor); + let scale = (1u32 << 24).checked_div(ba).unwrap_or(0); + let c = |s: u8, d: u8| { + let sum = u32::from(s) + .wrapping_mul(sa) + .wrapping_add(u32::from(d).wrapping_mul(factor)); + (sum.wrapping_mul(scale) >> 24) as u8 + }; + [c(sr, dr), c(sg, dg), c(sb, db), ba as u8] +} + +#[test] +fn animation_composites_frames() -> Result<(), String> { + let (cw, chh) = (8u32, 6u32); + let mut rng = Rng(555); + let mk = + |rng: &mut Rng, w: u32, h: u32, alpha: bool| image(rng, w as usize, h as usize, 3, alpha); + let f = |x, y, w, h, ms, no_blend, dispose_bg, argb| Frame { + x, + y, + w, + h, + ms, + no_blend, + dispose_bg, + argb, + }; + let mut frames = vec![ + f(0, 0, cw, chh, 10, true, false, mk(&mut rng, cw, chh, false)), + f(2, 2, 4, 3, 20, false, true, mk(&mut rng, 4, 3, true)), + f(4, 0, 3, 6, 30, false, false, mk(&mut rng, 3, 6, true)), + f(0, 4, 6, 2, 40, true, true, mk(&mut rng, 6, 2, true)), + f(2, 2, 2, 2, 50, false, false, mk(&mut rng, 2, 2, true)), + ]; + if cfg!(miri) { + frames.truncate(2); + } + let mut chunks = vec![vp8x(0x02 | 0x10, cw, chh)]; + let mut anim = vec![1, 2, 3, 4]; + anim.extend(7u16.to_le_bytes()); + chunks.push(chunk(b"ANIM", &anim)); + for fr in &frames { + chunks.push(anmf(fr, &mut rng)?); + } + chunks.push(chunk(b"EXIF", &[1, 2, 3])); + let file = riff(&chunks); + let got = decode(&file, usize::MAX, false)?; + if got.end != End::Trailer || got.frames.len() != frames.len() { + return Err(format!("{} frames, ended {:?}", got.frames.len(), got.end)); + } + // Expected canvases, composited here. + let mut canvas = vec![[0u8; 4]; mul(cw as usize, chh as usize)]; + let mut prev: Option<&Frame> = None; + for (i, fr) in frames.iter().enumerate() { + if let Some(p) = prev + && p.dispose_bg + { + for y in p.y..p.y.wrapping_add(p.h) { + for x in p.x..p.x.wrapping_add(p.w) { + set( + &mut canvas, + add(mul(y as usize, cw as usize), x as usize), + [0; 4], + ); + } + } + } + for y in 0..fr.h { + for x in 0..fr.w { + let s = at(&fr.argb, add(mul(y as usize, fr.w as usize), x as usize)); + let src = [ch(s, 16), ch(s, 8), ch(s, 0), ch(s, 24)]; + let cx = fr.x.wrapping_add(x) as usize; + let cy = fr.y.wrapping_add(y) as usize; + let to = add(mul(cy, cw as usize), cx); + let old = at(&canvas, to); + set( + &mut canvas, + to, + if fr.no_blend { src } else { blend(src, old) }, + ); + } + } + let want: Vec = canvas.iter().flatten().copied().collect(); + let (info, pixels) = got.frames.get(i).ok_or("a frame is missing")?; + if *pixels != want { + return Err(format!("frame {i}: canvas differs")); + } + let info = info.ok_or("no frame facts")?; + if ( + info.left, + info.top, + info.width, + info.height, + info.duration_ms, + ) != (fr.x, fr.y, fr.w, fr.h, fr.ms) + { + return Err(format!("frame {i}: facts {info:?}")); + } + prev = Some(fr); + } + // In chunks of 1 with empty chunks between, the same. + if decode(&file, 1, true)? != got { + return Err("chunked animation differs".into()); + } + Ok(()) +} + +fn error_of(file: &[u8]) -> Result { + match decode(file, usize::MAX, false) { + Ok(d) => match d.end { + End::Error(e) => Ok(e), + other => Err(format!("no error: {other:?}, {} frames", d.frames.len())), + }, + Err(e) => Ok(e), + } +} + +/// `(name, file, text the error must mention)`. +type Bad = (&'static str, Vec, &'static str); + +#[test] +fn malformed_files_are_errors() -> Result<(), String> { + let mut rng = Rng(9); + let (w, h) = (4usize, 3usize); + let img = image(&mut rng, w, h, 3, false); + let payload = vp8l_payload(&mut rng, &img, w, h, &[], &Opts::default())?; + let good = riff(&[chunk(b"VP8L", &payload)]); + check("the good file", &good, &rgba(&img))?; + let with = |f: &dyn Fn(&mut Vec)| { + let mut p = payload.clone(); + f(&mut p); + riff(&[chunk(b"VP8L", &p)]) + }; + let short = payload + .get(..payload.len().saturating_sub(3)) + .unwrap_or(&[]); + // The RIFF size claims less than the chunk holds. + let mut cut = good.clone(); + for (i, b) in 20u32.to_le_bytes().iter().enumerate() { + set(&mut cut, add(4, i), *b); + } + let cases: Vec = vec![ + ( + "bad signature byte", + with(&|p| set(p, 0, 0x2E)), + "signature", + ), + ( + "version 1", + with(&|p| { + let v = at(p, 4) | 0x20; + set(p, 4, v); + }), + "version", + ), + ("truncated data", riff(&[chunk(b"VP8L", short)]), "ends"), + ("a chunk past the RIFF end", cut, "past"), + ( + "lossy", + riff(&[chunk( + b"VP8 ", + &[0x10, 0, 0, 0x9D, 0x01, 0x2A, 4, 0, 3, 0, 0, 0], + )]), + "Unsupported", + ), + ( + "VP8X canvas disagrees", + riff(&[vp8x(0, 5, 3), chunk(b"VP8L", &payload)]), + "size its container says", + ), + ( + "no image", + riff(&[vp8x(0, 4, 3), chunk(b"EXIF", &[1, 2])]), + "no image", + ), + ( + "animation flag without ANIM", + riff(&[vp8x(2, 4, 3), chunk(b"VP8L", &payload)]), + "outside a frame", + ), + ]; + for (name, file, needle) in cases { + let e = error_of(&file)?; + if !e.contains(needle) { + return Err(format!("{name}: error {e:?} does not mention {needle:?}")); + } + } + Ok(()) +} + +#[test] +fn malformed_streams_are_errors() -> Result<(), String> { + // Streams a decoder must refuse, written bit by bit. + let hdr = |w: &mut Writer| { + w.put(0x2F, 8); + w.put(1, 14); + w.put(1, 14); + w.put(0, 4); + }; + let wrap = |w: Writer| riff(&[chunk(b"VP8L", &w.finish())]); + let mut cases: Vec = Vec::new(); + // The same transform twice. + let mut w = Writer::default(); + hdr(&mut w); + w.put(1, 1); + w.put(2, 2); + w.put(1, 1); + w.put(2, 2); + cases.push(("transform twice", wrap(w), "twice")); + // Colour cache of 12 bits (and of 0 bits). + for (bits, name) in [(12u32, "cache 12"), (0, "cache 0")] { + let mut w = Writer::default(); + hdr(&mut w); + w.put(0, 1); + w.put(1, 1); + w.put(bits, 4); + cases.push((name, wrap(w), "colour cache")); + } + // A normal code with 4 code length codes all of length 0. + let mut w = Writer::default(); + hdr(&mut w); + w.put(0, 1); // no transform + w.put(0, 1); // no cache + w.put(0, 1); // no meta + w.put(0, 1); // normal code + w.put(0, 4); // 4 code lengths + w.put(0, 12); + cases.push(("empty code length code", wrap(w), "code length code")); + for (name, file, needle) in cases { + let e = error_of(&file)?; + if !e.contains(needle) { + return Err(format!("{name}: error {e:?} does not mention {needle:?}")); + } + } + Ok(()) +} + +/// Modes 14 and 15, which RFC 9649 does not define, predict like mode 0 +/// (listed choice 8, libwebp's); `image-webp` does not, so only ours is asked. +#[test] +fn undefined_predictor_modes_are_black() -> Result<(), String> { + let (w, h) = (8usize, 4usize); + let mut rng = Rng(14); + let img = image(&mut rng, w, h, 5, true); + let a = still( + &mut rng, + &img, + w, + h, + &[T::Pred(2, vec![14, 15])], + &Opts::default(), + )?; + let got = decode(&a, usize::MAX, false)?; + let frame = got.frames.first().ok_or("no frame")?; + if frame.1 != rgba(&img) { + return Err("modes 14 and 15 do not predict like mode 0".into()); + } + Ok(()) +} diff --git a/docs/DEPENDENCIES.md b/docs/DEPENDENCIES.md index 864a5e744..364e4c85e 100644 --- a/docs/DEPENDENCIES.md +++ b/docs/DEPENDENCIES.md @@ -255,6 +255,9 @@ Changing a row is a reviewable act. Adding one means saying why below. | unicode-properties | 0.1.5 | test | Unicode General_Category (the only feature `rustybuzz` enables), required by `rustybuzz`. No dependencies. | | unicode-script | 0.5.9 | test | The Unicode Script property, required by `rustybuzz`. No dependencies. | | ab_glyph_rasterizer | 0.1.10 | test | Alex Butler's coverage rasterizer (used by `ab_glyph`, `glyph_brush` and `wgpu_glyph`), a descendant of `font-rs`: the reference engine in `clazel-glyph-mask`'s differential test (`tests/conformance_masks.rs`, phase-5.md item 5.17), the same signed-area accumulation in `f32` written by another author. Used without default features but with `std`, which is its only default; `cargo tree` shows no dependencies (`libm` is optional and off). It has `unsafe` only to call its line function through a pointer chosen by CPU feature detection (all variants are the same scalar code). Pinned to 0.1.10, its latest release. | +| image-webp | 0.2.4 | test | image-rs's WebP decoder and encoder (used by `image`), written by others: the reference engine in `clazel-webp`'s differential test (`tests/conformance_webp.rs`, phase-6.md item 6.8), whose pixels and animation frames the shipped decoder's are compared with, exactly (alpha-blended frames within 1, a listed difference), and which decodes every stream the test's VP8L encoder makes. Default features only (it has none that matter). Its dependencies are `byteorder-lite` and `quick-error`. Pinned to 0.2.4, its latest release when added (2026-11-18). | +| byteorder-lite | 0.1.0 | test | A small reader and writer of integers in either byte order, required by `image-webp`. No dependencies. | +| quick-error | 2.0.1 | test | A macro that defines error enums, required by `image-webp`. No dependencies. | | naga | 30.0.1 | tooling | The wgpu project's shader translator: `xtask/src/shaders.rs` compiles `clazel-gpu`'s GLSL to the committed SPIR-V with it and `cargo xtask check` confirms the two agree (phase-5.md item 5.17b: "compiled from GLSL by tooling that never ships"). No SPIR-V compiler is installed on the development machine, and this keeps the check in Rust. Features `glsl-in`, `spv-out`, and `wgsl-in`, which is unused but without which 30.0.1 does not build. | | naga-types | 30.0.1 | tooling | Required by `naga` (GLSL to SPIR-V for `clazel-gpu`'s shaders, in `xtask` only). | | arrayvec | 0.7.8 | tooling | Required by `naga` (GLSL to SPIR-V for `clazel-gpu`'s shaders, in `xtask` only). | diff --git a/docs/phases/phase-6.md b/docs/phases/phase-6.md index 295fb4173..0f883ca6a 100644 --- a/docs/phases/phase-6.md +++ b/docs/phases/phase-6.md @@ -892,6 +892,7 @@ another item's proof or tier to proceed, only its interface. | 6.0 Decisions | taken (2026-11-16): decoders at tier 1 by the maintainer; (a) to (j) as recommended, by a Claude session on the maintainer's delegation | | 6.1 Inventory and test data | built (2026-11-16); docs and data, no crate, nothing proven. `scripts/phase6-inventory.mjs` ran (not in CI) over HN front, item, login and reply pages and one Wikipedia article: HN's logo and vote arrows are SVG, so 6.11a was added; the only raster is `s.gif`. WPT forms, img element and `urlencoded-parser` vendored at `4ad507b2`; no table is generated yet and nothing reads them. Not done: no news site other than HN measured; format suites (PngSuite, libjpeg-turbo, Wuffs, DEFLATE) left to their items, licences recorded in 6.1; libwebp-test-data has no licence and the ITU-T T.83 streams are a gap; no dev-dependencies added | | 6.3 Image buffers and the decoder shape | built (2026-11-16) as the new crate `clazel-image-shape` (tier 0, target 1; no dependencies; separate from `clazel-image` to avoid a cycle and tier 0 code under tier 1 decoders). The `Decoder` trait (`decode_config`, `work_len`, `start`, `step`, `inv`), `Buffers`, `Config`, the 6.0(j) limits with a checked `rgba_len`, `premultiply`, the `decoder_step_harness!` Kani template and a toy raw-format decoder instantiating it. Proven: 6 Kani harnesses run in WSL, 6 pass (size function over every `u32` pair; premultiplication over every channel and alpha, in place over 3 pixels and a partial one; the template on the toy at images up to 2 by 2 and chunks up to 8 bytes; a mutated toy step fails it). Tested: 14 unit tests (toy at every chunk size, limits, exact rounding). Not done: no real decoder yet; `FrameDone` carries no frame timing (GIF, APNG and WebP add it per 6.0(d)); the template is untried from another crate; full `cargo xtask verify` and CI not run, so the tier stays 0 | +| 6.8 WebP: container and lossless | built (2026-11-19) as the new crate `clazel-webp` (tier 0, target 1; only dependency `clazel-image-shape`; `image-webp` 0.2.4 with `byteorder-lite` and `quick-error` are dev-dependencies, class `test`). RFC 9649 (pinned, SHA-256 in `lib.rs`): RIFF, `VP8X`, `ANIM` and `ANMF` with **every frame composited** (disposal, blending; `Webp::frame()`), `ICCP`/`ALPH`/`EXIF`/`XMP `/unknown chunks skipped, `VP8 ` recognised (its size read by `decode_config`) and refused with `Error::Unsupported` (item 6.9); VP8L: the four transforms (predictor with its fourteen modes, colour, subtract green, colour indexing with pixel bundling), colour cache, meta prefix codes, simple and normal code length codes, LZ77 backward references with the distance map; twelve listed choices in `lib.rs`. The lossless decoder is a byte-at-a-time machine of atomic actions over caller-owned buffers; prefix codes are decoded without tables; the work buffer is about 11.6 kB plus 12 bytes per 4 by 4 block, 6.4 kB per prefix code group held (at most 1,024) and, for an animation, the canvas again. Kani: **69 harnesses**, not yet green on CI (prefix code construction over any code lengths, decoding, the distance map, every predictor mode, each inverse transform over any row of 3 pixels, colour indexing at every bundling width in place, blending, and the step function in parts: one atomic action of the lossless decoder per phase, finishing an image of each role and transform kind, one container byte per phase; the whole `step` is not attempted, as for GIF). The first CI run lost its runner after two hours on the original 16, whose step harnesses had a symbolic phase; split, and each run on its own locally under Kani 0.68.0 with a 4 GB cap, they found gaps the invariants and the decoder now close (see the commits). Tested: 22 unit tests; Wuffs' WebP files (`tests/data/wuffs-webp`, commit `b36a81d`, through a new converter in `xtask/src/testdata/wuffs_webp.rs`; libwebp's own vectors have no licence, so they stand in) against `image-webp`: **all 11 lossless files exact** (up to 1,165 by 859), the animation exact on its first frame and within 1 per channel on blended ones (listed: `image-webp` rounds blending more precisely than libwebp's formula), 2 lossy files refused; every file at chunk sizes 1 to 4096 equal; 1,440 byte mutants never panic; a VP8L encoder written for the tests makes 531 images (ten orders of transforms, every predictor mode, bundling at every palette size, caches of 0 to 11 bits, meta prefix codes, references of every distance code) that decode to exactly the source and agree with `image-webp`; an animation of five frames against a compositor written in the test; twelve malformed inputs. Miri: every suite runs a stride sized to about 0.002 s native (stated in the test headers); not run here. Not done: lossy VP8 and `ALPH` (6.9); the loop count and frame timing (6.19); more than 1,024 prefix code groups; CI's Kani run and full verification not green yet (CI's Miri job for the crate passed), so the tier stays 0 | | 6.2 and 6.4 to 6.21 and 6.11a | not started | **Phase 6: planned (2026-11-16); decisions in 6.0 taken the same day. diff --git a/xtask/src/status.rs b/xtask/src/status.rs index eb2698882..a2ed629d9 100644 --- a/xtask/src/status.rs +++ b/xtask/src/status.rs @@ -72,6 +72,7 @@ const PLACES: &[(&str, Place)] = &[ ("clazel-layout", Place::Renderer), ("clazel-glyph-mask", Place::Renderer), ("clazel-image-shape", Place::Renderer), + ("clazel-webp", Place::Renderer), // The GPU process (item 5.17b): sandboxed and one per tab like the renderer. // The detail image has no panel for it yet; item 5.27 adds one. ("clazel-gpu", Place::Renderer), diff --git a/xtask/src/testdata.rs b/xtask/src/testdata.rs index 0945c7247..363410c80 100644 --- a/xtask/src/testdata.rs +++ b/xtask/src/testdata.rs @@ -42,6 +42,7 @@ mod wpt_url_percent; mod wpt_url_setters; mod wpt_urltestdata; mod wpt_webstorage; +mod wuffs_webp; mod wycheproof_ecdsa; mod wycheproof_rsa; mod x509_limbo; @@ -190,6 +191,7 @@ const CONVERTERS: &[(&str, Convert, &[&str])] = &[ wycheproof_rsa::FILES, ), ("x509-limbo", x509_limbo::convert, x509_limbo::FILES), + ("wuffs-webp", wuffs_webp::convert, wuffs_webp::FILES), ]; /// A converter for a binary artefact written beside a suite's Rust table. diff --git a/xtask/src/testdata/fonts.rs b/xtask/src/testdata/fonts.rs index ec2729380..4f94f458e 100644 --- a/xtask/src/testdata/fonts.rs +++ b/xtask/src/testdata/fonts.rs @@ -51,7 +51,7 @@ fn files_of(name: &str) -> Result<&'static [&'static str], String> { /// The SHA-256 that `SOURCE.md` gives for `file`, from the line that names /// it in backquotes. -fn listed_hash(source: &str, file: &str) -> Result { +pub(super) fn listed_hash(source: &str, file: &str) -> Result { let quoted = format!("`{file}`"); let line = source .lines() diff --git a/xtask/src/testdata/wuffs_webp.rs b/xtask/src/testdata/wuffs_webp.rs new file mode 100644 index 000000000..5c34004bf --- /dev/null +++ b/xtask/src/testdata/wuffs_webp.rs @@ -0,0 +1,92 @@ +//! Vendored WebP files for `clazel-webp` (phase-6.md item 6.8): Wuffs' +//! `test/data/*.webp` (the lossless ones, the animated one, and two small +//! lossy ones for the refusal test). +//! +//! The files are binary, so there is nothing to translate: like the GIF +//! converter (`wuffs_gif.rs`), this checks that each file is the one its +//! `SOURCE.md` names (a `` - `X.webp`: sha256 `…` `` line) and that it still +//! looks like a WebP (`RIFF`, four size bytes, `WEBP`), and writes a table +//! that embeds it with `include_bytes!`. No chunk is parsed here: that is the +//! crate's job, and the differential test's. + +use std::fmt::Write as _; +use std::path::Path; + +use sha2::{Digest, Sha256}; + +use super::fonts::listed_hash; +use super::hash::to_hex; + +/// `tests/data/wuffs-webp/`: Wuffs' `test/data/*.webp` that item 6.8 uses. +pub const FILES: &[&str] = &[ + "animated-red-blue.webp", + "bricks-color.lossless.webp", + "bricks-dither.lossless.webp", + "bricks-gray.lossless.webp", + "bricks-nodither.lossless.webp", + "harvesters.lossless.webp", + "hat.lossless.webp", + "hibiscus.primitive.lossless.webp", + "hibiscus.regular.lossless.webp", + "hippopotamus.lossless.webp", + "mona-lisa.21x32.lossless.webp", + "mona-lisa.21x32.q0.lossy.webp", + "pjw-thumbnail.lossless.webp", + "pjw-thumbnail.lossy.webp", +]; + +/// Check the suite in `dir` and write its table. +pub fn convert(dir: &Path) -> Result { + let name = dir + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| format!("{}: no directory name", dir.display()))?; + if name != "wuffs-webp" { + return Err(format!("no WebP list for suite {name:?}")); + } + let source = + std::fs::read_to_string(dir.join("SOURCE.md")).map_err(|e| format!("SOURCE.md: {e}"))?; + let mut out = String::new(); + let _ = writeln!( + out, + "// Generated by `cargo xtask testdata` from `tests/data/{name}/` \ + (xtask/src/testdata/wuffs_webp.rs). Do not edit.\n\ + \n\ + /// One vendored WebP file, checked against `SOURCE.md` when generated.\n\ + pub struct Case {{\n \ + /// The file's name in `tests/data/{name}/`.\n \ + pub file: &'static str,\n \ + /// Its SHA-256, as `SOURCE.md` lists it.\n \ + pub sha256: &'static str,\n \ + /// Its bytes.\n \ + pub bytes: &'static [u8],\n\ + }}\n\ + \n\ + /// Every file of the suite.\n\ + pub const FILES: &[Case] = &[" + ); + for file in FILES { + let bytes = std::fs::read(dir.join(file)).map_err(|e| format!("{file}: {e}"))?; + let hash = to_hex(&Sha256::digest(&bytes)); + let listed = listed_hash(&source, file)?; + if hash != listed { + return Err(format!( + "{file}: SHA-256 {hash} differs from SOURCE.md's {listed}" + )); + } + let riff = bytes.get(..4) == Some(b"RIFF") && bytes.get(8..12) == Some(b"WEBP"); + if !riff { + return Err(format!("{file}: does not start with `RIFF`, size, `WEBP`")); + } + let _ = writeln!( + out, + " Case {{\n \ + file: {file:?},\n \ + sha256: {hash:?},\n \ + bytes: include_bytes!(\"../data/{name}/{file}\"),\n \ + }}," + ); + } + out.push_str("];\n"); + Ok(out) +}