From 54355a07da0c631476bbd24105a9c4ded9147516 Mon Sep 17 00:00:00 2001 From: Alex Tiernan-Berry Date: Tue, 17 Nov 2026 11:52:01 +0000 Subject: [PATCH 1/5] Generate the lexer's ID_Start and ID_Continue tables from Unicode 18.0 (7.2) `cargo xtask tables` now writes crates/clazel-js-lexer/src/unicode_data.rs from DerivedCoreProperties.txt (Unicode 18.0.0), the copy vendored for clazel-js-regexp, after checking its SHA-256 against that SOURCE.md; `check` fails if the table is stale. `unicode.rs` looks code points up by bisection. Nothing ties the tables to the spec's uninterpreted unicode_id_start_non_ascii / unicode_id_continue_non_ascii: Verus proves only that a lookup is total and false outside U+0080..U+10FFFF. Tests check the tables are sorted and disjoint, ID_Start is within ID_Continue, and known code points. Co-Authored-By: Claude Opus 5.5 --- crates/clazel-js-lexer/src/lib.rs | 2 + crates/clazel-js-lexer/src/unicode.rs | 91 + crates/clazel-js-lexer/src/unicode_data.rs | 1532 +++++++++++++++++ .../clazel-js-lexer/tests/unicode_tables.rs | 73 + xtask/src/js_idents.rs | 191 ++ xtask/src/main.rs | 8 +- 6 files changed, 1896 insertions(+), 1 deletion(-) create mode 100644 crates/clazel-js-lexer/src/unicode.rs create mode 100644 crates/clazel-js-lexer/src/unicode_data.rs create mode 100644 crates/clazel-js-lexer/tests/unicode_tables.rs create mode 100644 xtask/src/js_idents.rs diff --git a/crates/clazel-js-lexer/src/lib.rs b/crates/clazel-js-lexer/src/lib.rs index e4cb7d5e0..3b43769eb 100644 --- a/crates/clazel-js-lexer/src/lib.rs +++ b/crates/clazel-js-lexer/src/lib.rs @@ -18,3 +18,5 @@ pub mod contract; pub mod examples; pub mod lemmas; pub mod spec; +pub mod unicode; +pub mod unicode_data; diff --git a/crates/clazel-js-lexer/src/unicode.rs b/crates/clazel-js-lexer/src/unicode.rs new file mode 100644 index 000000000..6904b0c14 --- /dev/null +++ b/crates/clazel-js-lexer/src/unicode.rs @@ -0,0 +1,91 @@ +//! `ID_Start` and `ID_Continue` above U+007F, for the executable lexer. +//! +//! The tables are [`super::unicode_data`], generated by `cargo xtask +//! tables` (`xtask/src/js_idents.rs`) from Unicode 18.0.0's +//! `DerivedCoreProperties.txt`. A lookup is a bisection over a table's +//! rows. +//! +//! **What is claimed:** nothing about agreement with the spec. The spec +//! leaves `ID_Start` and `ID_Continue` above U+007F uninterpreted +//! ([`super::spec::unicode_id_start_non_ascii`], +//! [`super::spec::unicode_id_continue_non_ascii`]: the standard says "the +//! latest version of the Unicode Standard"), and no axiom ties them to +//! these tables. Verus proves only that a lookup is total and answers +//! false outside U+0080..U+10FFFF. That the tables are the properties of +//! Unicode 18.0.0 rests on the generator, which `cargo xtask check` reruns, +//! and on the tests in `tests/unicode_tables.rs`. + +use super::unicode_data::*; +#[allow(unused_imports)] +use vstd::prelude::*; + +verus! { + +/// A row of a table: the code points `first..=last`. +#[derive(Clone, Copy)] +pub struct Range { + /// The first code point of the row. + pub first: u32, + /// The last code point of the row. + pub last: u32, +} + +/// Whether a row of `table` (sorted, disjoint) holds `c`, by bisection. +fn find(table: &[Range], c: u32) -> (r: bool) { + let mut lo: usize = 0; + let mut hi: usize = table.len(); + while lo < hi + invariant + lo <= hi <= table.len(), + decreases hi - lo, + { + // Never saturates: lo < hi. + let mid = lo.saturating_add(hi.saturating_sub(lo) / 2); + let row = match table.get(mid) { + Some(row) => *row, + None => { + return false; + }, + }; + if c < row.first { + hi = mid; + } else if c > row.last { + lo = mid.saturating_add(1); + } else { + return true; + } + } + false +} + +/// Whether `c` (above U+007F) has the Unicode property `ID_Start`. +pub fn id_start_non_ascii(c: u32) -> (r: bool) + ensures + r ==> 0x80 <= c <= 0x10FFFF, +{ + if c < 0x80 { + return false; + } + if c > 0x10FFFF { + return false; + } + let table: &[Range] = &ID_START; + find(table, c) +} + +/// Whether `c` (above U+007F) has the Unicode property `ID_Continue`. +pub fn id_continue_non_ascii(c: u32) -> (r: bool) + ensures + r ==> 0x80 <= c <= 0x10FFFF, +{ + if c < 0x80 { + return false; + } + if c > 0x10FFFF { + return false; + } + let table: &[Range] = &ID_CONTINUE; + find(table, c) +} + +} // verus! diff --git a/crates/clazel-js-lexer/src/unicode_data.rs b/crates/clazel-js-lexer/src/unicode_data.rs new file mode 100644 index 000000000..c20e55fe1 --- /dev/null +++ b/crates/clazel-js-lexer/src/unicode_data.rs @@ -0,0 +1,1532 @@ +//! The Unicode `ID_Start` and `ID_Continue` tables of the lexer (§12.7). +//! +// @generated by `cargo xtask tables` (xtask/src/js_idents.rs) from +// crates/clazel-js-regexp/tests/data/ucd-regexp/DerivedCoreProperties.txt +// (Unicode 18.0.0; URL and SHA-256 in SOURCE.md there). Do not edit: +// `cargo xtask check` fails if this differs from what the generator +// produces. +// +// What the generator derives: for each of ID_Start and ID_Continue, the +// code points above U+007F that DerivedCoreProperties.txt lists with it, as +// maximal ranges (first, last), sorted and disjoint. + +use super::unicode::Range; +#[allow(unused_imports)] +use vstd::prelude::*; + +verus! { + +/// `ID_Start` above U+007F (DerivedCoreProperties.txt), as maximal ranges. +#[verifier::opaque] +pub const ID_START: [Range; 693] = [ + Range { first: 0xAA, last: 0xAA }, + Range { first: 0xB5, last: 0xB5 }, + Range { first: 0xBA, last: 0xBA }, + Range { first: 0xC0, last: 0xD6 }, + Range { first: 0xD8, last: 0xF6 }, + Range { first: 0xF8, last: 0x2C1 }, + Range { first: 0x2C6, last: 0x2D1 }, + Range { first: 0x2E0, last: 0x2E4 }, + Range { first: 0x2EC, last: 0x2EC }, + Range { first: 0x2EE, last: 0x2EE }, + Range { first: 0x370, last: 0x374 }, + Range { first: 0x376, last: 0x377 }, + Range { first: 0x37A, last: 0x37D }, + Range { first: 0x37F, last: 0x37F }, + Range { first: 0x386, last: 0x386 }, + Range { first: 0x388, last: 0x38A }, + Range { first: 0x38C, last: 0x38C }, + Range { first: 0x38E, last: 0x3A1 }, + Range { first: 0x3A3, last: 0x3F5 }, + Range { first: 0x3F7, last: 0x481 }, + Range { first: 0x48A, last: 0x52F }, + Range { first: 0x531, last: 0x556 }, + Range { first: 0x558, last: 0x559 }, + Range { first: 0x560, last: 0x588 }, + Range { first: 0x58B, last: 0x58C }, + Range { first: 0x5D0, last: 0x5EA }, + Range { first: 0x5EF, last: 0x5F2 }, + Range { first: 0x620, last: 0x64A }, + Range { first: 0x66E, last: 0x66F }, + Range { first: 0x671, last: 0x6D3 }, + Range { first: 0x6D5, last: 0x6D5 }, + Range { first: 0x6E5, last: 0x6E6 }, + Range { first: 0x6EE, last: 0x6EF }, + Range { first: 0x6FA, last: 0x6FC }, + Range { first: 0x6FF, last: 0x6FF }, + Range { first: 0x710, last: 0x710 }, + Range { first: 0x712, last: 0x72F }, + Range { first: 0x74D, last: 0x7A5 }, + Range { first: 0x7B1, last: 0x7B1 }, + Range { first: 0x7CA, last: 0x7EA }, + Range { first: 0x7F4, last: 0x7F5 }, + Range { first: 0x7FA, last: 0x7FA }, + Range { first: 0x800, last: 0x815 }, + Range { first: 0x81A, last: 0x81A }, + Range { first: 0x824, last: 0x824 }, + Range { first: 0x828, last: 0x828 }, + Range { first: 0x840, last: 0x858 }, + Range { first: 0x860, last: 0x86A }, + Range { first: 0x870, last: 0x887 }, + Range { first: 0x889, last: 0x88F }, + Range { first: 0x8A0, last: 0x8C9 }, + Range { first: 0x904, last: 0x939 }, + Range { first: 0x93D, last: 0x93D }, + Range { first: 0x950, last: 0x950 }, + Range { first: 0x958, last: 0x961 }, + Range { first: 0x971, last: 0x980 }, + Range { first: 0x985, last: 0x98C }, + Range { first: 0x98F, last: 0x990 }, + Range { first: 0x993, last: 0x9A8 }, + Range { first: 0x9AA, last: 0x9B0 }, + Range { first: 0x9B2, last: 0x9B2 }, + Range { first: 0x9B6, last: 0x9B9 }, + Range { first: 0x9BD, last: 0x9BD }, + Range { first: 0x9CE, last: 0x9CE }, + Range { first: 0x9DC, last: 0x9DD }, + Range { first: 0x9DF, last: 0x9E1 }, + Range { first: 0x9F0, last: 0x9F1 }, + Range { first: 0x9FC, last: 0x9FC }, + Range { first: 0xA05, last: 0xA0A }, + Range { first: 0xA0F, last: 0xA10 }, + Range { first: 0xA13, last: 0xA28 }, + Range { first: 0xA2A, last: 0xA30 }, + Range { first: 0xA32, last: 0xA33 }, + Range { first: 0xA35, last: 0xA36 }, + Range { first: 0xA38, last: 0xA39 }, + Range { first: 0xA59, last: 0xA5C }, + Range { first: 0xA5E, last: 0xA5E }, + Range { first: 0xA72, last: 0xA74 }, + Range { first: 0xA85, last: 0xA8D }, + Range { first: 0xA8F, last: 0xA91 }, + Range { first: 0xA93, last: 0xAA8 }, + Range { first: 0xAAA, last: 0xAB0 }, + Range { first: 0xAB2, last: 0xAB3 }, + Range { first: 0xAB5, last: 0xAB9 }, + Range { first: 0xABD, last: 0xABD }, + Range { first: 0xAD0, last: 0xAD0 }, + Range { first: 0xAE0, last: 0xAE1 }, + Range { first: 0xAF9, last: 0xAF9 }, + Range { first: 0xB05, last: 0xB0C }, + Range { first: 0xB0F, last: 0xB10 }, + Range { first: 0xB13, last: 0xB28 }, + Range { first: 0xB2A, last: 0xB30 }, + Range { first: 0xB32, last: 0xB33 }, + Range { first: 0xB35, last: 0xB39 }, + Range { first: 0xB3D, last: 0xB3D }, + Range { first: 0xB5C, last: 0xB5D }, + Range { first: 0xB5F, last: 0xB61 }, + Range { first: 0xB71, last: 0xB71 }, + Range { first: 0xB83, last: 0xB83 }, + Range { first: 0xB85, last: 0xB8A }, + Range { first: 0xB8E, last: 0xB90 }, + Range { first: 0xB92, last: 0xB95 }, + Range { first: 0xB99, last: 0xB9A }, + Range { first: 0xB9C, last: 0xB9C }, + Range { first: 0xB9E, last: 0xB9F }, + Range { first: 0xBA3, last: 0xBA4 }, + Range { first: 0xBA8, last: 0xBAA }, + Range { first: 0xBAE, last: 0xBB9 }, + Range { first: 0xBD0, last: 0xBD0 }, + Range { first: 0xC05, last: 0xC0C }, + Range { first: 0xC0E, last: 0xC10 }, + Range { first: 0xC12, last: 0xC28 }, + Range { first: 0xC2A, last: 0xC39 }, + Range { first: 0xC3D, last: 0xC3D }, + Range { first: 0xC58, last: 0xC5A }, + Range { first: 0xC5C, last: 0xC5D }, + Range { first: 0xC60, last: 0xC61 }, + Range { first: 0xC80, last: 0xC80 }, + Range { first: 0xC85, last: 0xC8C }, + Range { first: 0xC8E, last: 0xC90 }, + Range { first: 0xC92, last: 0xCA8 }, + Range { first: 0xCAA, last: 0xCB3 }, + Range { first: 0xCB5, last: 0xCB9 }, + Range { first: 0xCBD, last: 0xCBD }, + Range { first: 0xCDC, last: 0xCDE }, + Range { first: 0xCE0, last: 0xCE1 }, + Range { first: 0xCF1, last: 0xCF2 }, + Range { first: 0xD04, last: 0xD0C }, + Range { first: 0xD0E, last: 0xD10 }, + Range { first: 0xD12, last: 0xD3A }, + Range { first: 0xD3D, last: 0xD3D }, + Range { first: 0xD4E, last: 0xD4E }, + Range { first: 0xD54, last: 0xD56 }, + Range { first: 0xD5F, last: 0xD61 }, + Range { first: 0xD7A, last: 0xD7F }, + Range { first: 0xD85, last: 0xD96 }, + Range { first: 0xD9A, last: 0xDB1 }, + Range { first: 0xDB3, last: 0xDBB }, + Range { first: 0xDBD, last: 0xDBD }, + Range { first: 0xDC0, last: 0xDC6 }, + Range { first: 0xE01, last: 0xE30 }, + Range { first: 0xE32, last: 0xE33 }, + Range { first: 0xE40, last: 0xE46 }, + Range { first: 0xE81, last: 0xE82 }, + Range { first: 0xE84, last: 0xE84 }, + Range { first: 0xE86, last: 0xE8A }, + Range { first: 0xE8C, last: 0xEA3 }, + Range { first: 0xEA5, last: 0xEA5 }, + Range { first: 0xEA7, last: 0xEB0 }, + Range { first: 0xEB2, last: 0xEB3 }, + Range { first: 0xEBD, last: 0xEBD }, + Range { first: 0xEC0, last: 0xEC4 }, + Range { first: 0xEC6, last: 0xEC6 }, + Range { first: 0xEDC, last: 0xEDF }, + Range { first: 0xF00, last: 0xF00 }, + Range { first: 0xF40, last: 0xF47 }, + Range { first: 0xF49, last: 0xF6C }, + Range { first: 0xF88, last: 0xF8C }, + Range { first: 0x1000, last: 0x102A }, + Range { first: 0x103F, last: 0x103F }, + Range { first: 0x1050, last: 0x1055 }, + Range { first: 0x105A, last: 0x105D }, + Range { first: 0x1061, last: 0x1061 }, + Range { first: 0x1065, last: 0x1066 }, + Range { first: 0x106E, last: 0x1070 }, + Range { first: 0x1075, last: 0x1081 }, + Range { first: 0x108E, last: 0x108E }, + Range { first: 0x10A0, last: 0x10C5 }, + Range { first: 0x10C7, last: 0x10C7 }, + Range { first: 0x10CD, last: 0x10CD }, + Range { first: 0x10D0, last: 0x10FA }, + Range { first: 0x10FC, last: 0x1248 }, + Range { first: 0x124A, last: 0x124D }, + Range { first: 0x1250, last: 0x1256 }, + Range { first: 0x1258, last: 0x1258 }, + Range { first: 0x125A, last: 0x125D }, + Range { first: 0x1260, last: 0x1288 }, + Range { first: 0x128A, last: 0x128D }, + Range { first: 0x1290, last: 0x12B0 }, + Range { first: 0x12B2, last: 0x12B5 }, + Range { first: 0x12B8, last: 0x12BE }, + Range { first: 0x12C0, last: 0x12C0 }, + Range { first: 0x12C2, last: 0x12C5 }, + Range { first: 0x12C8, last: 0x12D6 }, + Range { first: 0x12D8, last: 0x1310 }, + Range { first: 0x1312, last: 0x1315 }, + Range { first: 0x1318, last: 0x135A }, + Range { first: 0x1380, last: 0x138F }, + Range { first: 0x13A0, last: 0x13F5 }, + Range { first: 0x13F8, last: 0x13FD }, + Range { first: 0x1401, last: 0x166C }, + Range { first: 0x166F, last: 0x167F }, + Range { first: 0x1681, last: 0x169A }, + Range { first: 0x16A0, last: 0x16EA }, + Range { first: 0x16EE, last: 0x16F8 }, + Range { first: 0x1700, last: 0x1711 }, + Range { first: 0x171F, last: 0x1731 }, + Range { first: 0x1740, last: 0x1751 }, + Range { first: 0x1760, last: 0x176C }, + Range { first: 0x176E, last: 0x1770 }, + Range { first: 0x1780, last: 0x17B3 }, + Range { first: 0x17D7, last: 0x17D7 }, + Range { first: 0x17DC, last: 0x17DC }, + Range { first: 0x1820, last: 0x1878 }, + Range { first: 0x1880, last: 0x18A8 }, + Range { first: 0x18AA, last: 0x18AA }, + Range { first: 0x18B0, last: 0x18F5 }, + Range { first: 0x1900, last: 0x191E }, + Range { first: 0x1950, last: 0x196D }, + Range { first: 0x1970, last: 0x1974 }, + Range { first: 0x1980, last: 0x19AB }, + Range { first: 0x19B0, last: 0x19C9 }, + Range { first: 0x1A00, last: 0x1A16 }, + Range { first: 0x1A20, last: 0x1A54 }, + Range { first: 0x1AA7, last: 0x1AA7 }, + Range { first: 0x1B05, last: 0x1B33 }, + Range { first: 0x1B45, last: 0x1B4C }, + Range { first: 0x1B83, last: 0x1BA0 }, + Range { first: 0x1BAE, last: 0x1BAF }, + Range { first: 0x1BBA, last: 0x1BE5 }, + Range { first: 0x1C00, last: 0x1C23 }, + Range { first: 0x1C4D, last: 0x1C4F }, + Range { first: 0x1C5A, last: 0x1C7D }, + Range { first: 0x1C80, last: 0x1C8A }, + Range { first: 0x1C90, last: 0x1CBA }, + Range { first: 0x1CBD, last: 0x1CBF }, + Range { first: 0x1CE9, last: 0x1CEC }, + Range { first: 0x1CEE, last: 0x1CF3 }, + Range { first: 0x1CF5, last: 0x1CF6 }, + Range { first: 0x1CFA, last: 0x1CFA }, + Range { first: 0x1D00, last: 0x1DBF }, + Range { first: 0x1E00, last: 0x1F15 }, + Range { first: 0x1F18, last: 0x1F1D }, + Range { first: 0x1F20, last: 0x1F45 }, + Range { first: 0x1F48, last: 0x1F4D }, + Range { first: 0x1F50, last: 0x1F57 }, + Range { first: 0x1F59, last: 0x1F59 }, + Range { first: 0x1F5B, last: 0x1F5B }, + Range { first: 0x1F5D, last: 0x1F5D }, + Range { first: 0x1F5F, last: 0x1F7D }, + Range { first: 0x1F80, last: 0x1FB4 }, + Range { first: 0x1FB6, last: 0x1FBC }, + Range { first: 0x1FBE, last: 0x1FBE }, + Range { first: 0x1FC2, last: 0x1FC4 }, + Range { first: 0x1FC6, last: 0x1FCC }, + Range { first: 0x1FD0, last: 0x1FD3 }, + Range { first: 0x1FD6, last: 0x1FDB }, + Range { first: 0x1FE0, last: 0x1FEC }, + Range { first: 0x1FF2, last: 0x1FF4 }, + Range { first: 0x1FF6, last: 0x1FFC }, + Range { first: 0x2071, last: 0x2071 }, + Range { first: 0x207F, last: 0x207F }, + Range { first: 0x208F, last: 0x209F }, + Range { first: 0x2102, last: 0x2102 }, + Range { first: 0x2107, last: 0x2107 }, + Range { first: 0x210A, last: 0x2113 }, + Range { first: 0x2115, last: 0x2115 }, + Range { first: 0x2118, last: 0x211D }, + Range { first: 0x2124, last: 0x2124 }, + Range { first: 0x2126, last: 0x2126 }, + Range { first: 0x2128, last: 0x2128 }, + Range { first: 0x212A, last: 0x2139 }, + Range { first: 0x213C, last: 0x213F }, + Range { first: 0x2145, last: 0x2149 }, + Range { first: 0x214E, last: 0x214E }, + Range { first: 0x2160, last: 0x2188 }, + Range { first: 0x2C00, last: 0x2CE4 }, + Range { first: 0x2CEB, last: 0x2CEE }, + Range { first: 0x2CF2, last: 0x2CF3 }, + Range { first: 0x2D00, last: 0x2D25 }, + Range { first: 0x2D27, last: 0x2D27 }, + Range { first: 0x2D2D, last: 0x2D2D }, + Range { first: 0x2D30, last: 0x2D67 }, + Range { first: 0x2D6F, last: 0x2D6F }, + Range { first: 0x2D80, last: 0x2D96 }, + Range { first: 0x2DA0, last: 0x2DA6 }, + Range { first: 0x2DA8, last: 0x2DAE }, + Range { first: 0x2DB0, last: 0x2DB6 }, + Range { first: 0x2DB8, last: 0x2DBE }, + Range { first: 0x2DC0, last: 0x2DC6 }, + Range { first: 0x2DC8, last: 0x2DCE }, + Range { first: 0x2DD0, last: 0x2DD6 }, + Range { first: 0x2DD8, last: 0x2DDE }, + Range { first: 0x3005, last: 0x3007 }, + Range { first: 0x3021, last: 0x3029 }, + Range { first: 0x3031, last: 0x3035 }, + Range { first: 0x3038, last: 0x303C }, + Range { first: 0x3041, last: 0x3096 }, + Range { first: 0x309B, last: 0x309F }, + Range { first: 0x30A1, last: 0x30FA }, + Range { first: 0x30FC, last: 0x30FF }, + Range { first: 0x3105, last: 0x312F }, + Range { first: 0x3131, last: 0x318E }, + Range { first: 0x31A0, last: 0x31BF }, + Range { first: 0x31F0, last: 0x31FF }, + Range { first: 0x3400, last: 0x4DBF }, + Range { first: 0x4E00, last: 0xA48C }, + Range { first: 0xA4D0, last: 0xA4FD }, + Range { first: 0xA500, last: 0xA60C }, + Range { first: 0xA610, last: 0xA61F }, + Range { first: 0xA62A, last: 0xA62B }, + Range { first: 0xA640, last: 0xA66E }, + Range { first: 0xA67F, last: 0xA69D }, + Range { first: 0xA6A0, last: 0xA6EF }, + Range { first: 0xA717, last: 0xA71F }, + Range { first: 0xA722, last: 0xA788 }, + Range { first: 0xA78B, last: 0xA7DD }, + Range { first: 0xA7E2, last: 0xA7E2 }, + Range { first: 0xA7F1, last: 0xA801 }, + Range { first: 0xA803, last: 0xA805 }, + Range { first: 0xA807, last: 0xA80A }, + Range { first: 0xA80C, last: 0xA822 }, + Range { first: 0xA840, last: 0xA873 }, + Range { first: 0xA882, last: 0xA8B3 }, + Range { first: 0xA8F2, last: 0xA8F7 }, + Range { first: 0xA8FB, last: 0xA8FB }, + Range { first: 0xA8FD, last: 0xA8FE }, + Range { first: 0xA90A, last: 0xA925 }, + Range { first: 0xA930, last: 0xA946 }, + Range { first: 0xA960, last: 0xA97C }, + Range { first: 0xA984, last: 0xA9B2 }, + Range { first: 0xA9CF, last: 0xA9CF }, + Range { first: 0xA9E0, last: 0xA9E4 }, + Range { first: 0xA9E6, last: 0xA9EF }, + Range { first: 0xA9FA, last: 0xA9FE }, + Range { first: 0xAA00, last: 0xAA28 }, + Range { first: 0xAA40, last: 0xAA42 }, + Range { first: 0xAA44, last: 0xAA4B }, + Range { first: 0xAA60, last: 0xAA76 }, + Range { first: 0xAA7A, last: 0xAA7A }, + Range { first: 0xAA7E, last: 0xAAAF }, + Range { first: 0xAAB1, last: 0xAAB1 }, + Range { first: 0xAAB5, last: 0xAAB6 }, + Range { first: 0xAAB9, last: 0xAABD }, + Range { first: 0xAAC0, last: 0xAAC0 }, + Range { first: 0xAAC2, last: 0xAAC2 }, + Range { first: 0xAADB, last: 0xAADD }, + Range { first: 0xAAE0, last: 0xAAEA }, + Range { first: 0xAAF2, last: 0xAAF4 }, + Range { first: 0xAB01, last: 0xAB06 }, + Range { first: 0xAB09, last: 0xAB0E }, + Range { first: 0xAB11, last: 0xAB16 }, + Range { first: 0xAB20, last: 0xAB26 }, + Range { first: 0xAB28, last: 0xAB2E }, + Range { first: 0xAB30, last: 0xAB5A }, + Range { first: 0xAB5C, last: 0xAB69 }, + Range { first: 0xAB6C, last: 0xAB6D }, + Range { first: 0xAB70, last: 0xABE2 }, + Range { first: 0xAC00, last: 0xD7A3 }, + Range { first: 0xD7B0, last: 0xD7C6 }, + Range { first: 0xD7CB, last: 0xD7FB }, + Range { first: 0xF900, last: 0xFA6D }, + Range { first: 0xFA70, last: 0xFAD9 }, + Range { first: 0xFB00, last: 0xFB06 }, + Range { first: 0xFB13, last: 0xFB17 }, + Range { first: 0xFB1D, last: 0xFB1D }, + Range { first: 0xFB1F, last: 0xFB28 }, + Range { first: 0xFB2A, last: 0xFB36 }, + Range { first: 0xFB38, last: 0xFB3C }, + Range { first: 0xFB3E, last: 0xFB3E }, + Range { first: 0xFB40, last: 0xFB41 }, + Range { first: 0xFB43, last: 0xFB44 }, + Range { first: 0xFB46, last: 0xFBB1 }, + Range { first: 0xFBD3, last: 0xFD3D }, + Range { first: 0xFD50, last: 0xFD8F }, + Range { first: 0xFD92, last: 0xFDC7 }, + Range { first: 0xFDF0, last: 0xFDFB }, + Range { first: 0xFE70, last: 0xFE74 }, + Range { first: 0xFE76, last: 0xFEFC }, + Range { first: 0xFF21, last: 0xFF3A }, + Range { first: 0xFF41, last: 0xFF5A }, + Range { first: 0xFF66, last: 0xFFBE }, + Range { first: 0xFFC2, last: 0xFFC7 }, + Range { first: 0xFFCA, last: 0xFFCF }, + Range { first: 0xFFD2, last: 0xFFD7 }, + Range { first: 0xFFDA, last: 0xFFDC }, + Range { first: 0x10000, last: 0x1000B }, + Range { first: 0x1000D, last: 0x10026 }, + Range { first: 0x10028, last: 0x1003A }, + Range { first: 0x1003C, last: 0x1003D }, + Range { first: 0x1003F, last: 0x1004D }, + Range { first: 0x10050, last: 0x1005D }, + Range { first: 0x10080, last: 0x100FA }, + Range { first: 0x10140, last: 0x10174 }, + Range { first: 0x10280, last: 0x1029C }, + Range { first: 0x102A0, last: 0x102D0 }, + Range { first: 0x10300, last: 0x1031F }, + Range { first: 0x1032D, last: 0x1034A }, + Range { first: 0x10350, last: 0x10375 }, + Range { first: 0x10380, last: 0x1039D }, + Range { first: 0x103A0, last: 0x103C3 }, + Range { first: 0x103C8, last: 0x103CF }, + Range { first: 0x103D1, last: 0x103D5 }, + Range { first: 0x10400, last: 0x1049D }, + Range { first: 0x104B0, last: 0x104D3 }, + Range { first: 0x104D8, last: 0x104FB }, + Range { first: 0x10500, last: 0x10527 }, + Range { first: 0x10530, last: 0x10563 }, + Range { first: 0x10570, last: 0x1057A }, + Range { first: 0x1057C, last: 0x1058A }, + Range { first: 0x1058C, last: 0x10592 }, + Range { first: 0x10594, last: 0x10595 }, + Range { first: 0x10597, last: 0x105A1 }, + Range { first: 0x105A3, last: 0x105B1 }, + Range { first: 0x105B3, last: 0x105B9 }, + Range { first: 0x105BB, last: 0x105BC }, + Range { first: 0x105C0, last: 0x105F3 }, + Range { first: 0x10600, last: 0x10736 }, + Range { first: 0x10740, last: 0x10755 }, + Range { first: 0x10760, last: 0x10767 }, + Range { first: 0x10780, last: 0x10785 }, + Range { first: 0x10787, last: 0x107B0 }, + Range { first: 0x107B2, last: 0x107BF }, + Range { first: 0x10800, last: 0x10805 }, + Range { first: 0x10808, last: 0x10808 }, + Range { first: 0x1080A, last: 0x10835 }, + Range { first: 0x10837, last: 0x10838 }, + Range { first: 0x1083C, last: 0x1083C }, + Range { first: 0x1083F, last: 0x10855 }, + Range { first: 0x10860, last: 0x10876 }, + Range { first: 0x10880, last: 0x1089E }, + Range { first: 0x108E0, last: 0x108F2 }, + Range { first: 0x108F4, last: 0x108F5 }, + Range { first: 0x10900, last: 0x10915 }, + Range { first: 0x10920, last: 0x10939 }, + Range { first: 0x10940, last: 0x10959 }, + Range { first: 0x10980, last: 0x109B7 }, + Range { first: 0x109BE, last: 0x109BF }, + Range { first: 0x10A00, last: 0x10A00 }, + Range { first: 0x10A10, last: 0x10A13 }, + Range { first: 0x10A15, last: 0x10A17 }, + Range { first: 0x10A19, last: 0x10A35 }, + Range { first: 0x10A60, last: 0x10A7C }, + Range { first: 0x10A80, last: 0x10A9C }, + Range { first: 0x10AC0, last: 0x10AC7 }, + Range { first: 0x10AC9, last: 0x10AE4 }, + Range { first: 0x10B00, last: 0x10B35 }, + Range { first: 0x10B40, last: 0x10B55 }, + Range { first: 0x10B60, last: 0x10B72 }, + Range { first: 0x10B80, last: 0x10B91 }, + Range { first: 0x10C00, last: 0x10C48 }, + Range { first: 0x10C80, last: 0x10CB2 }, + Range { first: 0x10CC0, last: 0x10CF2 }, + Range { first: 0x10D00, last: 0x10D23 }, + Range { first: 0x10D4A, last: 0x10D65 }, + Range { first: 0x10D6F, last: 0x10D85 }, + Range { first: 0x10E80, last: 0x10EA9 }, + Range { first: 0x10EB0, last: 0x10EB1 }, + Range { first: 0x10EC2, last: 0x10EC7 }, + Range { first: 0x10ED9, last: 0x10EEE }, + Range { first: 0x10F00, last: 0x10F1C }, + Range { first: 0x10F27, last: 0x10F27 }, + Range { first: 0x10F30, last: 0x10F45 }, + Range { first: 0x10F70, last: 0x10F81 }, + Range { first: 0x10FB0, last: 0x10FC4 }, + Range { first: 0x10FE0, last: 0x10FF6 }, + Range { first: 0x11003, last: 0x11037 }, + Range { first: 0x11071, last: 0x11072 }, + Range { first: 0x11075, last: 0x11075 }, + Range { first: 0x11083, last: 0x110AF }, + Range { first: 0x110D0, last: 0x110E8 }, + Range { first: 0x11103, last: 0x11126 }, + Range { first: 0x11144, last: 0x11144 }, + Range { first: 0x11147, last: 0x11147 }, + Range { first: 0x11150, last: 0x11172 }, + Range { first: 0x11176, last: 0x11176 }, + Range { first: 0x11183, last: 0x111B2 }, + Range { first: 0x111C1, last: 0x111C4 }, + Range { first: 0x111DA, last: 0x111DA }, + Range { first: 0x111DC, last: 0x111DC }, + Range { first: 0x11200, last: 0x11211 }, + Range { first: 0x11213, last: 0x1122B }, + Range { first: 0x1123F, last: 0x11240 }, + Range { first: 0x11280, last: 0x11286 }, + Range { first: 0x11288, last: 0x11288 }, + Range { first: 0x1128A, last: 0x1128D }, + Range { first: 0x1128F, last: 0x1129D }, + Range { first: 0x1129F, last: 0x112A8 }, + Range { first: 0x112B0, last: 0x112DE }, + Range { first: 0x11305, last: 0x1130C }, + Range { first: 0x1130F, last: 0x11310 }, + Range { first: 0x11313, last: 0x11328 }, + Range { first: 0x1132A, last: 0x11330 }, + Range { first: 0x11332, last: 0x11333 }, + Range { first: 0x11335, last: 0x11339 }, + Range { first: 0x1133D, last: 0x1133D }, + Range { first: 0x11350, last: 0x11350 }, + Range { first: 0x1135D, last: 0x11361 }, + Range { first: 0x11380, last: 0x11389 }, + Range { first: 0x1138B, last: 0x1138B }, + Range { first: 0x1138E, last: 0x1138E }, + Range { first: 0x11390, last: 0x113B5 }, + Range { first: 0x113B7, last: 0x113B7 }, + Range { first: 0x113D1, last: 0x113D1 }, + Range { first: 0x113D3, last: 0x113D3 }, + Range { first: 0x11400, last: 0x11434 }, + Range { first: 0x11447, last: 0x1144A }, + Range { first: 0x1145F, last: 0x11461 }, + Range { first: 0x11480, last: 0x114AF }, + Range { first: 0x114C4, last: 0x114C5 }, + Range { first: 0x114C7, last: 0x114C7 }, + Range { first: 0x11580, last: 0x115AE }, + Range { first: 0x115D8, last: 0x115DB }, + Range { first: 0x11600, last: 0x1162F }, + Range { first: 0x11644, last: 0x11644 }, + Range { first: 0x11680, last: 0x116AA }, + Range { first: 0x116B8, last: 0x116B8 }, + Range { first: 0x11700, last: 0x1171A }, + Range { first: 0x11740, last: 0x11746 }, + Range { first: 0x11800, last: 0x1182B }, + Range { first: 0x118A0, last: 0x118DF }, + Range { first: 0x118FF, last: 0x11906 }, + Range { first: 0x11909, last: 0x11909 }, + Range { first: 0x1190C, last: 0x11913 }, + Range { first: 0x11915, last: 0x11916 }, + Range { first: 0x11918, last: 0x1192F }, + Range { first: 0x1193F, last: 0x1193F }, + Range { first: 0x11941, last: 0x11941 }, + Range { first: 0x119A0, last: 0x119A7 }, + Range { first: 0x119AA, last: 0x119D0 }, + Range { first: 0x119E1, last: 0x119E1 }, + Range { first: 0x119E3, last: 0x119E3 }, + Range { first: 0x11A00, last: 0x11A00 }, + Range { first: 0x11A0B, last: 0x11A32 }, + Range { first: 0x11A3A, last: 0x11A3A }, + Range { first: 0x11A50, last: 0x11A50 }, + Range { first: 0x11A5C, last: 0x11A89 }, + Range { first: 0x11A9D, last: 0x11A9D }, + Range { first: 0x11AB0, last: 0x11AF8 }, + Range { first: 0x11B0A, last: 0x11B0A }, + Range { first: 0x11BC0, last: 0x11BE0 }, + Range { first: 0x11C00, last: 0x11C08 }, + Range { first: 0x11C0A, last: 0x11C2E }, + Range { first: 0x11C40, last: 0x11C40 }, + Range { first: 0x11C72, last: 0x11C8F }, + Range { first: 0x11D00, last: 0x11D06 }, + Range { first: 0x11D08, last: 0x11D09 }, + Range { first: 0x11D0B, last: 0x11D30 }, + Range { first: 0x11D46, last: 0x11D46 }, + Range { first: 0x11D60, last: 0x11D65 }, + Range { first: 0x11D67, last: 0x11D68 }, + Range { first: 0x11D6A, last: 0x11D89 }, + Range { first: 0x11D98, last: 0x11D98 }, + Range { first: 0x11DB0, last: 0x11DDB }, + Range { first: 0x11DF1, last: 0x11DF1 }, + Range { first: 0x11EE0, last: 0x11EF2 }, + Range { first: 0x11F02, last: 0x11F02 }, + Range { first: 0x11F04, last: 0x11F10 }, + Range { first: 0x11F12, last: 0x11F33 }, + Range { first: 0x11FB0, last: 0x11FB0 }, + Range { first: 0x12000, last: 0x12399 }, + Range { first: 0x12400, last: 0x1246F }, + Range { first: 0x12475, last: 0x12543 }, + Range { first: 0x12550, last: 0x12686 }, + Range { first: 0x12F90, last: 0x12FF0 }, + Range { first: 0x13000, last: 0x1342F }, + Range { first: 0x13441, last: 0x13446 }, + Range { first: 0x13460, last: 0x143FA }, + Range { first: 0x14400, last: 0x14646 }, + Range { first: 0x16100, last: 0x1611D }, + Range { first: 0x16800, last: 0x16A38 }, + Range { first: 0x16A40, last: 0x16A5E }, + Range { first: 0x16A70, last: 0x16ABE }, + Range { first: 0x16AD0, last: 0x16AED }, + Range { first: 0x16B00, last: 0x16B2F }, + Range { first: 0x16B40, last: 0x16B43 }, + Range { first: 0x16B63, last: 0x16B77 }, + Range { first: 0x16B7D, last: 0x16B8F }, + Range { first: 0x16D40, last: 0x16D6C }, + Range { first: 0x16E40, last: 0x16E7F }, + Range { first: 0x16EA0, last: 0x16EB8 }, + Range { first: 0x16EBB, last: 0x16ED3 }, + Range { first: 0x16F00, last: 0x16F4A }, + Range { first: 0x16F50, last: 0x16F50 }, + Range { first: 0x16F93, last: 0x16F9F }, + Range { first: 0x16FE0, last: 0x16FE1 }, + Range { first: 0x16FE3, last: 0x16FE3 }, + Range { first: 0x16FF2, last: 0x16FF6 }, + Range { first: 0x17000, last: 0x18CDA }, + Range { first: 0x18CFF, last: 0x18D20 }, + Range { first: 0x18D80, last: 0x18DF2 }, + Range { first: 0x18E00, last: 0x19191 }, + Range { first: 0x191A0, last: 0x191D2 }, + Range { first: 0x1AFF0, last: 0x1AFF3 }, + Range { first: 0x1AFF5, last: 0x1AFFB }, + Range { first: 0x1AFFD, last: 0x1AFFE }, + Range { first: 0x1B000, last: 0x1B128 }, + Range { first: 0x1B132, last: 0x1B132 }, + Range { first: 0x1B150, last: 0x1B152 }, + Range { first: 0x1B155, last: 0x1B155 }, + Range { first: 0x1B164, last: 0x1B168 }, + Range { first: 0x1B170, last: 0x1B2FB }, + Range { first: 0x1BC00, last: 0x1BC6A }, + Range { first: 0x1BC70, last: 0x1BC7C }, + Range { first: 0x1BC80, last: 0x1BC88 }, + Range { first: 0x1BC90, last: 0x1BC99 }, + Range { first: 0x1D400, last: 0x1D454 }, + Range { first: 0x1D456, last: 0x1D49C }, + Range { first: 0x1D49E, last: 0x1D49F }, + Range { first: 0x1D4A2, last: 0x1D4A2 }, + Range { first: 0x1D4A5, last: 0x1D4A6 }, + Range { first: 0x1D4A9, last: 0x1D4AC }, + Range { first: 0x1D4AE, last: 0x1D4B9 }, + Range { first: 0x1D4BB, last: 0x1D4BB }, + Range { first: 0x1D4BD, last: 0x1D4C3 }, + Range { first: 0x1D4C5, last: 0x1D505 }, + Range { first: 0x1D507, last: 0x1D50A }, + Range { first: 0x1D50D, last: 0x1D514 }, + Range { first: 0x1D516, last: 0x1D51C }, + Range { first: 0x1D51E, last: 0x1D539 }, + Range { first: 0x1D53B, last: 0x1D53E }, + Range { first: 0x1D540, last: 0x1D544 }, + Range { first: 0x1D546, last: 0x1D546 }, + Range { first: 0x1D54A, last: 0x1D550 }, + Range { first: 0x1D552, last: 0x1D6A6 }, + Range { first: 0x1D6A8, last: 0x1D6C0 }, + Range { first: 0x1D6C2, last: 0x1D6DA }, + Range { first: 0x1D6DC, last: 0x1D6FA }, + Range { first: 0x1D6FC, last: 0x1D714 }, + Range { first: 0x1D716, last: 0x1D734 }, + Range { first: 0x1D736, last: 0x1D74E }, + Range { first: 0x1D750, last: 0x1D76E }, + Range { first: 0x1D770, last: 0x1D788 }, + Range { first: 0x1D78A, last: 0x1D7A8 }, + Range { first: 0x1D7AA, last: 0x1D7C2 }, + Range { first: 0x1D7C4, last: 0x1D7CB }, + Range { first: 0x1DF00, last: 0x1DF81 }, + Range { first: 0x1DF90, last: 0x1DF96 }, + Range { first: 0x1DFCD, last: 0x1DFFF }, + Range { first: 0x1E030, last: 0x1E06D }, + Range { first: 0x1E100, last: 0x1E12C }, + Range { first: 0x1E137, last: 0x1E13D }, + Range { first: 0x1E14E, last: 0x1E14E }, + Range { first: 0x1E290, last: 0x1E2AD }, + Range { first: 0x1E2C0, last: 0x1E2EB }, + Range { first: 0x1E4D0, last: 0x1E4EB }, + Range { first: 0x1E5D0, last: 0x1E5ED }, + Range { first: 0x1E5F0, last: 0x1E5F0 }, + Range { first: 0x1E6C0, last: 0x1E6DE }, + Range { first: 0x1E6E0, last: 0x1E6E2 }, + Range { first: 0x1E6E4, last: 0x1E6E5 }, + Range { first: 0x1E6E7, last: 0x1E6ED }, + Range { first: 0x1E6F0, last: 0x1E6F4 }, + Range { first: 0x1E6FE, last: 0x1E6FF }, + Range { first: 0x1E7E0, last: 0x1E7E6 }, + Range { first: 0x1E7E8, last: 0x1E7EB }, + Range { first: 0x1E7ED, last: 0x1E7EE }, + Range { first: 0x1E7F0, last: 0x1E7FE }, + Range { first: 0x1E800, last: 0x1E8C4 }, + Range { first: 0x1E900, last: 0x1E943 }, + Range { first: 0x1E94B, last: 0x1E94B }, + Range { first: 0x1EE00, last: 0x1EE03 }, + Range { first: 0x1EE05, last: 0x1EE1F }, + Range { first: 0x1EE21, last: 0x1EE22 }, + Range { first: 0x1EE24, last: 0x1EE24 }, + Range { first: 0x1EE27, last: 0x1EE27 }, + Range { first: 0x1EE29, last: 0x1EE32 }, + Range { first: 0x1EE34, last: 0x1EE37 }, + Range { first: 0x1EE39, last: 0x1EE39 }, + Range { first: 0x1EE3B, last: 0x1EE3B }, + Range { first: 0x1EE42, last: 0x1EE42 }, + Range { first: 0x1EE47, last: 0x1EE47 }, + Range { first: 0x1EE49, last: 0x1EE49 }, + Range { first: 0x1EE4B, last: 0x1EE4B }, + Range { first: 0x1EE4D, last: 0x1EE4F }, + Range { first: 0x1EE51, last: 0x1EE52 }, + Range { first: 0x1EE54, last: 0x1EE54 }, + Range { first: 0x1EE57, last: 0x1EE57 }, + Range { first: 0x1EE59, last: 0x1EE59 }, + Range { first: 0x1EE5B, last: 0x1EE5B }, + Range { first: 0x1EE5D, last: 0x1EE5D }, + Range { first: 0x1EE5F, last: 0x1EE5F }, + Range { first: 0x1EE61, last: 0x1EE62 }, + Range { first: 0x1EE64, last: 0x1EE64 }, + Range { first: 0x1EE67, last: 0x1EE6A }, + Range { first: 0x1EE6C, last: 0x1EE72 }, + Range { first: 0x1EE74, last: 0x1EE77 }, + Range { first: 0x1EE79, last: 0x1EE7C }, + Range { first: 0x1EE7E, last: 0x1EE7E }, + Range { first: 0x1EE80, last: 0x1EE89 }, + Range { first: 0x1EE8B, last: 0x1EE9B }, + Range { first: 0x1EEA1, last: 0x1EEA3 }, + Range { first: 0x1EEA5, last: 0x1EEA9 }, + Range { first: 0x1EEAB, last: 0x1EEBB }, + Range { first: 0x20000, last: 0x2A6DF }, + Range { first: 0x2A700, last: 0x2B81E }, + Range { first: 0x2B820, last: 0x2CEAD }, + Range { first: 0x2CEB0, last: 0x2EBE0 }, + Range { first: 0x2EBF0, last: 0x2EE5D }, + Range { first: 0x2F800, last: 0x2FA1D }, + Range { first: 0x30000, last: 0x3134A }, + Range { first: 0x31350, last: 0x33479 }, + Range { first: 0x3D000, last: 0x3FC3F }, +]; + +/// `ID_Continue` above U+007F (DerivedCoreProperties.txt), as maximal ranges. +#[verifier::opaque] +pub const ID_CONTINUE: [Range; 810] = [ + Range { first: 0xAA, last: 0xAA }, + Range { first: 0xB5, last: 0xB5 }, + Range { first: 0xB7, last: 0xB7 }, + Range { first: 0xBA, last: 0xBA }, + Range { first: 0xC0, last: 0xD6 }, + Range { first: 0xD8, last: 0xF6 }, + Range { first: 0xF8, last: 0x2C1 }, + Range { first: 0x2C6, last: 0x2D1 }, + Range { first: 0x2E0, last: 0x2E4 }, + Range { first: 0x2EC, last: 0x2EC }, + Range { first: 0x2EE, last: 0x2EE }, + Range { first: 0x300, last: 0x374 }, + Range { first: 0x376, last: 0x377 }, + Range { first: 0x37A, last: 0x37D }, + Range { first: 0x37F, last: 0x37F }, + Range { first: 0x386, last: 0x38A }, + Range { first: 0x38C, last: 0x38C }, + Range { first: 0x38E, last: 0x3A1 }, + Range { first: 0x3A3, last: 0x3F5 }, + Range { first: 0x3F7, last: 0x481 }, + Range { first: 0x483, last: 0x487 }, + Range { first: 0x48A, last: 0x52F }, + Range { first: 0x531, last: 0x556 }, + Range { first: 0x558, last: 0x559 }, + Range { first: 0x560, last: 0x588 }, + Range { first: 0x58B, last: 0x58C }, + Range { first: 0x591, last: 0x5BD }, + Range { first: 0x5BF, last: 0x5BF }, + Range { first: 0x5C1, last: 0x5C2 }, + Range { first: 0x5C4, last: 0x5C5 }, + Range { first: 0x5C7, last: 0x5C9 }, + Range { first: 0x5D0, last: 0x5EA }, + Range { first: 0x5EF, last: 0x5F2 }, + Range { first: 0x610, last: 0x61A }, + Range { first: 0x620, last: 0x669 }, + Range { first: 0x66E, last: 0x6D3 }, + Range { first: 0x6D5, last: 0x6DC }, + Range { first: 0x6DF, last: 0x6E8 }, + Range { first: 0x6EA, last: 0x6FC }, + Range { first: 0x6FF, last: 0x6FF }, + Range { first: 0x710, last: 0x74A }, + Range { first: 0x74D, last: 0x7B1 }, + Range { first: 0x7C0, last: 0x7F5 }, + Range { first: 0x7FA, last: 0x7FA }, + Range { first: 0x7FD, last: 0x7FD }, + Range { first: 0x800, last: 0x82D }, + Range { first: 0x840, last: 0x85B }, + Range { first: 0x860, last: 0x86A }, + Range { first: 0x870, last: 0x887 }, + Range { first: 0x889, last: 0x88F }, + Range { first: 0x897, last: 0x8E1 }, + Range { first: 0x8E3, last: 0x963 }, + Range { first: 0x966, last: 0x96F }, + Range { first: 0x971, last: 0x983 }, + Range { first: 0x985, last: 0x98C }, + Range { first: 0x98F, last: 0x990 }, + Range { first: 0x993, last: 0x9A8 }, + Range { first: 0x9AA, last: 0x9B0 }, + Range { first: 0x9B2, last: 0x9B2 }, + Range { first: 0x9B6, last: 0x9B9 }, + Range { first: 0x9BC, last: 0x9C4 }, + Range { first: 0x9C7, last: 0x9C8 }, + Range { first: 0x9CB, last: 0x9CE }, + Range { first: 0x9D7, last: 0x9D7 }, + Range { first: 0x9DC, last: 0x9DD }, + Range { first: 0x9DF, last: 0x9E3 }, + Range { first: 0x9E6, last: 0x9F1 }, + Range { first: 0x9FC, last: 0x9FC }, + Range { first: 0x9FE, last: 0x9FE }, + Range { first: 0xA01, last: 0xA03 }, + Range { first: 0xA05, last: 0xA0A }, + Range { first: 0xA0F, last: 0xA10 }, + Range { first: 0xA13, last: 0xA28 }, + Range { first: 0xA2A, last: 0xA30 }, + Range { first: 0xA32, last: 0xA33 }, + Range { first: 0xA35, last: 0xA36 }, + Range { first: 0xA38, last: 0xA39 }, + Range { first: 0xA3C, last: 0xA3C }, + Range { first: 0xA3E, last: 0xA42 }, + Range { first: 0xA47, last: 0xA48 }, + Range { first: 0xA4B, last: 0xA4D }, + Range { first: 0xA51, last: 0xA51 }, + Range { first: 0xA59, last: 0xA5C }, + Range { first: 0xA5E, last: 0xA5E }, + Range { first: 0xA66, last: 0xA75 }, + Range { first: 0xA81, last: 0xA83 }, + Range { first: 0xA85, last: 0xA8D }, + Range { first: 0xA8F, last: 0xA91 }, + Range { first: 0xA93, last: 0xAA8 }, + Range { first: 0xAAA, last: 0xAB0 }, + Range { first: 0xAB2, last: 0xAB3 }, + Range { first: 0xAB5, last: 0xAB9 }, + Range { first: 0xABC, last: 0xAC5 }, + Range { first: 0xAC7, last: 0xAC9 }, + Range { first: 0xACB, last: 0xACD }, + Range { first: 0xAD0, last: 0xAD0 }, + Range { first: 0xAE0, last: 0xAE3 }, + Range { first: 0xAE6, last: 0xAEF }, + Range { first: 0xAF9, last: 0xAFF }, + Range { first: 0xB01, last: 0xB03 }, + Range { first: 0xB05, last: 0xB0C }, + Range { first: 0xB0F, last: 0xB10 }, + Range { first: 0xB13, last: 0xB28 }, + Range { first: 0xB2A, last: 0xB30 }, + Range { first: 0xB32, last: 0xB33 }, + Range { first: 0xB35, last: 0xB39 }, + Range { first: 0xB3C, last: 0xB44 }, + Range { first: 0xB47, last: 0xB48 }, + Range { first: 0xB4B, last: 0xB4D }, + Range { first: 0xB53, last: 0xB57 }, + Range { first: 0xB5C, last: 0xB5D }, + Range { first: 0xB5F, last: 0xB63 }, + Range { first: 0xB66, last: 0xB6F }, + Range { first: 0xB71, last: 0xB71 }, + Range { first: 0xB82, last: 0xB83 }, + Range { first: 0xB85, last: 0xB8A }, + Range { first: 0xB8E, last: 0xB90 }, + Range { first: 0xB92, last: 0xB95 }, + Range { first: 0xB99, last: 0xB9A }, + Range { first: 0xB9C, last: 0xB9C }, + Range { first: 0xB9E, last: 0xB9F }, + Range { first: 0xBA3, last: 0xBA4 }, + Range { first: 0xBA8, last: 0xBAA }, + Range { first: 0xBAE, last: 0xBB9 }, + Range { first: 0xBBE, last: 0xBC2 }, + Range { first: 0xBC6, last: 0xBC8 }, + Range { first: 0xBCA, last: 0xBCD }, + Range { first: 0xBD0, last: 0xBD0 }, + Range { first: 0xBD7, last: 0xBD7 }, + Range { first: 0xBE6, last: 0xBEF }, + Range { first: 0xC00, last: 0xC0C }, + Range { first: 0xC0E, last: 0xC10 }, + Range { first: 0xC12, last: 0xC28 }, + Range { first: 0xC2A, last: 0xC39 }, + Range { first: 0xC3C, last: 0xC44 }, + Range { first: 0xC46, last: 0xC48 }, + Range { first: 0xC4A, last: 0xC4D }, + Range { first: 0xC55, last: 0xC56 }, + Range { first: 0xC58, last: 0xC5A }, + Range { first: 0xC5C, last: 0xC5D }, + Range { first: 0xC60, last: 0xC63 }, + Range { first: 0xC66, last: 0xC6F }, + Range { first: 0xC80, last: 0xC83 }, + Range { first: 0xC85, last: 0xC8C }, + Range { first: 0xC8E, last: 0xC90 }, + Range { first: 0xC92, last: 0xCA8 }, + Range { first: 0xCAA, last: 0xCB3 }, + Range { first: 0xCB5, last: 0xCB9 }, + Range { first: 0xCBC, last: 0xCC4 }, + Range { first: 0xCC6, last: 0xCC8 }, + Range { first: 0xCCA, last: 0xCCD }, + Range { first: 0xCD5, last: 0xCD6 }, + Range { first: 0xCDC, last: 0xCDE }, + Range { first: 0xCE0, last: 0xCE3 }, + Range { first: 0xCE6, last: 0xCEF }, + Range { first: 0xCF1, last: 0xCF3 }, + Range { first: 0xD00, last: 0xD0C }, + Range { first: 0xD0E, last: 0xD10 }, + Range { first: 0xD12, last: 0xD44 }, + Range { first: 0xD46, last: 0xD48 }, + Range { first: 0xD4A, last: 0xD4E }, + Range { first: 0xD54, last: 0xD57 }, + Range { first: 0xD5F, last: 0xD63 }, + Range { first: 0xD66, last: 0xD6F }, + Range { first: 0xD7A, last: 0xD7F }, + Range { first: 0xD81, last: 0xD83 }, + Range { first: 0xD85, last: 0xD96 }, + Range { first: 0xD9A, last: 0xDB1 }, + Range { first: 0xDB3, last: 0xDBB }, + Range { first: 0xDBD, last: 0xDBD }, + Range { first: 0xDC0, last: 0xDC6 }, + Range { first: 0xDCA, last: 0xDCA }, + Range { first: 0xDCF, last: 0xDD4 }, + Range { first: 0xDD6, last: 0xDD6 }, + Range { first: 0xDD8, last: 0xDDF }, + Range { first: 0xDE6, last: 0xDEF }, + Range { first: 0xDF2, last: 0xDF3 }, + Range { first: 0xE01, last: 0xE3A }, + Range { first: 0xE40, last: 0xE4E }, + Range { first: 0xE50, last: 0xE59 }, + Range { first: 0xE81, last: 0xE82 }, + Range { first: 0xE84, last: 0xE84 }, + Range { first: 0xE86, last: 0xE8A }, + Range { first: 0xE8C, last: 0xEA3 }, + Range { first: 0xEA5, last: 0xEA5 }, + Range { first: 0xEA7, last: 0xEBD }, + Range { first: 0xEC0, last: 0xEC4 }, + Range { first: 0xEC6, last: 0xEC6 }, + Range { first: 0xEC8, last: 0xECE }, + Range { first: 0xED0, last: 0xED9 }, + Range { first: 0xEDC, last: 0xEDF }, + Range { first: 0xF00, last: 0xF00 }, + Range { first: 0xF18, last: 0xF19 }, + Range { first: 0xF20, last: 0xF29 }, + Range { first: 0xF35, last: 0xF35 }, + Range { first: 0xF37, last: 0xF37 }, + Range { first: 0xF39, last: 0xF39 }, + Range { first: 0xF3E, last: 0xF47 }, + Range { first: 0xF49, last: 0xF6C }, + Range { first: 0xF71, last: 0xF84 }, + Range { first: 0xF86, last: 0xF97 }, + Range { first: 0xF99, last: 0xFBC }, + Range { first: 0xFC6, last: 0xFC6 }, + Range { first: 0x1000, last: 0x1049 }, + Range { first: 0x1050, last: 0x109D }, + Range { first: 0x10A0, last: 0x10C5 }, + Range { first: 0x10C7, last: 0x10C7 }, + Range { first: 0x10CD, last: 0x10CD }, + Range { first: 0x10D0, last: 0x10FA }, + Range { first: 0x10FC, last: 0x1248 }, + Range { first: 0x124A, last: 0x124D }, + Range { first: 0x1250, last: 0x1256 }, + Range { first: 0x1258, last: 0x1258 }, + Range { first: 0x125A, last: 0x125D }, + Range { first: 0x1260, last: 0x1288 }, + Range { first: 0x128A, last: 0x128D }, + Range { first: 0x1290, last: 0x12B0 }, + Range { first: 0x12B2, last: 0x12B5 }, + Range { first: 0x12B8, last: 0x12BE }, + Range { first: 0x12C0, last: 0x12C0 }, + Range { first: 0x12C2, last: 0x12C5 }, + Range { first: 0x12C8, last: 0x12D6 }, + Range { first: 0x12D8, last: 0x1310 }, + Range { first: 0x1312, last: 0x1315 }, + Range { first: 0x1318, last: 0x135A }, + Range { first: 0x135D, last: 0x135F }, + Range { first: 0x1369, last: 0x1371 }, + Range { first: 0x1380, last: 0x138F }, + Range { first: 0x13A0, last: 0x13F5 }, + Range { first: 0x13F8, last: 0x13FD }, + Range { first: 0x1401, last: 0x166C }, + Range { first: 0x166F, last: 0x167F }, + Range { first: 0x1681, last: 0x169A }, + Range { first: 0x16A0, last: 0x16EA }, + Range { first: 0x16EE, last: 0x16F8 }, + Range { first: 0x1700, last: 0x1715 }, + Range { first: 0x171F, last: 0x1734 }, + Range { first: 0x1740, last: 0x1753 }, + Range { first: 0x1760, last: 0x176C }, + Range { first: 0x176E, last: 0x1770 }, + Range { first: 0x1772, last: 0x1773 }, + Range { first: 0x1780, last: 0x17D3 }, + Range { first: 0x17D7, last: 0x17D7 }, + Range { first: 0x17DC, last: 0x17DD }, + Range { first: 0x17E0, last: 0x17E9 }, + Range { first: 0x180B, last: 0x180D }, + Range { first: 0x180F, last: 0x1819 }, + Range { first: 0x1820, last: 0x1878 }, + Range { first: 0x1880, last: 0x18AA }, + Range { first: 0x18B0, last: 0x18F5 }, + Range { first: 0x1900, last: 0x191E }, + Range { first: 0x1920, last: 0x192B }, + Range { first: 0x1930, last: 0x193B }, + Range { first: 0x1946, last: 0x196D }, + Range { first: 0x1970, last: 0x1974 }, + Range { first: 0x1980, last: 0x19AB }, + Range { first: 0x19B0, last: 0x19C9 }, + Range { first: 0x19D0, last: 0x19DA }, + Range { first: 0x1A00, last: 0x1A1B }, + Range { first: 0x1A20, last: 0x1A5E }, + Range { first: 0x1A60, last: 0x1A7C }, + Range { first: 0x1A7F, last: 0x1A89 }, + Range { first: 0x1A90, last: 0x1A99 }, + Range { first: 0x1AA7, last: 0x1AA7 }, + Range { first: 0x1AB0, last: 0x1ABD }, + Range { first: 0x1ABF, last: 0x1AF0 }, + Range { first: 0x1B00, last: 0x1B4C }, + Range { first: 0x1B50, last: 0x1B59 }, + Range { first: 0x1B6B, last: 0x1B73 }, + Range { first: 0x1B80, last: 0x1BF3 }, + Range { first: 0x1C00, last: 0x1C37 }, + Range { first: 0x1C40, last: 0x1C49 }, + Range { first: 0x1C4D, last: 0x1C7D }, + Range { first: 0x1C80, last: 0x1C8A }, + Range { first: 0x1C90, last: 0x1CBA }, + Range { first: 0x1CBD, last: 0x1CBF }, + Range { first: 0x1CD0, last: 0x1CD2 }, + Range { first: 0x1CD4, last: 0x1CFA }, + Range { first: 0x1D00, last: 0x1F15 }, + Range { first: 0x1F18, last: 0x1F1D }, + Range { first: 0x1F20, last: 0x1F45 }, + Range { first: 0x1F48, last: 0x1F4D }, + Range { first: 0x1F50, last: 0x1F57 }, + Range { first: 0x1F59, last: 0x1F59 }, + Range { first: 0x1F5B, last: 0x1F5B }, + Range { first: 0x1F5D, last: 0x1F5D }, + Range { first: 0x1F5F, last: 0x1F7D }, + Range { first: 0x1F80, last: 0x1FB4 }, + Range { first: 0x1FB6, last: 0x1FBC }, + Range { first: 0x1FBE, last: 0x1FBE }, + Range { first: 0x1FC2, last: 0x1FC4 }, + Range { first: 0x1FC6, last: 0x1FCC }, + Range { first: 0x1FD0, last: 0x1FD3 }, + Range { first: 0x1FD6, last: 0x1FDB }, + Range { first: 0x1FE0, last: 0x1FEC }, + Range { first: 0x1FF2, last: 0x1FF4 }, + Range { first: 0x1FF6, last: 0x1FFC }, + Range { first: 0x200C, last: 0x200D }, + Range { first: 0x203F, last: 0x2040 }, + Range { first: 0x2054, last: 0x2054 }, + Range { first: 0x2071, last: 0x2071 }, + Range { first: 0x207F, last: 0x207F }, + Range { first: 0x208F, last: 0x209F }, + Range { first: 0x20D0, last: 0x20DC }, + Range { first: 0x20E1, last: 0x20E1 }, + Range { first: 0x20E5, last: 0x20F0 }, + Range { first: 0x2102, last: 0x2102 }, + Range { first: 0x2107, last: 0x2107 }, + Range { first: 0x210A, last: 0x2113 }, + Range { first: 0x2115, last: 0x2115 }, + Range { first: 0x2118, last: 0x211D }, + Range { first: 0x2124, last: 0x2124 }, + Range { first: 0x2126, last: 0x2126 }, + Range { first: 0x2128, last: 0x2128 }, + Range { first: 0x212A, last: 0x2139 }, + Range { first: 0x213C, last: 0x213F }, + Range { first: 0x2145, last: 0x2149 }, + Range { first: 0x214E, last: 0x214E }, + Range { first: 0x2160, last: 0x2188 }, + Range { first: 0x2C00, last: 0x2CE4 }, + Range { first: 0x2CEB, last: 0x2CF3 }, + Range { first: 0x2D00, last: 0x2D25 }, + Range { first: 0x2D27, last: 0x2D27 }, + Range { first: 0x2D2D, last: 0x2D2D }, + Range { first: 0x2D30, last: 0x2D67 }, + Range { first: 0x2D6F, last: 0x2D6F }, + Range { first: 0x2D7F, last: 0x2D96 }, + Range { first: 0x2DA0, last: 0x2DA6 }, + Range { first: 0x2DA8, last: 0x2DAE }, + Range { first: 0x2DB0, last: 0x2DB6 }, + Range { first: 0x2DB8, last: 0x2DBE }, + Range { first: 0x2DC0, last: 0x2DC6 }, + Range { first: 0x2DC8, last: 0x2DCE }, + Range { first: 0x2DD0, last: 0x2DD6 }, + Range { first: 0x2DD8, last: 0x2DDE }, + Range { first: 0x2DE0, last: 0x2DFF }, + Range { first: 0x3005, last: 0x3007 }, + Range { first: 0x3021, last: 0x302F }, + Range { first: 0x3031, last: 0x3035 }, + Range { first: 0x3038, last: 0x303C }, + Range { first: 0x3041, last: 0x3096 }, + Range { first: 0x3099, last: 0x309F }, + Range { first: 0x30A1, last: 0x30FF }, + Range { first: 0x3105, last: 0x312F }, + Range { first: 0x3131, last: 0x318E }, + Range { first: 0x31A0, last: 0x31BF }, + Range { first: 0x31F0, last: 0x31FF }, + Range { first: 0x3400, last: 0x4DBF }, + Range { first: 0x4E00, last: 0xA48C }, + Range { first: 0xA4D0, last: 0xA4FD }, + Range { first: 0xA500, last: 0xA60C }, + Range { first: 0xA610, last: 0xA62B }, + Range { first: 0xA640, last: 0xA66F }, + Range { first: 0xA674, last: 0xA67D }, + Range { first: 0xA67F, last: 0xA6F1 }, + Range { first: 0xA717, last: 0xA71F }, + Range { first: 0xA722, last: 0xA788 }, + Range { first: 0xA78B, last: 0xA7DD }, + Range { first: 0xA7E2, last: 0xA7E2 }, + Range { first: 0xA7F1, last: 0xA827 }, + Range { first: 0xA82C, last: 0xA82C }, + Range { first: 0xA840, last: 0xA873 }, + Range { first: 0xA880, last: 0xA8C5 }, + Range { first: 0xA8D0, last: 0xA8D9 }, + Range { first: 0xA8E0, last: 0xA8F7 }, + Range { first: 0xA8FB, last: 0xA8FB }, + Range { first: 0xA8FD, last: 0xA92D }, + Range { first: 0xA930, last: 0xA953 }, + Range { first: 0xA960, last: 0xA97C }, + Range { first: 0xA980, last: 0xA9C0 }, + Range { first: 0xA9CF, last: 0xA9D9 }, + Range { first: 0xA9E0, last: 0xA9FE }, + Range { first: 0xAA00, last: 0xAA36 }, + Range { first: 0xAA40, last: 0xAA4D }, + Range { first: 0xAA50, last: 0xAA59 }, + Range { first: 0xAA60, last: 0xAA76 }, + Range { first: 0xAA7A, last: 0xAAC2 }, + Range { first: 0xAADB, last: 0xAADD }, + Range { first: 0xAAE0, last: 0xAAEF }, + Range { first: 0xAAF2, last: 0xAAF6 }, + Range { first: 0xAB01, last: 0xAB06 }, + Range { first: 0xAB09, last: 0xAB0E }, + Range { first: 0xAB11, last: 0xAB16 }, + Range { first: 0xAB20, last: 0xAB26 }, + Range { first: 0xAB28, last: 0xAB2E }, + Range { first: 0xAB30, last: 0xAB5A }, + Range { first: 0xAB5C, last: 0xAB69 }, + Range { first: 0xAB6C, last: 0xAB6D }, + Range { first: 0xAB70, last: 0xABEA }, + Range { first: 0xABEC, last: 0xABED }, + Range { first: 0xABF0, last: 0xABF9 }, + Range { first: 0xAC00, last: 0xD7A3 }, + Range { first: 0xD7B0, last: 0xD7C6 }, + Range { first: 0xD7CB, last: 0xD7FB }, + Range { first: 0xF900, last: 0xFA6D }, + Range { first: 0xFA70, last: 0xFAD9 }, + Range { first: 0xFB00, last: 0xFB06 }, + Range { first: 0xFB13, last: 0xFB17 }, + Range { first: 0xFB1D, last: 0xFB28 }, + Range { first: 0xFB2A, last: 0xFB36 }, + Range { first: 0xFB38, last: 0xFB3C }, + Range { first: 0xFB3E, last: 0xFB3E }, + Range { first: 0xFB40, last: 0xFB41 }, + Range { first: 0xFB43, last: 0xFB44 }, + Range { first: 0xFB46, last: 0xFBB1 }, + Range { first: 0xFBD3, last: 0xFD3D }, + Range { first: 0xFD50, last: 0xFD8F }, + Range { first: 0xFD92, last: 0xFDC7 }, + Range { first: 0xFDF0, last: 0xFDFB }, + Range { first: 0xFE00, last: 0xFE0F }, + Range { first: 0xFE20, last: 0xFE2F }, + Range { first: 0xFE33, last: 0xFE34 }, + Range { first: 0xFE4D, last: 0xFE4F }, + Range { first: 0xFE70, last: 0xFE74 }, + Range { first: 0xFE76, last: 0xFEFC }, + Range { first: 0xFF10, last: 0xFF19 }, + Range { first: 0xFF21, last: 0xFF3A }, + Range { first: 0xFF3F, last: 0xFF3F }, + Range { first: 0xFF41, last: 0xFF5A }, + Range { first: 0xFF65, last: 0xFFBE }, + Range { first: 0xFFC2, last: 0xFFC7 }, + Range { first: 0xFFCA, last: 0xFFCF }, + Range { first: 0xFFD2, last: 0xFFD7 }, + Range { first: 0xFFDA, last: 0xFFDC }, + Range { first: 0x10000, last: 0x1000B }, + Range { first: 0x1000D, last: 0x10026 }, + Range { first: 0x10028, last: 0x1003A }, + Range { first: 0x1003C, last: 0x1003D }, + Range { first: 0x1003F, last: 0x1004D }, + Range { first: 0x10050, last: 0x1005D }, + Range { first: 0x10080, last: 0x100FA }, + Range { first: 0x10140, last: 0x10174 }, + Range { first: 0x101FD, last: 0x101FD }, + Range { first: 0x10280, last: 0x1029C }, + Range { first: 0x102A0, last: 0x102D0 }, + Range { first: 0x102E0, last: 0x102E0 }, + Range { first: 0x10300, last: 0x1031F }, + Range { first: 0x1032D, last: 0x1034A }, + Range { first: 0x10350, last: 0x1037A }, + Range { first: 0x10380, last: 0x1039D }, + Range { first: 0x103A0, last: 0x103C3 }, + Range { first: 0x103C8, last: 0x103CF }, + Range { first: 0x103D1, last: 0x103D5 }, + Range { first: 0x10400, last: 0x1049D }, + Range { first: 0x104A0, last: 0x104A9 }, + Range { first: 0x104B0, last: 0x104D3 }, + Range { first: 0x104D8, last: 0x104FB }, + Range { first: 0x10500, last: 0x10527 }, + Range { first: 0x10530, last: 0x10563 }, + Range { first: 0x10570, last: 0x1057A }, + Range { first: 0x1057C, last: 0x1058A }, + Range { first: 0x1058C, last: 0x10592 }, + Range { first: 0x10594, last: 0x10595 }, + Range { first: 0x10597, last: 0x105A1 }, + Range { first: 0x105A3, last: 0x105B1 }, + Range { first: 0x105B3, last: 0x105B9 }, + Range { first: 0x105BB, last: 0x105BC }, + Range { first: 0x105C0, last: 0x105F3 }, + Range { first: 0x10600, last: 0x10736 }, + Range { first: 0x10740, last: 0x10755 }, + Range { first: 0x10760, last: 0x10767 }, + Range { first: 0x10780, last: 0x10785 }, + Range { first: 0x10787, last: 0x107B0 }, + Range { first: 0x107B2, last: 0x107BF }, + Range { first: 0x10800, last: 0x10805 }, + Range { first: 0x10808, last: 0x10808 }, + Range { first: 0x1080A, last: 0x10835 }, + Range { first: 0x10837, last: 0x10838 }, + Range { first: 0x1083C, last: 0x1083C }, + Range { first: 0x1083F, last: 0x10855 }, + Range { first: 0x10860, last: 0x10876 }, + Range { first: 0x10880, last: 0x1089E }, + Range { first: 0x108E0, last: 0x108F2 }, + Range { first: 0x108F4, last: 0x108F5 }, + Range { first: 0x10900, last: 0x10915 }, + Range { first: 0x10920, last: 0x10939 }, + Range { first: 0x10940, last: 0x10959 }, + Range { first: 0x10980, last: 0x109B7 }, + Range { first: 0x109BE, last: 0x109BF }, + Range { first: 0x10A00, last: 0x10A03 }, + Range { first: 0x10A05, last: 0x10A06 }, + Range { first: 0x10A0C, last: 0x10A13 }, + Range { first: 0x10A15, last: 0x10A17 }, + Range { first: 0x10A19, last: 0x10A35 }, + Range { first: 0x10A38, last: 0x10A3A }, + Range { first: 0x10A3F, last: 0x10A3F }, + Range { first: 0x10A60, last: 0x10A7C }, + Range { first: 0x10A80, last: 0x10A9C }, + Range { first: 0x10AC0, last: 0x10AC7 }, + Range { first: 0x10AC9, last: 0x10AE6 }, + Range { first: 0x10B00, last: 0x10B35 }, + Range { first: 0x10B40, last: 0x10B55 }, + Range { first: 0x10B60, last: 0x10B72 }, + Range { first: 0x10B80, last: 0x10B91 }, + Range { first: 0x10C00, last: 0x10C48 }, + Range { first: 0x10C80, last: 0x10CB2 }, + Range { first: 0x10CC0, last: 0x10CF2 }, + Range { first: 0x10D00, last: 0x10D27 }, + Range { first: 0x10D30, last: 0x10D39 }, + Range { first: 0x10D40, last: 0x10D65 }, + Range { first: 0x10D69, last: 0x10D6D }, + Range { first: 0x10D6F, last: 0x10D85 }, + Range { first: 0x10E80, last: 0x10EA9 }, + Range { first: 0x10EAB, last: 0x10EAC }, + Range { first: 0x10EB0, last: 0x10EB1 }, + Range { first: 0x10EC2, last: 0x10EC7 }, + Range { first: 0x10ECB, last: 0x10ECF }, + Range { first: 0x10ED9, last: 0x10EEE }, + Range { first: 0x10EF0, last: 0x10F1C }, + Range { first: 0x10F27, last: 0x10F27 }, + Range { first: 0x10F30, last: 0x10F50 }, + Range { first: 0x10F70, last: 0x10F85 }, + Range { first: 0x10FB0, last: 0x10FC4 }, + Range { first: 0x10FE0, last: 0x10FF6 }, + Range { first: 0x11000, last: 0x11046 }, + Range { first: 0x11066, last: 0x11075 }, + Range { first: 0x1107F, last: 0x110BA }, + Range { first: 0x110C2, last: 0x110C2 }, + Range { first: 0x110D0, last: 0x110E8 }, + Range { first: 0x110F0, last: 0x110F9 }, + Range { first: 0x11100, last: 0x11134 }, + Range { first: 0x11136, last: 0x1113F }, + Range { first: 0x11144, last: 0x11147 }, + Range { first: 0x11150, last: 0x11173 }, + Range { first: 0x11176, last: 0x11176 }, + Range { first: 0x11180, last: 0x111C4 }, + Range { first: 0x111C9, last: 0x111CC }, + Range { first: 0x111CE, last: 0x111DA }, + Range { first: 0x111DC, last: 0x111DC }, + Range { first: 0x11200, last: 0x11211 }, + Range { first: 0x11213, last: 0x11237 }, + Range { first: 0x1123E, last: 0x11241 }, + Range { first: 0x11280, last: 0x11286 }, + Range { first: 0x11288, last: 0x11288 }, + Range { first: 0x1128A, last: 0x1128D }, + Range { first: 0x1128F, last: 0x1129D }, + Range { first: 0x1129F, last: 0x112A8 }, + Range { first: 0x112B0, last: 0x112EA }, + Range { first: 0x112F0, last: 0x112F9 }, + Range { first: 0x11300, last: 0x11303 }, + Range { first: 0x11305, last: 0x1130C }, + Range { first: 0x1130F, last: 0x11310 }, + Range { first: 0x11313, last: 0x11328 }, + Range { first: 0x1132A, last: 0x11330 }, + Range { first: 0x11332, last: 0x11333 }, + Range { first: 0x11335, last: 0x11339 }, + Range { first: 0x1133B, last: 0x11344 }, + Range { first: 0x11347, last: 0x11348 }, + Range { first: 0x1134B, last: 0x1134D }, + Range { first: 0x11350, last: 0x11350 }, + Range { first: 0x11357, last: 0x11357 }, + Range { first: 0x1135D, last: 0x11363 }, + Range { first: 0x11366, last: 0x1136C }, + Range { first: 0x11370, last: 0x11374 }, + Range { first: 0x11380, last: 0x11389 }, + Range { first: 0x1138B, last: 0x1138B }, + Range { first: 0x1138E, last: 0x1138E }, + Range { first: 0x11390, last: 0x113B5 }, + Range { first: 0x113B7, last: 0x113C0 }, + Range { first: 0x113C2, last: 0x113C2 }, + Range { first: 0x113C5, last: 0x113C5 }, + Range { first: 0x113C7, last: 0x113CA }, + Range { first: 0x113CC, last: 0x113D3 }, + Range { first: 0x113E1, last: 0x113E2 }, + Range { first: 0x11400, last: 0x1144A }, + Range { first: 0x11450, last: 0x11459 }, + Range { first: 0x1145E, last: 0x11461 }, + Range { first: 0x11480, last: 0x114C5 }, + Range { first: 0x114C7, last: 0x114C7 }, + Range { first: 0x114D0, last: 0x114D9 }, + Range { first: 0x11580, last: 0x115B5 }, + Range { first: 0x115B8, last: 0x115C0 }, + Range { first: 0x115D8, last: 0x115DD }, + Range { first: 0x11600, last: 0x11640 }, + Range { first: 0x11644, last: 0x11644 }, + Range { first: 0x11650, last: 0x11659 }, + Range { first: 0x11680, last: 0x116B8 }, + Range { first: 0x116C0, last: 0x116C9 }, + Range { first: 0x116D0, last: 0x116E3 }, + Range { first: 0x11700, last: 0x1171A }, + Range { first: 0x1171D, last: 0x1172B }, + Range { first: 0x11730, last: 0x11739 }, + Range { first: 0x11740, last: 0x11746 }, + Range { first: 0x11800, last: 0x1183A }, + Range { first: 0x118A0, last: 0x118E9 }, + Range { first: 0x118FF, last: 0x11906 }, + Range { first: 0x11909, last: 0x11909 }, + Range { first: 0x1190C, last: 0x11913 }, + Range { first: 0x11915, last: 0x11916 }, + Range { first: 0x11918, last: 0x11935 }, + Range { first: 0x11937, last: 0x11938 }, + Range { first: 0x1193B, last: 0x11943 }, + Range { first: 0x11950, last: 0x11959 }, + Range { first: 0x119A0, last: 0x119A7 }, + Range { first: 0x119AA, last: 0x119D7 }, + Range { first: 0x119DA, last: 0x119E1 }, + Range { first: 0x119E3, last: 0x119E4 }, + Range { first: 0x11A00, last: 0x11A3E }, + Range { first: 0x11A47, last: 0x11A47 }, + Range { first: 0x11A50, last: 0x11A99 }, + Range { first: 0x11A9D, last: 0x11A9D }, + Range { first: 0x11AB0, last: 0x11AF8 }, + Range { first: 0x11B0A, last: 0x11B0A }, + Range { first: 0x11B60, last: 0x11B67 }, + Range { first: 0x11BC0, last: 0x11BE0 }, + Range { first: 0x11BF0, last: 0x11BF9 }, + Range { first: 0x11C00, last: 0x11C08 }, + Range { first: 0x11C0A, last: 0x11C36 }, + Range { first: 0x11C38, last: 0x11C40 }, + Range { first: 0x11C50, last: 0x11C59 }, + Range { first: 0x11C72, last: 0x11C8F }, + Range { first: 0x11C92, last: 0x11CA7 }, + Range { first: 0x11CA9, last: 0x11CB6 }, + Range { first: 0x11D00, last: 0x11D06 }, + Range { first: 0x11D08, last: 0x11D09 }, + Range { first: 0x11D0B, last: 0x11D36 }, + Range { first: 0x11D3A, last: 0x11D3A }, + Range { first: 0x11D3C, last: 0x11D3D }, + Range { first: 0x11D3F, last: 0x11D47 }, + Range { first: 0x11D50, last: 0x11D59 }, + Range { first: 0x11D60, last: 0x11D65 }, + Range { first: 0x11D67, last: 0x11D68 }, + Range { first: 0x11D6A, last: 0x11D8E }, + Range { first: 0x11D90, last: 0x11D91 }, + Range { first: 0x11D93, last: 0x11D98 }, + Range { first: 0x11DA0, last: 0x11DA9 }, + Range { first: 0x11DB0, last: 0x11DDB }, + Range { first: 0x11DE0, last: 0x11DE9 }, + Range { first: 0x11DF0, last: 0x11DF1 }, + Range { first: 0x11EE0, last: 0x11EF6 }, + Range { first: 0x11F00, last: 0x11F10 }, + Range { first: 0x11F12, last: 0x11F3A }, + Range { first: 0x11F3E, last: 0x11F42 }, + Range { first: 0x11F50, last: 0x11F5A }, + Range { first: 0x11FB0, last: 0x11FB0 }, + Range { first: 0x12000, last: 0x12399 }, + Range { first: 0x12400, last: 0x1246F }, + Range { first: 0x12475, last: 0x12543 }, + Range { first: 0x12550, last: 0x12686 }, + Range { first: 0x12F90, last: 0x12FF0 }, + Range { first: 0x13000, last: 0x1342F }, + Range { first: 0x13440, last: 0x13455 }, + Range { first: 0x13460, last: 0x143FA }, + Range { first: 0x14400, last: 0x14646 }, + Range { first: 0x16100, last: 0x16139 }, + Range { first: 0x16800, last: 0x16A38 }, + Range { first: 0x16A40, last: 0x16A5E }, + Range { first: 0x16A60, last: 0x16A69 }, + Range { first: 0x16A70, last: 0x16ABE }, + Range { first: 0x16AC0, last: 0x16AC9 }, + Range { first: 0x16AD0, last: 0x16AED }, + Range { first: 0x16AF0, last: 0x16AF4 }, + Range { first: 0x16B00, last: 0x16B36 }, + Range { first: 0x16B40, last: 0x16B43 }, + Range { first: 0x16B50, last: 0x16B59 }, + Range { first: 0x16B63, last: 0x16B77 }, + Range { first: 0x16B7D, last: 0x16B8F }, + Range { first: 0x16D40, last: 0x16D6C }, + Range { first: 0x16D70, last: 0x16D79 }, + Range { first: 0x16E40, last: 0x16E7F }, + Range { first: 0x16EA0, last: 0x16EB8 }, + Range { first: 0x16EBB, last: 0x16ED3 }, + Range { first: 0x16F00, last: 0x16F4A }, + Range { first: 0x16F4F, last: 0x16F87 }, + Range { first: 0x16F8F, last: 0x16F9F }, + Range { first: 0x16FE0, last: 0x16FE1 }, + Range { first: 0x16FE3, last: 0x16FE4 }, + Range { first: 0x16FF0, last: 0x16FF6 }, + Range { first: 0x17000, last: 0x18CDA }, + Range { first: 0x18CFF, last: 0x18D20 }, + Range { first: 0x18D80, last: 0x18DF2 }, + Range { first: 0x18E00, last: 0x19191 }, + Range { first: 0x191A0, last: 0x191D2 }, + Range { first: 0x1AFF0, last: 0x1AFF3 }, + Range { first: 0x1AFF5, last: 0x1AFFB }, + Range { first: 0x1AFFD, last: 0x1AFFE }, + Range { first: 0x1B000, last: 0x1B128 }, + Range { first: 0x1B132, last: 0x1B132 }, + Range { first: 0x1B150, last: 0x1B152 }, + Range { first: 0x1B155, last: 0x1B155 }, + Range { first: 0x1B164, last: 0x1B168 }, + Range { first: 0x1B170, last: 0x1B2FB }, + Range { first: 0x1BC00, last: 0x1BC6A }, + Range { first: 0x1BC70, last: 0x1BC7C }, + Range { first: 0x1BC80, last: 0x1BC88 }, + Range { first: 0x1BC90, last: 0x1BC99 }, + Range { first: 0x1BC9D, last: 0x1BC9E }, + Range { first: 0x1CCF0, last: 0x1CCF9 }, + Range { first: 0x1CF00, last: 0x1CF2D }, + Range { first: 0x1CF30, last: 0x1CF46 }, + Range { first: 0x1D127, last: 0x1D128 }, + Range { first: 0x1D165, last: 0x1D169 }, + Range { first: 0x1D16D, last: 0x1D172 }, + Range { first: 0x1D17B, last: 0x1D182 }, + Range { first: 0x1D185, last: 0x1D18B }, + Range { first: 0x1D1AA, last: 0x1D1AD }, + Range { first: 0x1D242, last: 0x1D244 }, + Range { first: 0x1D250, last: 0x1D252 }, + Range { first: 0x1D25B, last: 0x1D25C }, + Range { first: 0x1D25F, last: 0x1D25F }, + Range { first: 0x1D280, last: 0x1D281 }, + Range { first: 0x1D400, last: 0x1D454 }, + Range { first: 0x1D456, last: 0x1D49C }, + Range { first: 0x1D49E, last: 0x1D49F }, + Range { first: 0x1D4A2, last: 0x1D4A2 }, + Range { first: 0x1D4A5, last: 0x1D4A6 }, + Range { first: 0x1D4A9, last: 0x1D4AC }, + Range { first: 0x1D4AE, last: 0x1D4B9 }, + Range { first: 0x1D4BB, last: 0x1D4BB }, + Range { first: 0x1D4BD, last: 0x1D4C3 }, + Range { first: 0x1D4C5, last: 0x1D505 }, + Range { first: 0x1D507, last: 0x1D50A }, + Range { first: 0x1D50D, last: 0x1D514 }, + Range { first: 0x1D516, last: 0x1D51C }, + Range { first: 0x1D51E, last: 0x1D539 }, + Range { first: 0x1D53B, last: 0x1D53E }, + Range { first: 0x1D540, last: 0x1D544 }, + Range { first: 0x1D546, last: 0x1D546 }, + Range { first: 0x1D54A, last: 0x1D550 }, + Range { first: 0x1D552, last: 0x1D6A6 }, + Range { first: 0x1D6A8, last: 0x1D6C0 }, + Range { first: 0x1D6C2, last: 0x1D6DA }, + Range { first: 0x1D6DC, last: 0x1D6FA }, + Range { first: 0x1D6FC, last: 0x1D714 }, + Range { first: 0x1D716, last: 0x1D734 }, + Range { first: 0x1D736, last: 0x1D74E }, + Range { first: 0x1D750, last: 0x1D76E }, + Range { first: 0x1D770, last: 0x1D788 }, + Range { first: 0x1D78A, last: 0x1D7A8 }, + Range { first: 0x1D7AA, last: 0x1D7C2 }, + Range { first: 0x1D7C4, last: 0x1D7CB }, + Range { first: 0x1D7CE, last: 0x1D7FF }, + Range { first: 0x1DA00, last: 0x1DA36 }, + Range { first: 0x1DA3B, last: 0x1DA6C }, + Range { first: 0x1DA75, last: 0x1DA75 }, + Range { first: 0x1DA84, last: 0x1DA84 }, + Range { first: 0x1DA9B, last: 0x1DA9F }, + Range { first: 0x1DAA1, last: 0x1DAAF }, + Range { first: 0x1DF00, last: 0x1DF81 }, + Range { first: 0x1DF90, last: 0x1DF96 }, + Range { first: 0x1DFCD, last: 0x1E006 }, + Range { first: 0x1E008, last: 0x1E018 }, + Range { first: 0x1E01B, last: 0x1E021 }, + Range { first: 0x1E023, last: 0x1E024 }, + Range { first: 0x1E026, last: 0x1E02A }, + Range { first: 0x1E030, last: 0x1E06D }, + Range { first: 0x1E08F, last: 0x1E08F }, + Range { first: 0x1E100, last: 0x1E12C }, + Range { first: 0x1E130, last: 0x1E13D }, + Range { first: 0x1E140, last: 0x1E149 }, + Range { first: 0x1E14E, last: 0x1E14E }, + Range { first: 0x1E290, last: 0x1E2AE }, + Range { first: 0x1E2C0, last: 0x1E2F9 }, + Range { first: 0x1E4D0, last: 0x1E4F9 }, + Range { first: 0x1E5D0, last: 0x1E5FA }, + Range { first: 0x1E6C0, last: 0x1E6DE }, + Range { first: 0x1E6E0, last: 0x1E6F5 }, + Range { first: 0x1E6FE, last: 0x1E6FF }, + Range { first: 0x1E7E0, last: 0x1E7E6 }, + Range { first: 0x1E7E8, last: 0x1E7EB }, + Range { first: 0x1E7ED, last: 0x1E7EE }, + Range { first: 0x1E7F0, last: 0x1E7FE }, + Range { first: 0x1E800, last: 0x1E8C4 }, + Range { first: 0x1E8D0, last: 0x1E8D6 }, + Range { first: 0x1E900, last: 0x1E94B }, + Range { first: 0x1E950, last: 0x1E959 }, + Range { first: 0x1EE00, last: 0x1EE03 }, + Range { first: 0x1EE05, last: 0x1EE1F }, + Range { first: 0x1EE21, last: 0x1EE22 }, + Range { first: 0x1EE24, last: 0x1EE24 }, + Range { first: 0x1EE27, last: 0x1EE27 }, + Range { first: 0x1EE29, last: 0x1EE32 }, + Range { first: 0x1EE34, last: 0x1EE37 }, + Range { first: 0x1EE39, last: 0x1EE39 }, + Range { first: 0x1EE3B, last: 0x1EE3B }, + Range { first: 0x1EE42, last: 0x1EE42 }, + Range { first: 0x1EE47, last: 0x1EE47 }, + Range { first: 0x1EE49, last: 0x1EE49 }, + Range { first: 0x1EE4B, last: 0x1EE4B }, + Range { first: 0x1EE4D, last: 0x1EE4F }, + Range { first: 0x1EE51, last: 0x1EE52 }, + Range { first: 0x1EE54, last: 0x1EE54 }, + Range { first: 0x1EE57, last: 0x1EE57 }, + Range { first: 0x1EE59, last: 0x1EE59 }, + Range { first: 0x1EE5B, last: 0x1EE5B }, + Range { first: 0x1EE5D, last: 0x1EE5D }, + Range { first: 0x1EE5F, last: 0x1EE5F }, + Range { first: 0x1EE61, last: 0x1EE62 }, + Range { first: 0x1EE64, last: 0x1EE64 }, + Range { first: 0x1EE67, last: 0x1EE6A }, + Range { first: 0x1EE6C, last: 0x1EE72 }, + Range { first: 0x1EE74, last: 0x1EE77 }, + Range { first: 0x1EE79, last: 0x1EE7C }, + Range { first: 0x1EE7E, last: 0x1EE7E }, + Range { first: 0x1EE80, last: 0x1EE89 }, + Range { first: 0x1EE8B, last: 0x1EE9B }, + Range { first: 0x1EEA1, last: 0x1EEA3 }, + Range { first: 0x1EEA5, last: 0x1EEA9 }, + Range { first: 0x1EEAB, last: 0x1EEBB }, + Range { first: 0x1FBF0, last: 0x1FBF9 }, + Range { first: 0x20000, last: 0x2A6DF }, + Range { first: 0x2A700, last: 0x2B81E }, + Range { first: 0x2B820, last: 0x2CEAD }, + Range { first: 0x2CEB0, last: 0x2EBE0 }, + Range { first: 0x2EBF0, last: 0x2EE5D }, + Range { first: 0x2F800, last: 0x2FA1D }, + Range { first: 0x30000, last: 0x3134A }, + Range { first: 0x31350, last: 0x33479 }, + Range { first: 0x3D000, last: 0x3FC3F }, + Range { first: 0xE0100, last: 0xE01EF }, +]; + +} // verus! diff --git a/crates/clazel-js-lexer/tests/unicode_tables.rs b/crates/clazel-js-lexer/tests/unicode_tables.rs new file mode 100644 index 000000000..3969f89f7 --- /dev/null +++ b/crates/clazel-js-lexer/tests/unicode_tables.rs @@ -0,0 +1,73 @@ +//! The `ID_Start` and `ID_Continue` tables (`src/unicode_data.rs`, generated +//! from Unicode 18.0.0 by `xtask/src/js_idents.rs`) and their lookups: the +//! shape the bisection relies on, a property Unicode guarantees, and known +//! code points. Nothing about the tables is proven (`src/unicode.rs`). + +use clazel_js_lexer::unicode::{Range, id_continue_non_ascii, id_start_non_ascii}; +use clazel_js_lexer::unicode_data::{ID_CONTINUE, ID_START}; + +/// Sorted, disjoint, non-empty rows, all within U+0080..U+10FFFF. +fn well_formed(table: &[Range]) { + let mut previous: Option = None; + for row in table { + assert!(0x80 <= row.first && row.first <= row.last && row.last <= 0x10_FFFF); + if let Some(p) = previous { + // Maximal ranges: not even adjacent. + assert!(p.saturating_add(1) < row.first); + } + previous = Some(row.last); + } +} + +#[test] +fn tables_are_sorted_and_disjoint() { + well_formed(&ID_START); + well_formed(&ID_CONTINUE); +} + +/// Every lookup agrees with a linear scan of its table, and `ID_Start` is a +/// subset of `ID_Continue` (Unicode's stability policy for identifiers). +#[test] +fn lookups_agree_with_the_tables() { + let in_table = |table: &[Range], c: u32| table.iter().any(|r| r.first <= c && c <= r.last); + // Under Miri a stride of the code points (each scan is about 800 rows); + // natively a smaller stride than 1 would take minutes in a debug build, + // so every 7th code point plus every row's ends. + let stride = if cfg!(miri) { 40_009 } else { 7 }; + let mut points: Vec = (0..=0x11_0000u32).step_by(stride).collect(); + for row in ID_START.iter().chain(ID_CONTINUE.iter()) { + points.extend([ + row.first.saturating_sub(1), + row.first, + row.last, + row.last.saturating_add(1), + ]); + } + for c in points { + let start = id_start_non_ascii(c); + let cont = id_continue_non_ascii(c); + assert_eq!(start, c >= 0x80 && in_table(&ID_START, c), "U+{c:04X}"); + assert_eq!(cont, c >= 0x80 && in_table(&ID_CONTINUE, c), "U+{c:04X}"); + assert!(!start || cont, "U+{c:04X} is ID_Start but not ID_Continue"); + } +} + +#[test] +fn known_code_points() { + for c in [0xE9, 0xAA, 0x212E, 0x65E5, 0x1_0400, 0x2_0000, 0x3_134A] { + assert!(id_start_non_ascii(c), "U+{c:04X} is ID_Start"); + } + for c in [0x300, 0x661, 0x200C, 0x200D, 0xB7, 0x1369] { + assert!(!id_start_non_ascii(c), "U+{c:04X} is not ID_Start"); + assert!(id_continue_non_ascii(c), "U+{c:04X} is ID_Continue"); + } + for c in [ + 0x7F, 0x80, 0xA0, 0xD7, 0x2028, 0x2E2F, 0xD800, 0xFEFF, 0x1_F600, 0x10_FFFF, 0x11_0000, + ] { + assert!(!id_continue_non_ascii(c), "U+{c:04X} is not ID_Continue"); + } + // ASCII is not in the tables (the spec defines it). + for c in 0..0x80 { + assert!(!id_start_non_ascii(c) && !id_continue_non_ascii(c)); + } +} diff --git a/xtask/src/js_idents.rs b/xtask/src/js_idents.rs new file mode 100644 index 000000000..2253d90c8 --- /dev/null +++ b/xtask/src/js_idents.rs @@ -0,0 +1,191 @@ +//! The `ID_Start` and `ID_Continue` tables of `clazel-js-lexer` (Phase 7 +//! item 7.2). +//! +//! ECMA-262 §12.7 defines `UnicodeIDStart` and `UnicodeIDContinue` as the +//! code points with the Unicode properties `ID_Start` and `ID_Continue`. +//! The lexer's spec defines them for ASCII and leaves the rest +//! uninterpreted; the executable lexer reads them from +//! `crates/clazel-js-lexer/src/unicode_data.rs`, which this module writes +//! from `DerivedCoreProperties.txt` of Unicode 18.0.0, the copy vendored for +//! `clazel-js-regexp` (`crates/clazel-js-regexp/tests/data/ucd-regexp/`; +//! `cargo xtask testdata` checks that directory's inventory). `cargo xtask +//! tables` writes it; `check` and `verify` fail if the file's SHA-256 +//! differs from that `SOURCE.md` or the committed table differs from what +//! this produces. +//! +//! What it derives (also listed in the generated file's header): for each +//! of the two properties, the code points above U+007F listed with it, as +//! maximal ranges, sorted and disjoint. + +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; + +use crate::reviews; +use crate::unicode::{data_lines, range, read}; + +/// The vendored data directory, relative to the repository root. +const DATA_DIR: &str = "crates/clazel-js-regexp/tests/data/ucd-regexp"; +/// The one file read. +const FILE: &str = "DerivedCoreProperties.txt"; +/// The generated file, relative to the repository root. +const OUTPUT: &str = "crates/clazel-js-lexer/src/unicode_data.rs"; + +const MAX_CODE_POINT: u32 = 0x10_FFFF; + +/// Compare (or, with `write`, rewrite) the generated table. Returns +/// (problems, notes). +pub fn run(write: bool) -> (Vec, Vec) { + let dir = PathBuf::from(DATA_DIR); + let mut notes = Vec::new(); + if let Err(e) = check_hash(&dir) { + return (vec![e], notes); + } + let fresh = match generate(&dir) { + Ok(f) => f, + Err(e) => return (vec![format!("{DATA_DIR}: {e}")], notes), + }; + let mut problems = Vec::new(); + let current = std::fs::read_to_string(OUTPUT).ok(); + if current.as_deref() == Some(fresh.as_str()) { + notes.push(format!("{OUTPUT}: up to date")); + } else if write { + match std::fs::write(OUTPUT, &fresh) { + Ok(()) => notes.push(format!("{OUTPUT}: written")), + Err(e) => problems.push(format!("{OUTPUT}: {e}")), + } + } else if current.is_none() { + problems.push(format!("{OUTPUT}: missing; run `cargo xtask tables`")); + } else { + problems.push(format!( + "{OUTPUT}: stale (differs from what xtask/src/js_idents.rs produces); \ + run `cargo xtask tables` and commit the result" + )); + } + (problems, notes) +} + +/// The file's SHA-256 is the one its `SOURCE.md` lists, and its header +/// names Unicode 18.0.0. +fn check_hash(dir: &Path) -> Result<(), String> { + let source = std::fs::read_to_string(dir.join("SOURCE.md")) + .map_err(|e| format!("{DATA_DIR}/SOURCE.md: {e}"))?; + let listed = source + .lines() + .find_map(|l| { + let rest = l.trim().strip_prefix("SHA-256:")?.trim(); + let (hash, name) = rest.split_once(char::is_whitespace)?; + (name.trim().trim_matches('`') == FILE).then(|| hash.to_ascii_lowercase()) + }) + .ok_or(format!( + "{DATA_DIR}/SOURCE.md: no \"SHA-256: {FILE}\" line" + ))?; + let actual = reviews::sha256(&dir.join(FILE))?; + if actual != listed { + return Err(format!( + "{DATA_DIR}/{FILE}: SHA-256 {actual} differs from SOURCE.md ({listed})" + )); + } + let text = read(dir, FILE)?; + if !text + .lines() + .next() + .is_some_and(|l| l.contains("DerivedCoreProperties-18.0.0.txt")) + { + return Err(format!("{DATA_DIR}/{FILE}: not Unicode 18.0.0")); + } + Ok(()) +} + +/// The maximal ranges above U+007F of the code points `FILE` lists with +/// property `name`. +fn ranges(text: &str, name: &str) -> Result, String> { + let len = usize::try_from(MAX_CODE_POINT) + .map_err(|e| e.to_string())? + .saturating_add(1); + let mut has = vec![false; len]; + let mut found = false; + for (line, f) in data_lines(text) { + if f.get(1).copied() != Some(name) { + continue; + } + found = true; + let (a, b) = range(f.first().copied().unwrap_or(""), FILE, line)?; + if a > b || b > MAX_CODE_POINT { + return Err(format!("{FILE}:{line}: bad range")); + } + for c in a..=b { + let i = usize::try_from(c).map_err(|e| e.to_string())?; + if let Some(slot) = has.get_mut(i) { + *slot = true; + } + } + } + if !found { + return Err(format!("{FILE}: no {name} lines")); + } + let mut out: Vec<(u32, u32)> = Vec::new(); + for (c, &v) in (0u32..).zip(&has) { + if !v || c < 0x80 { + continue; + } + match out.last_mut() { + Some((_, last)) if last.checked_add(1) == Some(c) => *last = c, + _ => out.push((c, c)), + } + } + Ok(out) +} + +fn generate(dir: &Path) -> Result { + let text = read(dir, FILE)?; + let id_start = ranges(&text, "ID_Start")?; + let id_continue = ranges(&text, "ID_Continue")?; + let mut out = String::new(); + out.push_str(HEADER); + table( + &mut out, + "ID_START", + "`ID_Start` above U+007F (DerivedCoreProperties.txt), as maximal ranges.", + &id_start, + ); + table( + &mut out, + "ID_CONTINUE", + "`ID_Continue` above U+007F (DerivedCoreProperties.txt), as maximal ranges.", + &id_continue, + ); + out.push_str("\n} // verus!\n"); + Ok(out) +} + +const HEADER: &str = "\ +//! The Unicode `ID_Start` and `ID_Continue` tables of the lexer (§12.7). +//! +// @generated by `cargo xtask tables` (xtask/src/js_idents.rs) from +// crates/clazel-js-regexp/tests/data/ucd-regexp/DerivedCoreProperties.txt +// (Unicode 18.0.0; URL and SHA-256 in SOURCE.md there). Do not edit: +// `cargo xtask check` fails if this differs from what the generator +// produces. +// +// What the generator derives: for each of ID_Start and ID_Continue, the +// code points above U+007F that DerivedCoreProperties.txt lists with it, as +// maximal ranges (first, last), sorted and disjoint. + +use super::unicode::Range; +#[allow(unused_imports)] +use vstd::prelude::*; + +verus! { +"; + +fn table(out: &mut String, name: &str, doc: &str, rows: &[(u32, u32)]) { + let _ = write!( + out, + "\n/// {doc}\n#[verifier::opaque]\npub const {name}: [Range; {}] = [\n", + rows.len() + ); + for (a, b) in rows { + let _ = writeln!(out, " Range {{ first: 0x{a:X}, last: 0x{b:X} }},"); + } + out.push_str("];\n"); +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 16ceec385..31fda61e1 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -22,7 +22,8 @@ //! `verify` fail if a committed table is stale. //! - `cargo xtask tables`: regenerate the Unicode tables generated from //! vendored Unicode data (`crates/clazel-idna/src/uts46/data.rs` and -//! `crates/clazel-layout/src/text/data.rs`), the +//! `crates/clazel-layout/src/text/data.rs` and +//! `crates/clazel-js-lexer/src/unicode_data.rs`), the //! Public Suffix List, and the root certificate table from NSS's //! `certdata.txt` (`crates/clazel-x509/src/roots/data.rs`), and the //! user agent style sheet from HTML §15 (`crates/clazel-style/data/*.css`). @@ -39,6 +40,7 @@ mod conformance; mod deps; mod encoding_indexes; mod entities; +mod js_idents; mod linebreak; mod mutate; mod parts; @@ -367,6 +369,8 @@ fn static_checks(r: &mut Runner, crates: &[Crate], pins: &Pins) { r.step("generated test data", outcome(problems, notes)); let (problems, notes) = unicode::run(false); r.step("generated Unicode tables", outcome(problems, notes)); + let (problems, notes) = js_idents::run(false); + r.step("generated JS identifier tables", outcome(problems, notes)); let (problems, notes) = linebreak::run(false); r.step("generated line breaking tables", outcome(problems, notes)); let (problems, notes) = psl::run(false); @@ -1029,6 +1033,8 @@ fn generate_tables() -> ExitCode { } let (problems, notes) = unicode::run(true); r.step("generate Unicode tables", outcome(problems, notes)); + let (problems, notes) = js_idents::run(true); + r.step("generate JS identifier tables", outcome(problems, notes)); let (problems, notes) = linebreak::run(true); r.step("generate line breaking tables", outcome(problems, notes)); let (problems, notes) = psl::run(true); From 598a250772998fbb7ec7ed00bda98c4069ed089c Mon Sep 17 00:00:00 2001 From: Alex Tiernan-Berry Date: Tue, 17 Nov 2026 11:52:14 +0000 Subject: [PATCH 2/5] Implement the JS lexer, proven total and span-covering (7.2) src/exec.rs: next_input_element and next_token as Verus exec code, one scan_ function per production of spec.rs, no recursion. Verus proves (no escape hatch) that every input gives a result without panic or overflow, that Eof comes exactly at the end, that an element is a non-empty range at pos inside the source text (so elements tile the input), that the value's variant matches the kind, and that next_token's token is after pos, not trivia, with newline_before set only by a line terminator in between. Not proven: agreement with spec.rs (the kind derives the range, the length is longest_len, the errors are lex_error's, the values are SV/TV/TRV/ IdentifierCodePoints). So the functions do not implement api::JsLexer and the crate stays at tier 0. lemma_contract_implies_proven shows the proven part is a weakening of the contract. Co-Authored-By: Claude Opus 5.5 --- crates/clazel-js-lexer/Cargo.toml | 6 +- crates/clazel-js-lexer/src/exec.rs | 1916 ++++++++++++++++++++++++++++ crates/clazel-js-lexer/src/lib.rs | 11 +- 3 files changed, 1928 insertions(+), 5 deletions(-) create mode 100644 crates/clazel-js-lexer/src/exec.rs diff --git a/crates/clazel-js-lexer/Cargo.toml b/crates/clazel-js-lexer/Cargo.toml index e2dd9eb22..02ff533b8 100644 --- a/crates/clazel-js-lexer/Cargo.toml +++ b/crates/clazel-js-lexer/Cargo.toml @@ -12,8 +12,10 @@ vstd = "=0.0.0-2026-10-04-0306" # Verification tier (docs/VERIFICATION.md). `tier` is what is proven about the # shipped code today; `target-tier` is what the policy requires, since this # lexes scripts a page chose (phase-7.md 7.0(a)). Tier 0: the spec and the -# contracts are written (item 7.2, first half, CLAUDE.md rule 3) and check -# with Verus, but no implementation exists, so nothing shipped is proven. +# contracts are written (item 7.2, first half, CLAUDE.md rule 3), and the +# implementation (src/exec.rs) is proven total, panic-free and to tile its +# input, but not to agree with the spec: it does not meet src/contract.rs +# yet, and there is no independent transcription or mutation set. [package.metadata.clazel] tier = 0 target-tier = 2 diff --git a/crates/clazel-js-lexer/src/exec.rs b/crates/clazel-js-lexer/src/exec.rs new file mode 100644 index 000000000..721c95e10 --- /dev/null +++ b/crates/clazel-js-lexer/src/exec.rs @@ -0,0 +1,1916 @@ +//! The executable lexer (phase-7.md item 7.2, second half). +//! +//! Each `scan_` function reads one production of [`super::spec`] at a +//! position, as a loop with no recursion (a script is what a page chose; +//! Verus proves no panic, not no stack overflow), and returns where it +//! ends. [`next_input_element`] dispatches on the first code point, takes +//! the longest of the alternatives that can start there (§12), and then +//! applies the two checks the spec makes after the longest element is +//! chosen ([`lex_error`]); [`next_token`] repeats it over trivia. +//! +//! # What is proven, and what is not +//! +//! Verus proves, for every input, with no escape hatch: +//! +//! - **Totality:** every function terminates (each loop has a `decreases`) +//! and never panics: no index out of range, no overflow. +//! - **Span coverage:** [`next_input_element_proven`]: the end of the +//! source text gives `Eof`, and nothing else does; an element starts at +//! `pos`, ends after it and inside the source text; so elements from +//! repeated calls tile the source text. [`next_token_proven`]: the token +//! starts at or after `pos`, is not trivia, and its `newline_before` is +//! [`contains_line_terminator`] of some trivia element between them. +//! - **The value's shape:** [`value_shape_agrees`]: an element has the +//! kind of `Value` its kind has (a `StringLiteral` a `Value::String`, and +//! so on), and a regular expression's `body_end` is inside it. +//! - **Single code points:** the classifiers (`is_ws`, `is_lt`, the digit +//! classes, `hex_val`) equal the spec's, and `push_utf16` is +//! UTF16EncodeCodePoint (§11.1.1). +//! +//! It is **not** proven that the results agree with [`super::spec`]: that +//! the kind derives the range, that no longer element exists +//! ([`longest_len`]), that the errors are [`lex_error`]'s, and that the +//! values are the spec's relations. These are the rest of +//! [`super::contract`], which the functions here do not yet claim, so they +//! do not implement [`super::api::JsLexer`]. Their agreement with the spec +//! is tested (`tests/`), not proven. [`lemma_contract_implies_proven`] +//! shows that what is proven is implied by the full contract. +//! +//! Non-ASCII `ID_Start` and `ID_Continue` come from [`super::unicode`], +//! which the spec leaves uninterpreted: nothing is claimed about them. + +#[allow(unused_imports)] +use super::spec::*; +#[allow(unused_imports)] +use vstd::prelude::*; + +use super::api::{Element, Next, Step, Token, Value}; + +verus! { + +// --------------------------------------------------------------------- +// What the executable lexer is proven to satisfy. +// --------------------------------------------------------------------- + +/// The kind of [`Value`] an element has is the one its kind has (the shape +/// of [`super::contract::element_value_agrees`], not the values), and a +/// regular expression's closing `/` is inside it, after a non-empty body. +pub open spec fn value_shape_agrees(el: Element) -> bool { + match el.value { + Value::None => !(el.kind == ElementKind::IdentifierName + || el.kind == ElementKind::PrivateIdentifier || el.kind == ElementKind::StringLiteral + || el.kind == ElementKind::NumericLiteral + || el.kind == ElementKind::RegularExpressionLiteral + || el.kind == ElementKind::Template(TemplateKind::NoSubstitution) + || el.kind == ElementKind::Template(TemplateKind::Head) + || el.kind == ElementKind::Template(TemplateKind::Middle) + || el.kind == ElementKind::Template(TemplateKind::Tail)), + Value::Name { .. } => el.kind == ElementKind::IdentifierName || el.kind + == ElementKind::PrivateIdentifier, + Value::String { .. } => el.kind == ElementKind::StringLiteral, + Value::Template { .. } => el.kind == ElementKind::Template(TemplateKind::NoSubstitution) + || el.kind == ElementKind::Template(TemplateKind::Head) || el.kind + == ElementKind::Template(TemplateKind::Middle) || el.kind == ElementKind::Template( + TemplateKind::Tail, + ), + Value::Numeric { .. } => el.kind == ElementKind::NumericLiteral, + Value::RegExp { body_end } => el.kind == ElementKind::RegularExpressionLiteral + && el.start + 1 < body_end < el.end, + } +} + +/// What [`next_input_element`] is proven to return: totality and span +/// coverage. A weakening of [`super::contract::next_input_element_ensures`] +/// ([`lemma_contract_implies_proven`]). +pub open spec fn next_input_element_proven(src: Seq, pos: usize, r: Result) -> bool { + match r { + Err(_) => pos < src.len(), + Ok(Step::Eof) => pos == src.len(), + Ok(Step::Element(el)) => pos < src.len() && el.start == pos && pos < el.end && el.end + <= src.len() && value_shape_agrees(el), + } +} + +/// What [`next_token`] is proven to return: the token is after `pos`, +/// inside the source text and not trivia, and the flag is set only by a +/// line terminator code point between `pos` and the token. +pub open spec fn next_token_proven(src: Seq, pos: usize, r: Result) -> bool { + match r { + Err(_) => pos < src.len(), + Ok(Next::Eof { newline_before }) => newline_before ==> contains_line_terminator( + src, + pos as int, + src.len() as int, + ), + Ok(Next::Token(tok)) => pos <= tok.element.start < tok.element.end <= src.len() + && !is_trivia(tok.element.kind) && value_shape_agrees(tok.element) && ( + tok.newline_before ==> contains_line_terminator( + src, + pos as int, + tok.element.start as int, + )), + } +} + +/// The full contract implies what is proven of [`next_input_element`] +/// (except the value's shape, which the full contract states value by +/// value): the proven part is a weakening, not a different claim. +pub proof fn lemma_contract_implies_proven( + src: Seq, + pos: usize, + goal: Goal, + html: bool, + r: Result, +) + requires + pos <= src.len(), + super::contract::next_input_element_ensures(src, pos, goal, html, r), + ensures + match r { + Err(_) => pos < src.len(), + Ok(Step::Eof) => pos == src.len(), + Ok(Step::Element(el)) => pos < src.len() && el.start == pos && pos < el.end && el.end + <= src.len(), + }, +{ +} + +// --------------------------------------------------------------------- +// Reading code points. +// --------------------------------------------------------------------- + +/// What [`peek`] returns past the end of the source text. It is above +/// U+10FFFF, so it is no `SourceCharacter` and matches no class below, +/// just as a lookahead fails at the end (§11.1, `at` in the spec). +pub const EOF: u32 = 0xFFFF_FFFF; + +/// The code point at `k`, or [`EOF`]. +pub fn peek(src: &[u32], k: usize) -> (c: u32) + ensures + k < src.len() ==> c == src@[k as int], + k >= src.len() ==> c == EOF, +{ + match src.get(k) { + Some(c) => *c, + None => EOF, + } +} + +/// §11.1 `SourceCharacter`. +fn is_src(c: u32) -> (r: bool) + ensures + r == is_source_character(c), +{ + c <= 0x10FFFF +} + +/// `low <= c <= high`. +fn between(c: u32, low: u32, high: u32) -> (r: bool) + ensures + r == (low <= c <= high), +{ + low <= c && c <= high +} + +/// §12.3 `LineTerminator`: , , , . +fn is_lt(c: u32) -> (r: bool) + ensures + r == is_line_terminator(c), +{ + c == 0x0A || c == 0x0D || c == 0x2028 || c == 0x2029 +} + +/// `SourceCharacter but not LineTerminator` (§12.4 `SingleLineCommentChar`, +/// §12.9.5 `RegularExpressionNonTerminator`). +fn is_line_char(c: u32) -> (r: bool) + ensures + r == is_not_line_terminator(c), +{ + is_src(c) && !is_lt(c) +} + +/// §12.2 `WhiteSpace`: , , , , . +fn is_ws(c: u32) -> (r: bool) + ensures + r == (c == 0x0009 || c == 0x000B || c == 0x000C || c == 0xFEFF + || unicode_space_separator(c)), +{ + c == 0x0009 || c == 0x000B || c == 0x000C || c == 0xFEFF || c == 0x0020 || c == 0x00A0 || c + == 0x1680 || between(c, 0x2000, 0x200A) || c == 0x202F || c == 0x205F || c == 0x3000 +} + +/// §12.9.3 `DecimalDigit`. +fn is_dec(c: u32) -> (r: bool) + ensures + r == is_decimal_digit(c), +{ + between(c, 0x30, 0x39) +} + +/// §12.9.3 `OctalDigit`. +fn is_oct(c: u32) -> (r: bool) + ensures + r == is_octal_digit(c), +{ + between(c, 0x30, 0x37) +} + +/// §12.9.3 `HexDigit`. +fn is_hex(c: u32) -> (r: bool) + ensures + r == is_hex_digit(c), +{ + between(c, 0x30, 0x39) || between(c, 0x61, 0x66) || between(c, 0x41, 0x46) +} + +/// `BinaryDigit`, `OctalDigit`, `DecimalDigit` or `HexDigit` by radix. +fn is_radix(radix: u32, c: u32) -> (r: bool) + requires + radix == 2 || radix == 8 || radix == 10 || radix == 16, + ensures + r == is_radix_digit(radix as int, c), +{ + if radix == 2 { + c == 0x30 || c == 0x31 + } else if radix == 8 { + is_oct(c) + } else if radix == 10 { + is_dec(c) + } else { + is_hex(c) + } +} + +/// The MV of a `HexDigit` (§12.9.3.2). +fn hex_val(c: u32) -> (r: u32) + requires + is_hex_digit(c), + ensures + r as int == hex_digit_value(c), + r < 16, +{ + if between(c, 0x30, 0x39) { + c.saturating_sub(0x30) + } else if between(c, 0x61, 0x66) { + c.saturating_sub(0x61).saturating_add(10) + } else { + c.saturating_sub(0x41).saturating_add(10) + } +} + +/// §12.7 `IdentifierStartChar :: UnicodeIDStart | $ | _`. Above U+007F, +/// [`super::unicode`]'s table, about which nothing is claimed. +fn id_start_char(c: u32) -> (r: bool) + ensures + c < 0x80 ==> r == is_identifier_start_char(c), + r ==> c <= 0x10FFFF, + r ==> c != 0x5C, +{ + if c < 0x80 { + between(c, 0x61, 0x7A) || between(c, 0x41, 0x5A) || c == 0x24 || c == 0x5F + } else { + super::unicode::id_start_non_ascii(c) + } +} + +/// §12.7 `IdentifierPartChar :: UnicodeIDContinue | $`. +fn id_part_char(c: u32) -> (r: bool) + ensures + c < 0x80 ==> r == is_identifier_part_char(c), + r ==> c <= 0x10FFFF, + r ==> c != 0x5C, +{ + if c < 0x80 { + between(c, 0x61, 0x7A) || between(c, 0x41, 0x5A) || between(c, 0x30, 0x39) || c + == 0x24 || c == 0x5F + } else { + super::unicode::id_continue_non_ascii(c) + } +} + +/// Appends UTF16EncodeCodePoint(`cp`) (§11.1.1). +fn push_utf16(out: &mut Vec, cp: u32) + requires + cp <= 0x10FFFF, + ensures + final(out)@ == old(out)@ + utf16_encode_code_point(cp as int), +{ + if cp <= 0xFFFF { + // 2. "If cp ≤ 0xFFFF, return the String value consisting of the + // code unit whose numeric value is cp." + out.push(cp as u16); + assert(final(out)@ =~= old(out)@ + utf16_encode_code_point(cp as int)); + } else { + let v = cp.saturating_sub(0x10000); + // 3. cu1 is floor((cp - 0x10000) / 0x400) + 0xD800. + let cu1 = (v / 0x400).saturating_add(0xD800) as u16; + // 4. cu2 is ((cp - 0x10000) modulo 0x400) + 0xDC00. + let cu2 = (v % 0x400).saturating_add(0xDC00) as u16; + // 5. The string-concatenation of cu1 and cu2. + out.push(cu1); + out.push(cu2); + assert(final(out)@ =~= old(out)@ + utf16_encode_code_point(cp as int)); + } +} + +/// Appends the UTF-16 of `src[i..j)`, each code point a `SourceCharacter`. +fn push_range_utf16(out: &mut Vec, src: &[u32], i: usize, j: usize) + requires + i <= j <= src.len(), +{ + let mut k = i; + while k < j + invariant + i <= k <= j <= src.len(), + decreases j - k, + { + let c = peek(src, k); + if c <= 0x10FFFF { + push_utf16(out, c); + } + k = k.saturating_add(1); + } +} + +// --------------------------------------------------------------------- +// §12.4 Comments, §12.5 Hashbang Comments, Annex B.1.1 +// --------------------------------------------------------------------- + +/// §12.4 `SingleLineCommentChars?` from `k`: the first index at or after +/// `k` that is not a `SourceCharacter` other than a `LineTerminator`. +fn scan_line_chars(src: &[u32], k: usize) -> (r: usize) + requires + k <= src.len(), + ensures + k <= r <= src.len(), + all_of(src@, k as int, r as int, |c: u32| is_not_line_terminator(c)), + r < src.len() ==> !is_not_line_terminator(src@[r as int]), +{ + let mut j = k; + loop + invariant + k <= j <= src.len(), + all_of(src@, k as int, j as int, |c: u32| is_not_line_terminator(c)), + decreases src.len() - j, + { + let c = peek(src, j); + if is_line_char(c) { + j = j.saturating_add(1); + } else { + return j; + } + } +} + +/// §12.4 `MultiLineComment` (or, with `!multi_line`, Annex B's +/// `SingleLineDelimitedComment`) after its `/*` at `k`: the end of the +/// first `*/` and whether a `LineTerminator` came before it. `None` if the +/// source text ends, or a code point that is not a `SourceCharacter` (or, +/// with `!multi_line`, a `LineTerminator`) comes, first. +fn scan_block_comment(src: &[u32], k: usize, multi_line: bool) -> (r: Option<(usize, bool)>) + requires + k <= src.len(), + ensures + match r { + Some((e, _)) => k + 2 <= e <= src.len(), + None => true, + }, +{ + let mut j = k; + let mut lt = false; + loop + invariant + k <= j <= src.len(), + decreases src.len() - j, + { + let c = peek(src, j); + if c == 0x2A && peek(src, j.saturating_add(1)) == 0x2F { + return Some((j.saturating_add(2), lt)); + } + if !is_src(c) || (!multi_line && is_lt(c)) { + return None; + } + if is_lt(c) { + lt = true; + } + j = j.saturating_add(1); + } +} + +/// Annex B.1.1 `WhiteSpaceSequence?` from `k`. +fn scan_ws(src: &[u32], k: usize) -> (r: usize) + requires + k <= src.len(), + ensures + k <= r <= src.len(), +{ + let mut j = k; + loop + invariant + k <= j <= src.len(), + decreases src.len() - j, + { + if is_ws(peek(src, j)) { + j = j.saturating_add(1); + } else { + return j; + } + } +} + +/// Annex B.1.1 the end of `-->` SingleLineCommentChars? at `k`. +fn scan_html_arrow(src: &[u32], k: usize) -> (r: Option) + requires + k <= src.len(), + ensures + match r { + Some(e) => k + 3 <= e <= src.len(), + None => true, + }, +{ + if peek(src, k) == 0x2D && peek(src, k.saturating_add(1)) == 0x2D && peek( + src, + k.saturating_add(2), + ) == 0x3E { + Some(scan_line_chars(src, k.saturating_add(3))) + } else { + None + } +} + +/// Annex B.1.1 `HTMLCloseComment :: WhiteSpaceSequence? +/// SingleLineDelimitedCommentSequence? --> SingleLineCommentChars?` at `k`. +/// Each part is determined by the code point that starts it (white space, +/// `/*` or `-->`), so the parse is the only one, and it ends where the +/// line does. +fn scan_html_close(src: &[u32], k: usize) -> (r: Option) + requires + k <= src.len(), + ensures + match r { + Some(e) => k + 3 <= e <= src.len(), + None => true, + }, +{ + let mut j = scan_ws(src, k); + loop + invariant + k <= j <= src.len(), + decreases src.len() - j, + { + if peek(src, j) == 0x2F && peek(src, j.saturating_add(1)) == 0x2A { + // SingleLineDelimitedComment WhiteSpaceSequence? + match scan_block_comment(src, j.saturating_add(2), false) { + Some((e, _)) => { + j = scan_ws(src, e); + }, + None => { + return None; + }, + } + } else { + return scan_html_arrow(src, j); + } + } +} + +/// The `LineTerminatorSequence` at `k` (§12.3): is one. +fn line_terminator_sequence_end(src: &[u32], k: usize) -> (r: usize) + requires + k < src.len(), + is_line_terminator(src@[k as int]), + ensures + k < r <= src.len(), +{ + if peek(src, k) == 0x0D && peek(src, k.saturating_add(1)) == 0x0A { + k.saturating_add(2) + } else { + k.saturating_add(1) + } +} + +// --------------------------------------------------------------------- +// §12.7 Names +// --------------------------------------------------------------------- + +/// The MV of the `HexDigits[~Sep]` that start at `k` (all of them), capped +/// at 0x110000 (any value above U+10FFFF is `NotCodePoint`), and the end. +fn scan_hex_digits(src: &[u32], k: usize) -> (r: (u32, usize)) + requires + k <= src.len(), + ensures + r.0 <= 0x110000, + k <= r.1 <= src.len(), +{ + let mut j = k; + let mut v: u32 = 0; + loop + invariant + k <= j <= src.len(), + v <= 0x110000, + decreases src.len() - j, + { + let c = peek(src, j); + if is_hex(c) { + let d = hex_val(c); + if v <= 0x10FFFF { + let w = v.saturating_mul(16).saturating_add(d); + v = if w > 0x10FFFF { + 0x110000 + } else { + w + }; + } + j = j.saturating_add(1); + } else { + return (v, j); + } + } +} + +/// §12.9.4 `Hex4Digits` at `k`: its MV. +fn scan_hex4(src: &[u32], k: usize) -> (r: Option) + requires + k <= src.len(), + ensures + match r { + Some(v) => k + 4 <= src.len() && v <= 0xFFFF, + None => true, + }, +{ + let a = peek(src, k); + let b = peek(src, k.saturating_add(1)); + let c = peek(src, k.saturating_add(2)); + let d = peek(src, k.saturating_add(3)); + if is_hex(a) && is_hex(b) && is_hex(c) && is_hex(d) { + let v = hex_val(a).saturating_mul(0x1000).saturating_add( + hex_val(b).saturating_mul(0x100), + ).saturating_add(hex_val(c).saturating_mul(0x10)).saturating_add(hex_val(d)); + Some(v) + } else { + None + } +} + +/// §12.9.4 `UnicodeEscapeSequence :: u Hex4Digits | u{ CodePoint }` at `k` +/// (after the backslash): its code point (§12.7.1.3 IdentifierCodePoint) +/// and its end. +fn scan_unicode_escape(src: &[u32], k: usize) -> (r: Option<(u32, usize)>) + requires + k <= src.len(), + ensures + match r { + Some((cp, e)) => k < e <= src.len() && cp <= 0x10FFFF, + None => true, + }, +{ + if peek(src, k) != 0x75 { + return None; + } + if peek(src, k.saturating_add(1)) == 0x7B { + // 2. `u{` CodePoint `}`, with CodePoint's MV at most 0x10FFFF. + let (v, e) = scan_hex_digits(src, k.saturating_add(2)); + if e > k.saturating_add(2) && v <= 0x10FFFF && peek(src, e) == 0x7D { + Some((v, e.saturating_add(1))) + } else { + None + } + } else { + // 1. `u` Hex4Digits + let v = scan_hex4(src, k.saturating_add(1))?; + Some((v, k.saturating_add(5))) + } +} + +/// §12.7 `IdentifierName` at `k`, greedily (each `IdentifierPart` is +/// determined by its first code point, so the longest derivation is the +/// greedy one): IdentifierCodePoints (§12.7.1.2), whether an escape was +/// used, whether the §12.7.1.1 early errors pass, and the end. +fn scan_identifier(src: &[u32], k: usize) -> (r: Option<(Vec, bool, bool, usize)>) + requires + k <= src.len(), + ensures + match r { + Some((_, _, _, e)) => k < e <= src.len(), + None => true, + }, +{ + let mut cps: Vec = Vec::new(); + let mut escaped = false; + let mut valid = true; + let c = peek(src, k); + let mut j: usize; + if c == 0x5C { + // IdentifierStart :: \ UnicodeEscapeSequence; §12.7.1.1: "It is a + // Syntax Error if the IdentifierCodePoint of UnicodeEscapeSequence + // is not some Unicode code point matched by the IdentifierStartChar + // lexical grammar production." + match scan_unicode_escape(src, k.saturating_add(1)) { + Some((cp, e)) => { + cps.push(cp); + escaped = true; + valid = id_start_char(cp); + j = e; + }, + None => { + return None; + }, + } + } else if id_start_char(c) { + // IdentifierStart :: IdentifierStartChar + cps.push(c); + j = k.saturating_add(1); + } else { + return None; + } + loop + invariant + k < j <= src.len(), + decreases src.len() - j, + { + let c = peek(src, j); + if c == 0x5C { + // IdentifierPart :: \ UnicodeEscapeSequence, with §12.7.1.1's + // rule for IdentifierPartChar. + match scan_unicode_escape(src, j.saturating_add(1)) { + Some((cp, e)) => { + cps.push(cp); + escaped = true; + if !id_part_char(cp) { + valid = false; + } + j = e; + }, + None => { + return Some((cps, escaped, valid, j)); + }, + } + } else if id_part_char(c) { + // IdentifierPart :: IdentifierPartChar + cps.push(c); + j = j.saturating_add(1); + } else { + return Some((cps, escaped, valid, j)); + } + } +} + +// --------------------------------------------------------------------- +// §12.8 Punctuators +// --------------------------------------------------------------------- + +/// §12.8 `Punctuator` (`OptionalChainingPunctuator` and `OtherPunctuator`) +/// at `k`: the end of the longest one, or `k` if none starts there. +fn scan_punctuator(src: &[u32], k: usize) -> (r: usize) + requires + k < src.len(), + ensures + k <= r <= src.len(), +{ + let c0 = peek(src, k); + let c1 = peek(src, k.saturating_add(1)); + let c2 = peek(src, k.saturating_add(2)); + let c3 = peek(src, k.saturating_add(3)); + let n: usize = if c0 == 0x7B || c0 == 0x28 || c0 == 0x29 || c0 == 0x5B || c0 == 0x5D || c0 + == 0x3B || c0 == 0x2C || c0 == 0x7E || c0 == 0x3A { + // { ( ) [ ] ; , ~ : + 1 + } else if c0 == 0x2E { + // ... . + if c1 == 0x2E && c2 == 0x2E { + 3 + } else { + 1 + } + } else if c0 == 0x3C { + // <<= << <= < + if c1 == 0x3C && c2 == 0x3D { + 3 + } else if c1 == 0x3C || c1 == 0x3D { + 2 + } else { + 1 + } + } else if c0 == 0x3E { + // >>>= >>> >>= >> >= > + if c1 == 0x3E && c2 == 0x3E && c3 == 0x3D { + 4 + } else if c1 == 0x3E && (c2 == 0x3E || c2 == 0x3D) { + 3 + } else if c1 == 0x3E || c1 == 0x3D { + 2 + } else { + 1 + } + } else if c0 == 0x3D { + // === == => = + if c1 == 0x3D && c2 == 0x3D { + 3 + } else if c1 == 0x3D || c1 == 0x3E { + 2 + } else { + 1 + } + } else if c0 == 0x21 { + // !== != ! + if c1 == 0x3D && c2 == 0x3D { + 3 + } else if c1 == 0x3D { + 2 + } else { + 1 + } + } else if c0 == 0x2B || c0 == 0x2D { + // ++ += +, -- -= - + if c1 == c0 || c1 == 0x3D { + 2 + } else { + 1 + } + } else if c0 == 0x2A { + // **= ** *= * + if c1 == 0x2A && c2 == 0x3D { + 3 + } else if c1 == 0x2A || c1 == 0x3D { + 2 + } else { + 1 + } + } else if c0 == 0x25 || c0 == 0x5E { + // %= %, ^= ^ + if c1 == 0x3D { + 2 + } else { + 1 + } + } else if c0 == 0x26 || c0 == 0x7C { + // &&= && &= &, ||= || |= | + if c1 == c0 && c2 == 0x3D { + 3 + } else if c1 == c0 || c1 == 0x3D { + 2 + } else { + 1 + } + } else if c0 == 0x3F { + // ??= ?? ?. ?, with OptionalChainingPunctuator :: ?. + // [lookahead ∉ DecimalDigit] + if c1 == 0x3F && c2 == 0x3D { + 3 + } else if c1 == 0x3F || (c1 == 0x2E && !is_dec(c2)) { + 2 + } else { + 1 + } + } else { + 0 + }; + k.saturating_add(n) +} + +// --------------------------------------------------------------------- +// §12.9.3 Numeric Literals +// --------------------------------------------------------------------- + +/// §12.9.3 `Digits[+Sep]` of a radix, from a digit at `k`: digits, each +/// pair of them possibly separated by one `_`. Greedy, since a `_` is +/// taken only when a digit follows it. +fn scan_digits_sep(src: &[u32], k: usize, radix: u32) -> (r: usize) + requires + k < src.len(), + radix == 2 || radix == 8 || radix == 10 || radix == 16, + ensures + k < r <= src.len(), +{ + let mut j = k.saturating_add(1); + loop + invariant + k < j <= src.len(), + radix == 2 || radix == 8 || radix == 10 || radix == 16, + decreases src.len() - j, + { + let c = peek(src, j); + if is_radix(radix, c) { + j = j.saturating_add(1); + } else if c == 0x5F && is_radix(radix, peek(src, j.saturating_add(1))) { + j = j.saturating_add(2); + } else { + return j; + } + } +} + +/// §12.9.3 `ExponentPart[+Sep]?` at `k`: its end, or `k`. +fn scan_exponent(src: &[u32], k: usize) -> (r: usize) + requires + k <= src.len(), + ensures + k <= r <= src.len(), +{ + let c = peek(src, k); + if c == 0x65 || c == 0x45 { + let s = peek(src, k.saturating_add(1)); + let m = if s == 0x2B || s == 0x2D { + k.saturating_add(2) + } else { + k.saturating_add(1) + }; + if is_dec(peek(src, m)) { + scan_digits_sep(src, m, 10) + } else { + k + } + } else { + k + } +} + +/// The rest of a §12.9.3 `DecimalLiteral` after its `DecimalIntegerLiteral` +/// ends at `k`: `. DecimalDigits[+Sep]? ExponentPart[+Sep]?` or +/// `ExponentPart[+Sep]?`. +fn scan_decimal_tail(src: &[u32], k: usize) -> (r: usize) + requires + k <= src.len(), + ensures + k <= r <= src.len(), +{ + if peek(src, k) == 0x2E { + let m = k.saturating_add(1); + let n = if is_dec(peek(src, m)) { + scan_digits_sep(src, m, 10) + } else { + m + }; + scan_exponent(src, n) + } else { + scan_exponent(src, k) + } +} + +/// §12.9.3 `NumericLiteral` at `k`, the longest: its end, whether it is a +/// BigInt literal, and whether it is legacy (§12.9.3.1). +fn scan_numeric(src: &[u32], k: usize) -> (r: Option<(usize, bool, bool)>) + requires + k < src.len(), + ensures + match r { + Some((e, _, _)) => k < e <= src.len(), + None => true, + }, +{ + let c0 = peek(src, k); + let c1 = peek(src, k.saturating_add(1)); + if c0 == 0x2E { + // DecimalLiteral :: . DecimalDigits[+Sep] ExponentPart[+Sep]? + if is_dec(c1) { + let e = scan_digits_sep(src, k.saturating_add(1), 10); + Some((scan_exponent(src, e), false, false)) + } else { + None + } + } else if c0 == 0x30 { + let c2 = peek(src, k.saturating_add(2)); + let radix: u32 = if c1 == 0x62 || c1 == 0x42 { + 2 + } else if c1 == 0x6F || c1 == 0x4F { + 8 + } else if c1 == 0x78 || c1 == 0x58 { + 16 + } else { + 0 + }; + if radix != 0 && is_radix(radix, c2) { + // NonDecimalIntegerLiteral[+Sep] BigIntLiteralSuffix? + let e = scan_digits_sep(src, k.saturating_add(2), radix); + if peek(src, e) == 0x6E { + Some((e.saturating_add(1), true, false)) + } else { + Some((e, false, false)) + } + } else if c1 == 0x6E { + // DecimalBigIntegerLiteral :: 0 BigIntLiteralSuffix + Some((k.saturating_add(2), true, false)) + } else if is_dec(c1) { + // `0` and decimal digits, with no separator: a + // LegacyOctalIntegerLiteral if they are all octal, else a + // NonOctalDecimalIntegerLiteral, which a DecimalLiteral may + // continue. + let mut j = k.saturating_add(1); + let mut octal = true; + loop + invariant + k < j <= src.len(), + decreases src.len() - j, + { + let c = peek(src, j); + if is_dec(c) { + if !is_oct(c) { + octal = false; + } + j = j.saturating_add(1); + } else { + break; + } + } + if octal { + Some((j, false, true)) + } else { + Some((scan_decimal_tail(src, j), false, true)) + } + } else { + // DecimalIntegerLiteral :: 0 + Some((scan_decimal_tail(src, k.saturating_add(1)), false, false)) + } + } else if is_dec(c0) { + // DecimalIntegerLiteral :: NonZeroDigit NumericLiteralSeparator? + // DecimalDigits[+Sep], or a DecimalBigIntegerLiteral. + let e = scan_digits_sep(src, k, 10); + if peek(src, e) == 0x6E { + Some((e.saturating_add(1), true, false)) + } else { + Some((scan_decimal_tail(src, e), false, false)) + } + } else { + None + } +} + +/// §12.9.3: "The SourceCharacter immediately following a NumericLiteral +/// must not be an IdentifierStart or DecimalDigit." +fn numeric_follow_error(src: &[u32], j: usize) -> (r: bool) + requires + j <= src.len(), +{ + let c = peek(src, j); + if is_dec(c) || id_start_char(c) { + true + } else if c == 0x5C { + scan_unicode_escape(src, j.saturating_add(1)).is_some() + } else { + false + } +} + +// --------------------------------------------------------------------- +// §12.9.4 String Literals +// --------------------------------------------------------------------- + +/// §12.9.4 `LegacyOctalEscapeSequence` at `k`, whose first digit `d` is an +/// octal digit: its MV, pushed as one code unit, and its end. The +/// lookaheads make it the longest run its first digit allows: three digits +/// after `0` to `3`, two after `4` to `7`. +fn scan_legacy_octal_escape(src: &[u32], k: usize, units: &mut Vec) -> (r: usize) + requires + k < src.len(), + is_octal_digit(src@[k as int]), + ensures + k < r <= src.len(), +{ + let d = peek(src, k); + let mut v: u32 = d.saturating_sub(0x30); + let mut e = k.saturating_add(1); + let c1 = peek(src, e); + if is_oct(c1) { + v = v.saturating_mul(8).saturating_add(c1.saturating_sub(0x30)); + e = e.saturating_add(1); + let c2 = peek(src, e); + if d <= 0x33 && is_oct(c2) { + v = v.saturating_mul(8).saturating_add(c2.saturating_sub(0x30)); + e = e.saturating_add(1); + } + } + units.push(v as u16); + e +} + +/// §12.9.4 `\ EscapeSequence` or `LineContinuation`, the backslash at +/// `k`: appends the SV (§12.9.4.2) and returns the end. +fn scan_string_escape(src: &[u32], k: usize, units: &mut Vec) -> (r: Option) + requires + k < src.len(), + ensures + match r { + Some(e) => k + 1 < e <= src.len(), + None => true, + }, +{ + let m = k.saturating_add(1); + let d = peek(src, m); + if is_lt(d) { + // LineContinuation: the empty String. + Some(line_terminator_sequence_end(src, m)) + } else if d == 0x27 || d == 0x22 || d == 0x5C { + // SingleEscapeCharacter ' " \: the character itself. + units.push(d as u16); + Some(m.saturating_add(1)) + } else if d == 0x62 || d == 0x66 || d == 0x6E || d == 0x72 || d == 0x74 || d == 0x76 { + // SingleEscapeCharacter b f n r t v (Table "String Single Character + // Escape Sequences"). + let u: u16 = if d == 0x62 { + 0x08 + } else if d == 0x74 { + 0x09 + } else if d == 0x6E { + 0x0A + } else if d == 0x76 { + 0x0B + } else if d == 0x66 { + 0x0C + } else { + 0x0D + }; + units.push(u); + Some(m.saturating_add(1)) + } else if d == 0x30 && !is_dec(peek(src, m.saturating_add(1))) { + // `0` [lookahead ∉ DecimalDigit]: 0x0000. + units.push(0); + Some(m.saturating_add(1)) + } else if d == 0x30 && !is_oct(peek(src, m.saturating_add(1))) { + // LegacyOctalEscapeSequence :: `0` [lookahead ∈ { 8, 9 }]: 0. + units.push(0); + Some(m.saturating_add(1)) + } else if is_oct(d) { + // LegacyOctalEscapeSequence, the other alternatives. + Some(scan_legacy_octal_escape(src, m, units)) + } else if d == 0x38 || d == 0x39 { + // NonOctalDecimalEscapeSequence: 0x0038 or 0x0039. + units.push(d as u16); + Some(m.saturating_add(1)) + } else if d == 0x78 { + // HexEscapeSequence :: x HexDigit HexDigit + let a = peek(src, m.saturating_add(1)); + let b = peek(src, m.saturating_add(2)); + if is_hex(a) && is_hex(b) { + units.push(hex_val(a).saturating_mul(16).saturating_add(hex_val(b)) as u16); + Some(m.saturating_add(3)) + } else { + None + } + } else if d == 0x75 { + // UnicodeEscapeSequence: UTF16EncodeCodePoint of its MV. + match scan_unicode_escape(src, m) { + Some((cp, e)) => { + push_utf16(units, cp); + Some(e) + }, + None => None, + } + } else if is_src(d) { + // NonEscapeCharacter: the code point. + push_utf16(units, d); + Some(m.saturating_add(1)) + } else { + None + } +} + +/// §12.9.4 `StringLiteral` at `k` (a quote): its SV and its end. +fn scan_string(src: &[u32], k: usize) -> (r: Option<(Vec, usize)>) + requires + k < src.len(), + src@[k as int] == 0x22 || src@[k as int] == 0x27, + ensures + match r { + Some((_, e)) => k + 2 <= e <= src.len(), + None => true, + }, +{ + let quote = peek(src, k); + let mut units: Vec = Vec::new(); + let mut j = k.saturating_add(1); + loop + invariant + k < j <= src.len(), + quote == 0x22 || quote == 0x27, + decreases src.len() - j, + { + let c = peek(src, j); + if c == quote { + return Some((units, j.saturating_add(1))); + } else if c == 0x5C { + match scan_string_escape(src, j, &mut units) { + Some(e) => { + j = e; + }, + None => { + return None; + }, + } + } else if c == 0x2028 || c == 0x2029 { + // , : the code unit. + units.push(c as u16); + j = j.saturating_add(1); + } else if is_line_char(c) { + // SourceCharacter but not one of the quote or \ or + // LineTerminator. + push_utf16(&mut units, c); + j = j.saturating_add(1); + } else { + return None; + } + } +} + +// --------------------------------------------------------------------- +// §12.9.5 Regular Expression Literals +// --------------------------------------------------------------------- + +/// §12.9.5 `RegularExpressionClass` at `k` (a `[`): its end. +fn scan_regexp_class(src: &[u32], k: usize) -> (r: Option) + requires + k < src.len(), + ensures + match r { + Some(e) => k < e <= src.len(), + None => true, + }, +{ + let mut j = k.saturating_add(1); + loop + invariant + k < j <= src.len(), + decreases src.len() - j, + { + let c = peek(src, j); + if c == 0x5D { + return Some(j.saturating_add(1)); + } else if c == 0x5C { + // RegularExpressionBackslashSequence + if is_line_char(peek(src, j.saturating_add(1))) { + j = j.saturating_add(2); + } else { + return None; + } + } else if is_line_char(c) { + j = j.saturating_add(1); + } else { + return None; + } + } +} + +/// §12.9.5 `RegularExpressionLiteral` at `k` (a `/`): the index of the +/// closing `/` and the end of the flags. +fn scan_regexp(src: &[u32], k: usize) -> (r: Option<(usize, usize)>) + requires + k < src.len(), + ensures + match r { + Some((b, e)) => k + 1 < b < e <= src.len(), + None => true, + }, +{ + let first = peek(src, k.saturating_add(1)); + // RegularExpressionFirstChar is not `*` or `/`. + if first == 0x2A || first == 0x2F { + return None; + } + let mut j = k.saturating_add(1); + loop + invariant + k < j <= src.len(), + j == k + 1 ==> (k + 1 >= src.len() || src@[(k + 1) as int] != 0x2F), + decreases src.len() - j, + { + let c = peek(src, j); + if c == 0x2F { + // The closing `/`, then RegularExpressionFlags. + let mut f = j.saturating_add(1); + loop + invariant + k + 1 < j < f <= src.len(), + decreases src.len() - f, + { + if id_part_char(peek(src, f)) { + f = f.saturating_add(1); + } else { + return Some((j, f)); + } + } + } else if c == 0x5C { + if is_line_char(peek(src, j.saturating_add(1))) { + j = j.saturating_add(2); + } else { + return None; + } + } else if c == 0x5B { + match scan_regexp_class(src, j) { + Some(e) => { + j = e; + }, + None => { + return None; + }, + } + } else if is_line_char(c) { + j = j.saturating_add(1); + } else { + return None; + } + } +} + +// --------------------------------------------------------------------- +// §12.9.6 Template Literal Lexical Components +// --------------------------------------------------------------------- + +/// A template component's TV (`None` once a `NotEscapeSequence` makes it +/// *undefined*) and TRV, as they are built. +struct TemplateValues { + cooked: Vec, + defined: bool, + raw: Vec, +} + +/// §12.9.6 `\ TemplateEscapeSequence`, `\ NotEscapeSequence` or +/// `LineContinuation`, the backslash at `k`: appends the TV and TRV +/// (§12.9.6.1, §12.9.6.2) and returns the end. +fn scan_template_escape(src: &[u32], k: usize, values: &mut TemplateValues) -> (r: Option) + requires + k < src.len(), + ensures + match r { + Some(e) => k + 1 < e <= src.len(), + None => true, + }, +{ + let m = k.saturating_add(1); + let d = peek(src, m); + if is_lt(d) { + // LineContinuation: TV is empty; TRV is \ and the TRV of the + // LineTerminatorSequence ( and are ). + let e = line_terminator_sequence_end(src, m); + values.raw.push(0x5C); + if d == 0x2028 || d == 0x2029 { + values.raw.push(d as u16); + } else { + values.raw.push(0x0A); + } + return Some(e); + } + // In every other case the TRV is the UTF-16 of the matched code points. + let e: usize; + if d == 0x27 || d == 0x22 || d == 0x5C { + values.cooked.push(d as u16); + e = m.saturating_add(1); + } else if d == 0x62 || d == 0x66 || d == 0x6E || d == 0x72 || d == 0x74 || d == 0x76 { + let u: u16 = if d == 0x62 { + 0x08 + } else if d == 0x74 { + 0x09 + } else if d == 0x6E { + 0x0A + } else if d == 0x76 { + 0x0B + } else if d == 0x66 { + 0x0C + } else { + 0x0D + }; + values.cooked.push(u); + e = m.saturating_add(1); + } else if d == 0x30 { + if is_dec(peek(src, m.saturating_add(1))) { + // NotEscapeSequence :: 0 DecimalDigit + values.defined = false; + e = m.saturating_add(2); + } else { + // TemplateEscapeSequence :: 0 [lookahead ∉ DecimalDigit] + values.cooked.push(0); + e = m.saturating_add(1); + } + } else if is_dec(d) { + // NotEscapeSequence :: DecimalDigit but not 0 + values.defined = false; + e = m.saturating_add(1); + } else if d == 0x78 { + let a = peek(src, m.saturating_add(1)); + let b = peek(src, m.saturating_add(2)); + if is_hex(a) && is_hex(b) { + // HexEscapeSequence + values.cooked.push(hex_val(a).saturating_mul(16).saturating_add(hex_val(b)) as u16); + e = m.saturating_add(3); + } else if is_hex(a) { + // NotEscapeSequence :: x HexDigit [lookahead ∉ HexDigit] + values.defined = false; + e = m.saturating_add(2); + } else { + // NotEscapeSequence :: x [lookahead ∉ HexDigit] + values.defined = false; + e = m.saturating_add(1); + } + } else if d == 0x75 { + match scan_unicode_escape(src, m) { + Some((cp, end)) => { + // UnicodeEscapeSequence + push_utf16(&mut values.cooked, cp); + e = end; + }, + None => { + // NotEscapeSequence, alternatives 5 to 11: `u` and the hex + // digits that follow (up to three), or `u{` and all of + // them; each ends where no further hex digit follows. + values.defined = false; + if peek(src, m.saturating_add(1)) == 0x7B { + let (_, h) = scan_hex_digits(src, m.saturating_add(2)); + e = h; + } else { + let mut h = m.saturating_add(1); + while is_hex(peek(src, h)) && h < m.saturating_add(4) + invariant + m < h <= src.len(), + decreases src.len() - h, + { + h = h.saturating_add(1); + } + e = h; + } + }, + } + } else if is_src(d) { + // CharacterEscapeSequence :: NonEscapeCharacter + push_utf16(&mut values.cooked, d); + e = m.saturating_add(1); + } else { + return None; + } + push_range_utf16(&mut values.raw, src, k, e); + Some(e) +} + +/// What [`scan_template`] finds: whether the component closes with `` ` ``, +/// its TV (`None` for *undefined*), its TRV, and its end. +type TemplateScan = (bool, Option>, Vec, usize); + +/// §12.9.6 a template component at `k` (`` ` `` for `NoSubstitutionTemplate` +/// and `TemplateHead`, `}` for `TemplateMiddle` and `TemplateTail`): whether +/// it closes with `` ` `` (rather than `${`), its TV and TRV, and its end. +fn scan_template(src: &[u32], k: usize) -> (r: Option) + requires + k < src.len(), + ensures + match r { + Some((_, _, _, e)) => k + 2 <= e <= src.len(), + None => true, + }, +{ + let mut values = TemplateValues { cooked: Vec::new(), defined: true, raw: Vec::new() }; + let mut j = k.saturating_add(1); + loop + invariant + k < j <= src.len(), + decreases src.len() - j, + { + let c = peek(src, j); + if c == 0x60 || (c == 0x24 && peek(src, j.saturating_add(1)) == 0x7B) { + let closes = c == 0x60; + let end = if closes { + j.saturating_add(1) + } else { + j.saturating_add(2) + }; + let cooked = if values.defined { + Some(values.cooked) + } else { + None + }; + return Some((closes, cooked, values.raw, end)); + } else if c == 0x5C { + match scan_template_escape(src, j, &mut values) { + Some(e) => { + j = e; + }, + None => { + return None; + }, + } + } else if is_lt(c) { + // LineTerminatorSequence: TV and TRV are its TRV. + let e = line_terminator_sequence_end(src, j); + let u: u16 = if c == 0x2028 || c == 0x2029 { + c as u16 + } else { + 0x0A + }; + values.cooked.push(u); + values.raw.push(u); + j = e; + } else if is_src(c) { + // `$` [lookahead ≠ {], or SourceCharacter but not one of ` \ $ + // LineTerminator: UTF16EncodeCodePoint of it. + push_utf16(&mut values.cooked, c); + push_utf16(&mut values.raw, c); + j = j.saturating_add(1); + } else { + return None; + } + } +} + +// --------------------------------------------------------------------- +// §12 The goal symbols and the longest input element. +// --------------------------------------------------------------------- + +/// The goal has `RegularExpressionLiteral`. +fn goal_regexp(goal: Goal) -> (r: bool) + ensures + r == (goal == Goal::InputElementRegExp || goal == Goal::InputElementRegExpOrTemplateTail + || goal == Goal::InputElementHashbangOrRegExp), +{ + matches!( + goal, + Goal::InputElementRegExp | Goal::InputElementRegExpOrTemplateTail + | Goal::InputElementHashbangOrRegExp + ) +} + +/// The goal has `DivPunctuator`. +fn goal_div(goal: Goal) -> (r: bool) + ensures + r == (goal == Goal::InputElementDiv || goal == Goal::InputElementTemplateTail), +{ + matches!(goal, Goal::InputElementDiv | Goal::InputElementTemplateTail) +} + +/// The goal has `RightBracePunctuator`. +fn goal_right_brace(goal: Goal) -> (r: bool) + ensures + r == (goal == Goal::InputElementDiv || goal == Goal::InputElementRegExp), +{ + matches!(goal, Goal::InputElementDiv | Goal::InputElementRegExp) +} + +/// The goal has `TemplateSubstitutionTail`. +fn goal_template_tail(goal: Goal) -> (r: bool) + ensures + r == (goal == Goal::InputElementRegExpOrTemplateTail || goal + == Goal::InputElementTemplateTail), +{ + matches!(goal, Goal::InputElementRegExpOrTemplateTail | Goal::InputElementTemplateTail) +} + +/// An element with no value. +fn plain(kind: ElementKind, start: usize, end: usize) -> (r: Element) + requires + !(kind == ElementKind::IdentifierName || kind == ElementKind::PrivateIdentifier || kind + == ElementKind::StringLiteral || kind == ElementKind::NumericLiteral || kind + == ElementKind::RegularExpressionLiteral || kind == ElementKind::Template( + TemplateKind::NoSubstitution, + ) || kind == ElementKind::Template(TemplateKind::Head) || kind == ElementKind::Template( + TemplateKind::Middle, + ) || kind == ElementKind::Template(TemplateKind::Tail)), + ensures + r.kind == kind, + r.start == start, + r.end == end, + value_shape_agrees(r), +{ + Element { kind, start, end, value: Value::None } +} + +/// An element and the error that the checks after the longest element is +/// chosen raise for it, if any (§12.9.3, §12.7.1.1; [`lex_error`]). +type Candidate = (Element, Option); + +/// The candidate's element is a non-empty range at `pos`. +pub open spec fn candidate_ok(src: Seq, pos: usize, c: (Element, Option)) -> bool { + c.0.start == pos && pos < c.0.end <= src.len() && value_shape_agrees(c.0) +} + +/// The longer of two candidates (the first on a tie). +fn longer(a: Option, b: Option) -> (r: Option) + ensures + match a { + Some(x) => match b { + Some(y) => r == Some(x) || r == Some(y), + None => r == Some(x), + }, + None => r == b, + }, +{ + match (a, b) { + (Some(x), Some(y)) => { + if y.0.end > x.0.end { + Some(y) + } else { + Some(x) + } + }, + (Some(x), None) => Some(x), + (None, y) => y, + } +} + +/// A comment that starts with `/` at `pos`: `//` (§12.4 +/// `SingleLineComment`) or `/*` (§12.4 `MultiLineComment`; with `html`, +/// Annex B.1.1's `SingleLineDelimitedComment`, or its `MultiLineComment` +/// with an optional `HTMLCloseComment` after the `*/` when the comment +/// holds a `LineTerminator`). +fn lex_slash_comment(src: &[u32], pos: usize, html: bool) -> (r: Option) + requires + pos < src.len(), + ensures + match r { + Some(c) => candidate_ok(src@, pos, c), + None => true, + }, +{ + let c1 = peek(src, pos.saturating_add(1)); + if c1 == 0x2F { + let e = scan_line_chars(src, pos.saturating_add(2)); + Some((plain(ElementKind::Comment, pos, e), None)) + } else if c1 == 0x2A { + match scan_block_comment(src, pos.saturating_add(2), true) { + Some((e, lt)) => { + let end = if html && lt { + match scan_html_close(src, e) { + Some(f) => f, + None => e, + } + } else { + e + }; + Some((plain(ElementKind::Comment, pos, end), None)) + }, + None => None, + } + } else { + None + } +} + +/// Every alternative of `goal` that can start with the code point at +/// `pos`, the longest of them. +fn lex_common(src: &[u32], pos: usize, goal: Goal, html: bool) -> (r: Option) + requires + pos < src.len(), + ensures + match r { + Some(c) => candidate_ok(src@, pos, c), + None => true, + }, +{ + let c = peek(src, pos); + let c1 = peek(src, pos.saturating_add(1)); + if is_ws(c) { + // WhiteSpace + Some((plain(ElementKind::WhiteSpace, pos, pos.saturating_add(1)), None)) + } else if is_lt(c) { + // LineTerminator; with `html`, Annex B.1.1's + // SingleLineHTMLCloseComment :: LineTerminatorSequence + // HTMLCloseComment, which is longer when it matches. + let lt = Some((plain(ElementKind::LineTerminator, pos, pos.saturating_add(1)), None)); + if html { + let s = line_terminator_sequence_end(src, pos); + longer(lt, html_close_candidate(src, pos, s)) + } else { + lt + } + } else if c == 0x2F { + // Comment, RegularExpressionLiteral or DivPunctuator. + let comment = lex_slash_comment(src, pos, html); + let regexp = if goal_regexp(goal) { + regexp_candidate(src, pos) + } else { + None + }; + let div = if goal_div(goal) { + let e = if c1 == 0x3D { + pos.saturating_add(2) + } else { + pos.saturating_add(1) + }; + Some((plain(ElementKind::DivPunctuator, pos, e), None)) + } else { + None + }; + longer(longer(comment, regexp), div) + } else if c == 0x7D { + // TemplateMiddle / TemplateTail or RightBracePunctuator. + if goal_template_tail(goal) { + lex_template(src, pos, true) + } else if goal_right_brace(goal) { + Some((plain(ElementKind::RightBracePunctuator, pos, pos.saturating_add(1)), None)) + } else { + None + } + } else if c == 0x60 { + // NoSubstitutionTemplate / TemplateHead + lex_template(src, pos, false) + } else if c == 0x22 || c == 0x27 { + // StringLiteral + let (units, e) = scan_string(src, pos)?; + Some( + ( + Element { + kind: ElementKind::StringLiteral, + start: pos, + end: e, + value: Value::String { units }, + }, + None, + ), + ) + } else if c == 0x23 { + // PrivateIdentifier :: # IdentifierName + match scan_identifier(src, pos.saturating_add(1)) { + Some((cps, escaped, valid, e)) => { + let err = if valid { + None + } else { + Some(LexError::InvalidIdentifierEscape) + }; + Some( + ( + Element { + kind: ElementKind::PrivateIdentifier, + start: pos, + end: e, + value: Value::Name { code_points: cps, escaped }, + }, + err, + ), + ) + }, + None => None, + } + } else if is_dec(c) || (c == 0x2E && is_dec(c1)) { + // NumericLiteral + match scan_numeric(src, pos) { + Some((e, bigint, legacy)) => { + let err = if numeric_follow_error(src, e) { + Some(LexError::IdentifierAfterNumericLiteral) + } else { + None + }; + Some( + ( + Element { + kind: ElementKind::NumericLiteral, + start: pos, + end: e, + value: Value::Numeric { bigint, legacy }, + }, + err, + ), + ) + }, + None => None, + } + } else if c == 0x5C || id_start_char(c) { + // IdentifierName + match scan_identifier(src, pos) { + Some((cps, escaped, valid, e)) => { + let err = if valid { + None + } else { + Some(LexError::InvalidIdentifierEscape) + }; + Some( + ( + Element { + kind: ElementKind::IdentifierName, + start: pos, + end: e, + value: Value::Name { code_points: cps, escaped }, + }, + err, + ), + ) + }, + None => None, + } + } else if html && c == 0x3C && c1 == 0x21 && peek(src, pos.saturating_add(2)) == 0x2D && peek( + src, + pos.saturating_add(3), + ) == 0x2D { + // Annex B.1.1 SingleLineHTMLOpenComment :: y\nz", DIV, true)?, + vec![ + ("IdentifierName", 0, 1), + ("Comment", 1, 7), + ("LineTerminator", 7, 8), + ("IdentifierName", 8, 9) + ] + ); + expect("\r\n /* a */ /*b*/ --> c", DIV, true, "Comment", 23)?; + expect("\n--> c", DIV, false, "LineTerminator", 1)?; + // A delimited comment holding a line terminator stops it. + expect("\n/*\n*/-->", DIV, true, "LineTerminator", 1)?; + // Annex B MultiLineComment: one holding a line terminator may be + // followed by an HTMLCloseComment. + expect("/*\n*/ --> c\nd", DIV, true, "Comment", 11)?; + expect("/* */ --> c", DIV, true, "Comment", 5)?; + // HTMLCloseComment at the start of InputElementHashbangOrRegExp. + expect("--> c\nd", START, true, "Comment", 5)?; + expect(" /**/ --> c", START, true, "Comment", 12)?; + expect("--> c", DIV, true, "Punctuator", 2)?; + expect("--> c", START, false, "Punctuator", 2)?; + Ok(()) +} + +#[test] +fn identifiers() -> Result<(), String> { + let el = expect("abc d", DIV, true, "IdentifierName", 3)?; + assert_eq!(name(&el)?, (cps("abc"), false)); + let el = expect("$_a1", DIV, true, "IdentifierName", 4)?; + assert_eq!(name(&el)?, (cps("$_a1"), false)); + let el = expect("\\u0061b", DIV, true, "IdentifierName", 7)?; + assert_eq!(name(&el)?, (cps("ab"), true)); + let el = expect("a\\u{62}\\u{0000063}", DIV, true, "IdentifierName", 18)?; + assert_eq!(name(&el)?, (cps("abc"), true)); + let el = expect("a\\u0030", DIV, true, "IdentifierName", 7)?; + assert_eq!(name(&el)?, (cps("a0"), true)); + // §12.7.1.1: an escape must name an IdentifierStartChar (first) or an + // IdentifierPartChar. + expect_err("\\u0030a", DIV, true, LexError::InvalidIdentifierEscape)?; + expect_err("a\\u002D", DIV, true, LexError::InvalidIdentifierEscape)?; + expect_err("\\u{1F600}", DIV, true, LexError::InvalidIdentifierEscape)?; + expect_err("\\uD800", DIV, true, LexError::InvalidIdentifierEscape)?; + // Malformed escapes are not part of the name. + expect("a\\u00", DIV, true, "IdentifierName", 1)?; + expect("a\\u{110000}", DIV, true, "IdentifierName", 1)?; + expect_err("\\u{}", DIV, true, LexError::NoInputElement)?; + expect_err("\\x41", DIV, true, LexError::NoInputElement)?; + // Reserved words are names; the parser decides (§12.7.2). + expect("while", DIV, true, "IdentifierName", 5)?; + // Private names. + let el = expect("#foo.x", DIV, true, "PrivateIdentifier", 4)?; + assert_eq!(name(&el)?, (cps("foo"), false)); + let el = expect("#\\u0061", DIV, true, "PrivateIdentifier", 7)?; + assert_eq!(name(&el)?, (cps("a"), true)); + expect_err("#", DIV, true, LexError::NoInputElement)?; + expect_err("# a", DIV, true, LexError::NoInputElement)?; + expect_err("#\\u0030", DIV, true, LexError::InvalidIdentifierEscape)?; + Ok(()) +} + +#[test] +fn unicode_identifiers() -> Result<(), String> { + let el = expect("café x", DIV, true, "IdentifierName", 4)?; + assert_eq!(name(&el)?, (cps("café"), false)); + // U+212E is Other_ID_Start. + expect("\u{212E}", DIV, true, "IdentifierName", 1)?; + expect("日本語", DIV, true, "IdentifierName", 3)?; + // U+10400, outside the BMP. + expect("\u{10400}", DIV, true, "IdentifierName", 1)?; + // U+0300 (a combining mark) is ID_Continue but not ID_Start. + expect("a\u{300}", DIV, true, "IdentifierName", 2)?; + expect_err("\u{300}", DIV, true, LexError::NoInputElement)?; + // ZWNJ and ZWJ are ID_Continue (since Unicode 15.1). + expect("a\u{200C}\u{200D}", DIV, true, "IdentifierName", 3)?; + expect("a\\u200D", DIV, true, "IdentifierName", 7)?; + // Arabic-Indic digits continue a name but do not start one. + expect("a\u{661}", DIV, true, "IdentifierName", 2)?; + expect_err("\u{661}", DIV, true, LexError::NoInputElement)?; + // Emoji are neither. + expect_err("\u{1F600}", DIV, true, LexError::NoInputElement)?; + expect("a\u{1F600}", DIV, true, "IdentifierName", 1)?; + Ok(()) +} + +#[test] +fn punctuators() -> Result<(), String> { + for (s, n) in [ + ("{", 1), + ("(", 1), + (")", 1), + ("[", 1), + ("]", 1), + (".", 1), + ("...", 3), + ("..", 1), + (";", 1), + (",", 1), + ("<", 1), + ("<=", 2), + ("<<", 2), + ("<<=", 3), + (">", 1), + (">=", 2), + (">>", 2), + (">>=", 3), + (">>>", 3), + (">>>=", 4), + ("=", 1), + ("==", 2), + ("===", 3), + ("====", 3), + ("=>", 2), + ("!", 1), + ("!=", 2), + ("!==", 3), + ("+", 1), + ("++", 2), + ("+=", 2), + ("+++", 2), + ("-", 1), + ("--", 2), + ("-=", 2), + ("*", 1), + ("*=", 2), + ("**", 2), + ("**=", 3), + ("%", 1), + ("%=", 2), + ("&", 1), + ("&&", 2), + ("&=", 2), + ("&&=", 3), + ("|", 1), + ("||", 2), + ("|=", 2), + ("||=", 3), + ("^", 1), + ("^=", 2), + ("~", 1), + ("?", 1), + ("??", 2), + ("??=", 3), + ("?.", 2), + ("?.a", 2), + (":", 1), + ] { + expect(s, DIV, true, "Punctuator", n)?; + } + // OptionalChainingPunctuator :: ?. [lookahead ∉ DecimalDigit] + expect("?.5:1", DIV, true, "Punctuator", 1)?; + expect("/", DIV, true, "DivPunctuator", 1)?; + expect("/=", DIV, true, "DivPunctuator", 2)?; + expect("/=", TAIL, true, "DivPunctuator", 2)?; + expect("}", DIV, true, "RightBracePunctuator", 1)?; + expect("}", RE, true, "RightBracePunctuator", 1)?; + expect_err("@", DIV, true, LexError::NoInputElement)?; + Ok(()) +} + +#[test] +fn numeric_literals() -> Result<(), String> { + for (s, n, bigint, legacy) in [ + ("0", 1, false, false), + ("0;", 1, false, false), + ("123", 3, false, false), + ("1_000_000", 9, false, false), + ("1.5", 3, false, false), + ("1.", 2, false, false), + ("1.e3", 4, false, false), + ("0.e3", 4, false, false), + (".5", 2, false, false), + (".5e-3", 5, false, false), + ("1E+10", 5, false, false), + ("1_0.0_1e1_0", 11, false, false), + ("0.5", 3, false, false), + ("0x1F", 4, false, false), + ("0XdEaD_bEeF", 11, false, false), + ("0b1010", 6, false, false), + ("0o17", 4, false, false), + ("1n", 2, true, false), + ("0n", 2, true, false), + ("1_0n", 4, true, false), + ("0x1Fn", 5, true, false), + ("0b1n", 4, true, false), + ("07", 2, false, true), + ("0777", 4, false, true), + ("08", 2, false, true), + ("019", 3, false, true), + ("08.5", 4, false, true), + ("09e1", 4, false, true), + // A LegacyOctalIntegerLiteral is not a DecimalIntegerLiteral: `07.5` + // is `07` then `.5`. + ("07.5", 2, false, true), + ("1..toString", 2, false, false), + ("1 in", 1, false, false), + ] { + let el = expect(s, DIV, true, "NumericLiteral", n)?; + assert_eq!(numeric(&el)?, (bigint, legacy), "{s:?}"); + } + // §12.9.3: no IdentifierStart or DecimalDigit right after. + for s in [ + "3in", "1n2", "0x", "0b2", "0b12", "0xg", "1e", "1e+", "1_", "1__0", "0_1", "1._5", "08n", + "07n", "00n", "1.5n", "07e1", "1$", "1\\u0061", "1é", + ] { + expect_err(s, DIV, true, LexError::IdentifierAfterNumericLiteral)?; + } + // A malformed escape after a number is not an IdentifierStart. + expect("1\\u00", DIV, true, "NumericLiteral", 1)?; + Ok(()) +} + +#[test] +fn string_literals() -> Result<(), String> { + let el = expect("\"abc\"x", DIV, true, "StringLiteral", 5)?; + assert_eq!(string(&el)?, units("abc")); + let el = expect("''", DIV, true, "StringLiteral", 2)?; + assert_eq!(string(&el)?, units("")); + for (s, v) in [ + ("'a\\nb'", "a\nb"), + ("'\\b\\f\\n\\r\\t\\v'", "\u{8}\u{c}\n\r\t\u{b}"), + ("'\\'\\\"\\\\'", "'\"\\"), + ("'\\x41'", "A"), + ("'\\u0041'", "A"), + ("'\\u{1F600}'", "\u{1F600}"), + ("'\u{1F600}'", "\u{1F600}"), + ("'\\0'", "\0"), + ("'\\08'", "\08"), + ("'\\101'", "A"), + ("'\\1010'", "A0"), + ("'\\377'", "\u{ff}"), + ("'\\400'", " 0"), + ("'\\47'", "'"), + ("'\\7'", "\u{7}"), + ("'\\8\\9'", "89"), + ("'\\a\\q'", "aq"), + ("'a\\\nb'", "ab"), + ("'a\\\r\nb'", "ab"), + ("'a\\\u{2028}b'", "ab"), + ("'a\u{2028}b'", "a\u{2028}b"), + ("\"'\"", "'"), + ] { + let el = expect(s, DIV, true, "StringLiteral", cps(s).len())?; + assert_eq!(string(&el)?, units(v), "{s:?}"); + } + let el = expect("'\\uD800'", DIV, true, "StringLiteral", 8)?; + assert_eq!(string(&el)?, vec![0xD800]); + for s in [ + "'abc", + "'a\nb'", + "'a\rb'", + "'\\x4'", + "'\\xg0'", + "'\\u004'", + "'\\u{110000}'", + "'\\u{}'", + "'\\", + "\"a'", + ] { + expect_err(s, DIV, true, LexError::NoInputElement)?; + } + Ok(()) +} + +#[test] +fn templates() -> Result<(), String> { + let el = expect("`abc`", DIV, true, "NoSubstitutionTemplate", 5)?; + assert_eq!(template(&el)?, (Some(units("abc")), units("abc"))); + let el = expect("`a${b}`", DIV, true, "TemplateHead", 4)?; + assert_eq!(template(&el)?, (Some(units("a")), units("a"))); + let el = expect("``", DIV, true, "NoSubstitutionTemplate", 2)?; + assert_eq!(template(&el)?, (Some(units("")), units(""))); + let el = expect("`$`", DIV, true, "NoSubstitutionTemplate", 3)?; + assert_eq!(template(&el)?, (Some(units("$")), units("$"))); + let el = expect("`a\\n`", DIV, true, "NoSubstitutionTemplate", 5)?; + assert_eq!(template(&el)?, (Some(units("a\n")), units("a\\n"))); + let el = expect("`\r\n\r`", DIV, true, "NoSubstitutionTemplate", 5)?; + assert_eq!(template(&el)?, (Some(units("\n\n")), units("\n\n"))); + let el = expect("`\\\r\n`", DIV, true, "NoSubstitutionTemplate", 5)?; + assert_eq!(template(&el)?, (Some(units("")), units("\\\n"))); + let el = expect("`\\u{1F600}`", DIV, true, "NoSubstitutionTemplate", 11)?; + assert_eq!( + template(&el)?, + (Some(units("\u{1F600}")), units("\\u{1F600}")) + ); + // NotEscapeSequence: the TV is undefined, the TRV is the source. + for (s, raw) in [ + ("`\\unicode`", "\\unicode"), + ("`\\u{`", "\\u{"), + ("`\\u{}`", "\\u{}"), + ("`\\u{110000}`", "\\u{110000}"), + ("`\\u{12`", "\\u{12"), + ("`\\u12`", "\\u12"), + ("`\\x`", "\\x"), + ("`\\x1`", "\\x1"), + ("`\\xg`", "\\xg"), + ("`\\1`", "\\1"), + ("`\\01`", "\\01"), + ("`\\8`", "\\8"), + ] { + let el = expect(s, DIV, true, "NoSubstitutionTemplate", cps(s).len())?; + assert_eq!(template(&el)?, (None, units(raw)), "{s:?}"); + } + let el = expect("`\\0`", DIV, true, "NoSubstitutionTemplate", 4)?; + assert_eq!(template(&el)?, (Some(units("\0")), units("\\0"))); + let el = expect("}a`", TAIL, true, "TemplateTail", 3)?; + assert_eq!(template(&el)?, (Some(units("a")), units("a"))); + expect("}a${", RE_TAIL, true, "TemplateMiddle", 4)?; + expect("}", DIV, true, "RightBracePunctuator", 1)?; + expect_err("}", TAIL, true, LexError::NoInputElement)?; + expect_err("`abc", DIV, true, LexError::NoInputElement)?; + expect_err("`\\", DIV, true, LexError::NoInputElement)?; + Ok(()) +} + +#[test] +fn regular_expressions() -> Result<(), String> { + let el = expect("/ab+c/gi.x", RE, true, "RegularExpressionLiteral", 8)?; + assert!(matches!(el.value, Value::RegExp { body_end: 5 })); + expect("/[/]/", RE, true, "RegularExpressionLiteral", 5)?; + expect("/[\\]/]/", RE, true, "RegularExpressionLiteral", 7)?; + expect("/a\\/b/", RE, true, "RegularExpressionLiteral", 6)?; + expect("/=/", RE, true, "RegularExpressionLiteral", 3)?; + expect("/=/", DIV, true, "DivPunctuator", 2)?; + expect("/a/", START, true, "RegularExpressionLiteral", 3)?; + expect("/a/", RE_TAIL, true, "RegularExpressionLiteral", 3)?; + expect("/a/\\u0067", RE, true, "RegularExpressionLiteral", 3)?; + expect("/a/ü", RE, true, "RegularExpressionLiteral", 4)?; + for s in ["/a", "/a\n/", "/[a/", "/a\\\n/", "/*a/", "/"] { + expect_err(s, RE, true, LexError::NoInputElement)?; + } + Ok(()) +} + +/// The tokens of `src` in `goal` from 0, as (start, newline_before), and +/// the final `Eof`'s flag. +fn tokens(src: &[u32], goal: Goal) -> Result<(Vec<(usize, bool)>, bool), String> { + let mut pos = 0; + let mut seen = Vec::new(); + loop { + match next_token(src, pos, goal, true) { + Ok(Next::Token(tok)) => { + assert!(tok.element.start >= pos); + seen.push((tok.element.start, tok.newline_before)); + pos = tok.element.end; + } + Ok(Next::Eof { newline_before }) => return Ok((seen, newline_before)), + Err(e) => return Err(format!("at {pos}: {}", error_name(e))), + } + } +} + +#[test] +fn tokens_and_line_terminators() -> Result<(), String> { + let src = cps("a /* x */ b\n/*\n*/c // d\n\u{2028}e"); + assert_eq!( + tokens(&src, DIV)?, + (vec![(0, false), (10, false), (17, true), (25, true)], false) + ); + assert_eq!(tokens(&cps("x /*\n*/"), DIV)?, (vec![(0, false)], true)); + assert_eq!(tokens(&cps(""), DIV)?, (vec![], false)); + assert!(matches!( + next_token(&cps(" @"), 0, DIV, true), + Err(LexError::NoInputElement) + )); + Ok(()) +} + +/// Every input of up to three code points from an alphabet that reaches +/// every branch's first code point gives a result, at every position, and +/// an element is a non-empty range at the position (what Verus proves, run +/// here as well so that Miri sees each branch execute). +#[test] +fn small_inputs_are_total() { + const ALPHABET: &[u32] = &[ + 0x09, + 0x0A, + 0x0D, + 0x20, + 0x21, + 0x22, + 0x23, + 0x24, + 0x27, + 0x2A, + 0x2B, + 0x2D, + 0x2E, + 0x2F, + 0x30, + 0x31, + 0x38, + 0x3C, + 0x3D, + 0x3E, + 0x3F, + 0x5B, + 0x5C, + 0x5D, + 0x5F, + 0x60, + 0x61, + 0x6E, + 0x75, + 0x78, + 0x7B, + 0x7D, + 0xE9, + 0x2028, + 0xFEFF, + 0x1_F600, + 0x11_0000, + u32::MAX, + ]; + const GOALS: [Goal; 5] = [DIV, RE, RE_TAIL, TAIL, START]; + // Under Miri a stride of the inputs (it interprets every step); the + // full set natively. + let stride = if cfg!(miri) { 97 } else { 1 }; + let mut inputs: Vec> = ALPHABET.iter().map(|&a| vec![a]).collect(); + for &a in ALPHABET { + for &b in ALPHABET { + inputs.push(vec![a, b]); + for &c in ALPHABET { + inputs.push(vec![a, b, c]); + } + } + } + let mut checked = 0; + for src in inputs.iter().step_by(stride) { + for goal in GOALS { + for html in [false, true] { + for pos in 0..=src.len() { + match next_input_element(src, pos, goal, html) { + Ok(Step::Eof) => assert_eq!(pos, src.len()), + Ok(Step::Element(el)) => { + assert!(el.start == pos && pos < el.end && el.end <= src.len()); + } + Err(_) => assert!(pos < src.len()), + } + if let Ok(Next::Token(tok)) = next_token(src, pos, goal, html) { + assert!(pos <= tok.element.start && tok.element.end <= src.len()); + } + } + } + } + checked += 1; + } + assert!(checked > 0); +} diff --git a/crates/clazel-js-lexer/tests/test262.rs b/crates/clazel-js-lexer/tests/test262.rs new file mode 100644 index 000000000..9e6fe5b3d --- /dev/null +++ b/crates/clazel-js-lexer/tests/test262.rs @@ -0,0 +1,231 @@ +//! The lexer over the vendored Test262 subset (`tests/data/test262/`, its +//! table `tests/generated/test262.rs`). +//! +//! This is not yet Test262's conformance run: that goes through the parser +//! (phase-7.md 7.3), which chooses each token's goal symbol. Without a +//! parser the goal is guessed from the previous token (below), so these +//! tests check what does not depend on the guess: +//! +//! - **Totality and span coverage**, on every file: from every position the +//! lexer reaches, `next_input_element` returns, and the elements tile the +//! file (after an error it resumes one code point on). +//! - **No spurious error**: every test that must parse (no `negative:`, not +//! an expected failure) lexes to the end with the guessed goals. (Were +//! the guess wrong for one, a parser's goal would differ; none is.) + +use clazel_js_lexer::api::{Element, Next, Step, Value}; +use clazel_js_lexer::exec::{next_input_element, next_token}; +use clazel_js_lexer::spec::{ElementKind, Goal, TemplateKind}; +use std::collections::BTreeSet; +use std::path::PathBuf; + +/// A test's row in the generated table. +struct Case { + path: &'static str, + flags: &'static [&'static str], + features: &'static [&'static str], + includes: &'static [&'static str], + negative: Option<(&'static str, &'static str)>, + expect: Expect, +} + +/// Whether a test is expected to pass, and if not why. +enum Expect { + Run, + Fail(&'static str), +} + +include!("generated/test262.rs"); + +fn root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data/test262/test") +} + +fn read(case: &Case) -> Result, String> { + let path = root().join(case.path); + let text = std::fs::read_to_string(&path).map_err(|e| format!("{}: {e}", case.path))?; + Ok(text.chars().map(u32::from).collect()) +} + +/// Annex B.1.1 applies to scripts, not modules. +fn html(case: &Case) -> bool { + !case.flags.contains(&"module") +} + +/// Every element from 0 in `goal`, resuming one code point after an error: +/// the elements and the skipped code points tile the source text. +fn tile(src: &[u32], goal: Goal, html: bool) { + let mut pos: usize = 0; + loop { + match next_input_element(src, pos, goal, html) { + Ok(Step::Eof) => { + assert_eq!(pos, src.len()); + return; + } + Ok(Step::Element(el)) => { + assert!(el.start == pos && pos < el.end && el.end <= src.len()); + pos = el.end; + } + Err(_) => { + assert!(pos < src.len()); + pos = pos.saturating_add(1); + } + } + } +} + +/// The names after which a `/` starts a regular expression. +const BEFORE_EXPRESSION: &[&str] = &[ + "return", + "typeof", + "instanceof", + "in", + "of", + "new", + "delete", + "void", + "throw", + "case", + "do", + "else", + "yield", + "await", + "extends", +]; + +/// Whether a `/` after `el` starts a regular expression, as a guess. +fn regexp_after(el: &Element, src: &[u32]) -> bool { + let text: String = src + .get(el.start..el.end) + .unwrap_or(&[]) + .iter() + .filter_map(|&c| char::from_u32(c)) + .collect(); + match el.kind { + ElementKind::Punctuator => !matches!(text.as_str(), ")" | "]"), + ElementKind::RightBracePunctuator + | ElementKind::DivPunctuator + | ElementKind::Template(TemplateKind::Head | TemplateKind::Middle) => true, + ElementKind::IdentifierName => match &el.value { + Value::Name { + escaped: false, + code_points, + } => { + let name: String = code_points + .iter() + .filter_map(|&c| char::from_u32(c)) + .collect(); + BEFORE_EXPRESSION.contains(&name.as_str()) + } + _ => false, + }, + _ => false, + } +} + +/// Lexes `src` to the end with `next_token`, guessing each goal symbol from +/// the token before (and a stack of open braces, to know when a `}` closes +/// a template substitution). Returns the error's position, if any. +fn lex_tokens(src: &[u32], html: bool) -> Result<(), usize> { + let mut pos = 0; + let mut regexp = true; + let mut braces: Vec = Vec::new(); + loop { + let in_template = braces.last() == Some(&true); + let goal = if pos == 0 { + Goal::InputElementHashbangOrRegExp + } else { + match (regexp, in_template) { + (true, true) => Goal::InputElementRegExpOrTemplateTail, + (false, true) => Goal::InputElementTemplateTail, + (true, false) => Goal::InputElementRegExp, + (false, false) => Goal::InputElementDiv, + } + }; + match next_token(src, pos, goal, html) { + Ok(Next::Eof { .. }) => return Ok(()), + Ok(Next::Token(tok)) => { + let el = &tok.element; + assert!(pos <= el.start && el.start < el.end && el.end <= src.len()); + match el.kind { + ElementKind::Punctuator if src.get(el.start) == Some(&u32::from('{')) => { + braces.push(false); + } + ElementKind::Template(TemplateKind::Head) => braces.push(true), + ElementKind::RightBracePunctuator + | ElementKind::Template(TemplateKind::Tail) => { + braces.pop(); + } + _ => {} + } + regexp = regexp_after(el, src); + pos = el.end; + } + Err(_) => { + // The position of the token that failed: after its trivia. + let mut p = pos; + while let Ok(Step::Element(el)) = next_input_element(src, p, goal, html) { + p = el.end; + } + return Err(p); + } + } + } +} + +#[test] +#[cfg_attr(miri, ignore = "reads files")] +fn test262_lexes_total_and_tiled() -> Result<(), String> { + assert_eq!(TREE_SHA256.len(), 64); + let mut spurious = Vec::new(); + let mut rejected_negative = 0; + let mut negative = 0; + let mut features = BTreeSet::new(); + let mut with_includes = 0; + let mut expected_failures = 0; + let mut reasons = BTreeSet::new(); + for case in TEST262 { + let src = read(case)?; + let html = html(case); + features.extend(case.features.iter().copied()); + if !case.includes.is_empty() { + with_includes += 1; + } + for goal in [Goal::InputElementDiv, Goal::InputElementRegExp] { + tile(&src, goal, html); + } + let result = lex_tokens(&src, html); + if let Expect::Fail(reason) = case.expect { + reasons.insert(reason); + } + match (case.negative, &case.expect) { + (Some(_), _) => { + negative += 1; + if result.is_err() { + rejected_negative += 1; + } + } + (None, Expect::Run) => { + if let Err(at) = result { + spurious.push(format!("{} at {at}", case.path)); + } + } + (None, Expect::Fail(_)) => expected_failures += 1, + } + } + println!( + "{} tests ({} features, {with_includes} with includes); {negative} negative, \ + {rejected_negative} of them rejected by the lexer; {expected_failures} positive \ + expected failures skipped; {} reasons for expected failures", + TEST262.len(), + features.len(), + reasons.len() + ); + assert!( + spurious.is_empty(), + "{} tests that must parse do not lex:\n{}", + spurious.len(), + spurious.join("\n") + ); + Ok(()) +} From 8a94a28aaecd06f0d26c0c3a277c597d6fec8390 Mon Sep 17 00:00:00 2001 From: Alex Tiernan-Berry Date: Tue, 17 Nov 2026 11:59:03 +0000 Subject: [PATCH 4/5] Prove the JS lexer's simple kinds derive their range (7.2) next_input_element and next_token now also ensure kind_agrees_proven: a WhiteSpace, LineTerminator, HashbangComment, DivPunctuator or RightBracePunctuator element is derived by its kind in the goal (element_matches, the contract's own conjunct). The other kinds' derivation, the longest-element rule, the errors and the values remain unproven, as the module header lists. Co-Authored-By: Claude Opus 5.5 --- crates/clazel-js-lexer/src/exec.rs | 122 ++++++++++++++++++++++------- 1 file changed, 94 insertions(+), 28 deletions(-) diff --git a/crates/clazel-js-lexer/src/exec.rs b/crates/clazel-js-lexer/src/exec.rs index 721c95e10..d9c3c1e06 100644 --- a/crates/clazel-js-lexer/src/exec.rs +++ b/crates/clazel-js-lexer/src/exec.rs @@ -23,18 +23,24 @@ //! - **The value's shape:** [`value_shape_agrees`]: an element has the //! kind of `Value` its kind has (a `StringLiteral` a `Value::String`, and //! so on), and a regular expression's `body_end` is inside it. +//! - **The derivation of the simple kinds:** [`kind_agrees_proven`]: a +//! `WhiteSpace`, `LineTerminator`, `HashbangComment`, `DivPunctuator` or +//! `RightBracePunctuator` element is derived by its kind in the goal +//! ([`element_matches`]). //! - **Single code points:** the classifiers (`is_ws`, `is_lt`, the digit //! classes, `hex_val`) equal the spec's, and `push_utf16` is //! UTF16EncodeCodePoint (§11.1.1). //! -//! It is **not** proven that the results agree with [`super::spec`]: that -//! the kind derives the range, that no longer element exists -//! ([`longest_len`]), that the errors are [`lex_error`]'s, and that the -//! values are the spec's relations. These are the rest of -//! [`super::contract`], which the functions here do not yet claim, so they -//! do not implement [`super::api::JsLexer`]. Their agreement with the spec -//! is tested (`tests/`), not proven. [`lemma_contract_implies_proven`] -//! shows that what is proven is implied by the full contract. +//! It is **not** proven that the results agree with [`super::spec`] +//! otherwise: that the kind derives the range for the other kinds +//! (comments, names, punctuators, literals, templates, regular +//! expressions), that no longer element exists ([`longest_len`]), that the +//! errors are [`lex_error`]'s, that `newline_before` is [`skip_trivia`]'s, +//! and that the values are the spec's relations. These are the rest of +//! [`super::contract`], which the functions here do not claim, so they do +//! not implement [`super::api::JsLexer`]. Their agreement with the spec is +//! tested (`tests/`), not proven. [`lemma_contract_implies_proven`] shows +//! that what is proven is implied by the full contract. //! //! Non-ASCII `ID_Start` and `ID_Continue` come from [`super::unicode`], //! which the spec leaves uninterpreted: nothing is claimed about them. @@ -79,22 +85,53 @@ pub open spec fn value_shape_agrees(el: Element) -> bool { } } -/// What [`next_input_element`] is proven to return: totality and span -/// coverage. A weakening of [`super::contract::next_input_element_ensures`] +/// For the kinds whose derivation is proven, the kind derives the +/// element's range in the goal ([`element_matches`], one conjunct of +/// [`super::contract::next_input_element_ensures`]): `WhiteSpace`, +/// `LineTerminator`, `HashbangComment`, `DivPunctuator` and +/// `RightBracePunctuator`. For the other kinds nothing is claimed here. +pub open spec fn kind_agrees_proven(goal: Goal, html: bool, src: Seq, el: Element) -> bool { + (el.kind == ElementKind::WhiteSpace || el.kind == ElementKind::LineTerminator || el.kind + == ElementKind::HashbangComment || el.kind == ElementKind::DivPunctuator || el.kind + == ElementKind::RightBracePunctuator) ==> element_matches( + goal, + html, + el.kind, + src, + el.start as int, + el.end as int, + ) +} + +/// What [`next_input_element`] is proven to return: totality, span +/// coverage, the value's shape, and the derivation of the simple kinds. A +/// weakening of [`super::contract::next_input_element_ensures`] /// ([`lemma_contract_implies_proven`]). -pub open spec fn next_input_element_proven(src: Seq, pos: usize, r: Result) -> bool { +pub open spec fn next_input_element_proven( + src: Seq, + pos: usize, + goal: Goal, + html: bool, + r: Result, +) -> bool { match r { Err(_) => pos < src.len(), Ok(Step::Eof) => pos == src.len(), Ok(Step::Element(el)) => pos < src.len() && el.start == pos && pos < el.end && el.end - <= src.len() && value_shape_agrees(el), + <= src.len() && value_shape_agrees(el) && kind_agrees_proven(goal, html, src, el), } } /// What [`next_token`] is proven to return: the token is after `pos`, /// inside the source text and not trivia, and the flag is set only by a /// line terminator code point between `pos` and the token. -pub open spec fn next_token_proven(src: Seq, pos: usize, r: Result) -> bool { +pub open spec fn next_token_proven( + src: Seq, + pos: usize, + goal: Goal, + html: bool, + r: Result, +) -> bool { match r { Err(_) => pos < src.len(), Ok(Next::Eof { newline_before }) => newline_before ==> contains_line_terminator( @@ -103,18 +140,17 @@ pub open spec fn next_token_proven(src: Seq, pos: usize, r: Result pos <= tok.element.start < tok.element.end <= src.len() - && !is_trivia(tok.element.kind) && value_shape_agrees(tok.element) && ( - tok.newline_before ==> contains_line_terminator( - src, - pos as int, - tok.element.start as int, - )), + && !is_trivia(tok.element.kind) && value_shape_agrees(tok.element) + && kind_agrees_proven(goal, html, src, tok.element) && (tok.newline_before + ==> contains_line_terminator(src, pos as int, tok.element.start as int)), } } /// The full contract implies what is proven of [`next_input_element`] /// (except the value's shape, which the full contract states value by /// value): the proven part is a weakening, not a different claim. +/// ([`kind_agrees_proven`] is the contract's `element_matches` conjunct +/// itself, restricted to some kinds.) pub proof fn lemma_contract_implies_proven( src: Seq, pos: usize, @@ -130,7 +166,7 @@ pub proof fn lemma_contract_implies_proven( Err(_) => pos < src.len(), Ok(Step::Eof) => pos == src.len(), Ok(Step::Element(el)) => pos < src.len() && el.start == pos && pos < el.end && el.end - <= src.len(), + <= src.len() && kind_agrees_proven(goal, html, src, el), }, { } @@ -1477,6 +1513,17 @@ pub open spec fn candidate_ok(src: Seq, pos: usize, c: (Element, Option, + pos: usize, + c: (Element, Option), +) -> bool { + candidate_ok(src, pos, c) && kind_agrees_proven(goal, html, src, c.0) +} + /// The longer of two candidates (the first on a tie). fn longer(a: Option, b: Option) -> (r: Option) ensures @@ -1511,7 +1558,7 @@ fn lex_slash_comment(src: &[u32], pos: usize, html: bool) -> (r: Option candidate_ok(src@, pos, c), + Some(c) => candidate_ok(src@, pos, c) && c.0.kind == ElementKind::Comment, None => true, }, { @@ -1546,7 +1593,7 @@ fn lex_common(src: &[u32], pos: usize, goal: Goal, html: bool) -> (r: Option candidate_ok(src@, pos, c), + Some(c) => candidate_agrees(goal, html, src@, pos, c), None => true, }, { @@ -1554,6 +1601,7 @@ fn lex_common(src: &[u32], pos: usize, goal: Goal, html: bool) -> (r: Option (r: Option (r: Option (r: Option< pos < src.len(), ensures match r { - Some(c) => candidate_ok(src@, pos, c), + Some(c) => candidate_ok(src@, pos, c) && c.0.kind is Template, None => true, }, { @@ -1749,12 +1811,15 @@ fn lex_start(src: &[u32], html: bool) -> (r: Option) 0 < src.len(), ensures match r { - Some(c) => candidate_ok(src@, 0, c), + Some(c) => candidate_agrees(Goal::InputElementHashbangOrRegExp, html, src@, 0, c), None => true, }, { let hashbang = if peek(src, 0) == 0x23 && peek(src, 1) == 0x21 { + // §12.5 HashbangComment :: `#!` SingleLineCommentChars? let e = scan_line_chars(src, 2); + assert(src@.subrange(0, 2) =~= seq!['#', '!'].map_values(|c: char| c as u32)); + assert(hashbang_comment(src@, 0, e as int)); Some((plain(ElementKind::HashbangComment, 0, e), None)) } else { None @@ -1774,7 +1839,7 @@ fn html_close_candidate(src: &[u32], pos: usize, k: usize) -> (r: Option candidate_ok(src@, pos, c), + Some(c) => candidate_ok(src@, pos, c) && c.0.kind == ElementKind::Comment, None => true, }, { @@ -1788,7 +1853,8 @@ fn regexp_candidate(src: &[u32], pos: usize) -> (r: Option) pos < src.len(), ensures match r { - Some(c) => candidate_ok(src@, pos, c), + Some(c) => candidate_ok(src@, pos, c) && c.0.kind + == ElementKind::RegularExpressionLiteral, None => true, }, { @@ -1819,7 +1885,7 @@ pub fn next_input_element(src: &[u32], pos: usize, goal: Goal, html: bool) -> (r requires pos <= src.len(), ensures - next_input_element_proven(src@, pos, r), + next_input_element_proven(src@, pos, goal, html, r), { if pos >= src.len() { return Ok(Step::Eof); @@ -1882,7 +1948,7 @@ pub fn next_token(src: &[u32], pos: usize, goal: Goal, html: bool) -> (r: Result requires pos <= src.len(), ensures - next_token_proven(src@, pos, r), + next_token_proven(src@, pos, goal, html, r), { let mut p = pos; let mut newline = false; From 56f5b13351712ad901b36c7c41b757c331a42ee9 Mon Sep 17 00:00:00 2001 From: Alex Tiernan-Berry Date: Tue, 17 Nov 2026 12:02:00 +0000 Subject: [PATCH 5/5] Size the JS lexer tests' Miri strides to about a minute (7.2) Co-Authored-By: Claude Opus 5.5 --- crates/clazel-js-lexer/tests/lexer.rs | 2 +- crates/clazel-js-lexer/tests/unicode_tables.rs | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/clazel-js-lexer/tests/lexer.rs b/crates/clazel-js-lexer/tests/lexer.rs index 0e5775b8f..9dc873b1e 100644 --- a/crates/clazel-js-lexer/tests/lexer.rs +++ b/crates/clazel-js-lexer/tests/lexer.rs @@ -634,7 +634,7 @@ fn small_inputs_are_total() { const GOALS: [Goal; 5] = [DIV, RE, RE_TAIL, TAIL, START]; // Under Miri a stride of the inputs (it interprets every step); the // full set natively. - let stride = if cfg!(miri) { 97 } else { 1 }; + let stride = if cfg!(miri) { 331 } else { 1 }; let mut inputs: Vec> = ALPHABET.iter().map(|&a| vec![a]).collect(); for &a in ALPHABET { for &b in ALPHABET { diff --git a/crates/clazel-js-lexer/tests/unicode_tables.rs b/crates/clazel-js-lexer/tests/unicode_tables.rs index 3969f89f7..eae4018a9 100644 --- a/crates/clazel-js-lexer/tests/unicode_tables.rs +++ b/crates/clazel-js-lexer/tests/unicode_tables.rs @@ -30,12 +30,17 @@ fn tables_are_sorted_and_disjoint() { #[test] fn lookups_agree_with_the_tables() { let in_table = |table: &[Range], c: u32| table.iter().any(|r| r.first <= c && c <= r.last); - // Under Miri a stride of the code points (each scan is about 800 rows); - // natively a smaller stride than 1 would take minutes in a debug build, - // so every 7th code point plus every row's ends. - let stride = if cfg!(miri) { 40_009 } else { 7 }; + // Each linear scan reads about 800 rows, so natively every 7th code + // point (all of them would take minutes in a debug build) plus every + // row's ends; under Miri, which interprets each step, a wider stride of + // both. + let (stride, row_stride) = if cfg!(miri) { (40_009, 97) } else { (7, 1) }; let mut points: Vec = (0..=0x11_0000u32).step_by(stride).collect(); - for row in ID_START.iter().chain(ID_CONTINUE.iter()) { + for row in ID_START + .iter() + .chain(ID_CONTINUE.iter()) + .step_by(row_stride) + { points.extend([ row.first.saturating_sub(1), row.first,