diff --git a/crates/clazel-js-lexer/Cargo.toml b/crates/clazel-js-lexer/Cargo.toml index e2dd9eb22..02ff533b8 100644 --- a/crates/clazel-js-lexer/Cargo.toml +++ b/crates/clazel-js-lexer/Cargo.toml @@ -12,8 +12,10 @@ vstd = "=0.0.0-2026-10-04-0306" # Verification tier (docs/VERIFICATION.md). `tier` is what is proven about the # shipped code today; `target-tier` is what the policy requires, since this # lexes scripts a page chose (phase-7.md 7.0(a)). Tier 0: the spec and the -# contracts are written (item 7.2, first half, CLAUDE.md rule 3) and check -# with Verus, but no implementation exists, so nothing shipped is proven. +# contracts are written (item 7.2, first half, CLAUDE.md rule 3), and the +# implementation (src/exec.rs) is proven total, panic-free and to tile its +# input, but not to agree with the spec: it does not meet src/contract.rs +# yet, and there is no independent transcription or mutation set. [package.metadata.clazel] tier = 0 target-tier = 2 diff --git a/crates/clazel-js-lexer/src/exec.rs b/crates/clazel-js-lexer/src/exec.rs new file mode 100644 index 000000000..d9c3c1e06 --- /dev/null +++ b/crates/clazel-js-lexer/src/exec.rs @@ -0,0 +1,1982 @@ +//! The executable lexer (phase-7.md item 7.2, second half). +//! +//! Each `scan_` function reads one production of [`super::spec`] at a +//! position, as a loop with no recursion (a script is what a page chose; +//! Verus proves no panic, not no stack overflow), and returns where it +//! ends. [`next_input_element`] dispatches on the first code point, takes +//! the longest of the alternatives that can start there (§12), and then +//! applies the two checks the spec makes after the longest element is +//! chosen ([`lex_error`]); [`next_token`] repeats it over trivia. +//! +//! # What is proven, and what is not +//! +//! Verus proves, for every input, with no escape hatch: +//! +//! - **Totality:** every function terminates (each loop has a `decreases`) +//! and never panics: no index out of range, no overflow. +//! - **Span coverage:** [`next_input_element_proven`]: the end of the +//! source text gives `Eof`, and nothing else does; an element starts at +//! `pos`, ends after it and inside the source text; so elements from +//! repeated calls tile the source text. [`next_token_proven`]: the token +//! starts at or after `pos`, is not trivia, and its `newline_before` is +//! [`contains_line_terminator`] of some trivia element between them. +//! - **The value's shape:** [`value_shape_agrees`]: an element has the +//! kind of `Value` its kind has (a `StringLiteral` a `Value::String`, and +//! so on), and a regular expression's `body_end` is inside it. +//! - **The derivation of the simple kinds:** [`kind_agrees_proven`]: a +//! `WhiteSpace`, `LineTerminator`, `HashbangComment`, `DivPunctuator` or +//! `RightBracePunctuator` element is derived by its kind in the goal +//! ([`element_matches`]). +//! - **Single code points:** the classifiers (`is_ws`, `is_lt`, the digit +//! classes, `hex_val`) equal the spec's, and `push_utf16` is +//! UTF16EncodeCodePoint (§11.1.1). +//! +//! It is **not** proven that the results agree with [`super::spec`] +//! otherwise: that the kind derives the range for the other kinds +//! (comments, names, punctuators, literals, templates, regular +//! expressions), that no longer element exists ([`longest_len`]), that the +//! errors are [`lex_error`]'s, that `newline_before` is [`skip_trivia`]'s, +//! and that the values are the spec's relations. These are the rest of +//! [`super::contract`], which the functions here do not claim, so they do +//! not implement [`super::api::JsLexer`]. Their agreement with the spec is +//! tested (`tests/`), not proven. [`lemma_contract_implies_proven`] shows +//! that what is proven is implied by the full contract. +//! +//! Non-ASCII `ID_Start` and `ID_Continue` come from [`super::unicode`], +//! which the spec leaves uninterpreted: nothing is claimed about them. + +#[allow(unused_imports)] +use super::spec::*; +#[allow(unused_imports)] +use vstd::prelude::*; + +use super::api::{Element, Next, Step, Token, Value}; + +verus! { + +// --------------------------------------------------------------------- +// What the executable lexer is proven to satisfy. +// --------------------------------------------------------------------- + +/// The kind of [`Value`] an element has is the one its kind has (the shape +/// of [`super::contract::element_value_agrees`], not the values), and a +/// regular expression's closing `/` is inside it, after a non-empty body. +pub open spec fn value_shape_agrees(el: Element) -> bool { + match el.value { + Value::None => !(el.kind == ElementKind::IdentifierName + || el.kind == ElementKind::PrivateIdentifier || el.kind == ElementKind::StringLiteral + || el.kind == ElementKind::NumericLiteral + || el.kind == ElementKind::RegularExpressionLiteral + || el.kind == ElementKind::Template(TemplateKind::NoSubstitution) + || el.kind == ElementKind::Template(TemplateKind::Head) + || el.kind == ElementKind::Template(TemplateKind::Middle) + || el.kind == ElementKind::Template(TemplateKind::Tail)), + Value::Name { .. } => el.kind == ElementKind::IdentifierName || el.kind + == ElementKind::PrivateIdentifier, + Value::String { .. } => el.kind == ElementKind::StringLiteral, + Value::Template { .. } => el.kind == ElementKind::Template(TemplateKind::NoSubstitution) + || el.kind == ElementKind::Template(TemplateKind::Head) || el.kind + == ElementKind::Template(TemplateKind::Middle) || el.kind == ElementKind::Template( + TemplateKind::Tail, + ), + Value::Numeric { .. } => el.kind == ElementKind::NumericLiteral, + Value::RegExp { body_end } => el.kind == ElementKind::RegularExpressionLiteral + && el.start + 1 < body_end < el.end, + } +} + +/// For the kinds whose derivation is proven, the kind derives the +/// element's range in the goal ([`element_matches`], one conjunct of +/// [`super::contract::next_input_element_ensures`]): `WhiteSpace`, +/// `LineTerminator`, `HashbangComment`, `DivPunctuator` and +/// `RightBracePunctuator`. For the other kinds nothing is claimed here. +pub open spec fn kind_agrees_proven(goal: Goal, html: bool, src: Seq, el: Element) -> bool { + (el.kind == ElementKind::WhiteSpace || el.kind == ElementKind::LineTerminator || el.kind + == ElementKind::HashbangComment || el.kind == ElementKind::DivPunctuator || el.kind + == ElementKind::RightBracePunctuator) ==> element_matches( + goal, + html, + el.kind, + src, + el.start as int, + el.end as int, + ) +} + +/// What [`next_input_element`] is proven to return: totality, span +/// coverage, the value's shape, and the derivation of the simple kinds. A +/// weakening of [`super::contract::next_input_element_ensures`] +/// ([`lemma_contract_implies_proven`]). +pub open spec fn next_input_element_proven( + src: Seq, + pos: usize, + goal: Goal, + html: bool, + r: Result, +) -> bool { + match r { + Err(_) => pos < src.len(), + Ok(Step::Eof) => pos == src.len(), + Ok(Step::Element(el)) => pos < src.len() && el.start == pos && pos < el.end && el.end + <= src.len() && value_shape_agrees(el) && kind_agrees_proven(goal, html, src, el), + } +} + +/// What [`next_token`] is proven to return: the token is after `pos`, +/// inside the source text and not trivia, and the flag is set only by a +/// line terminator code point between `pos` and the token. +pub open spec fn next_token_proven( + src: Seq, + pos: usize, + goal: Goal, + html: bool, + r: Result, +) -> bool { + match r { + Err(_) => pos < src.len(), + Ok(Next::Eof { newline_before }) => newline_before ==> contains_line_terminator( + src, + pos as int, + src.len() as int, + ), + Ok(Next::Token(tok)) => pos <= tok.element.start < tok.element.end <= src.len() + && !is_trivia(tok.element.kind) && value_shape_agrees(tok.element) + && kind_agrees_proven(goal, html, src, tok.element) && (tok.newline_before + ==> contains_line_terminator(src, pos as int, tok.element.start as int)), + } +} + +/// The full contract implies what is proven of [`next_input_element`] +/// (except the value's shape, which the full contract states value by +/// value): the proven part is a weakening, not a different claim. +/// ([`kind_agrees_proven`] is the contract's `element_matches` conjunct +/// itself, restricted to some kinds.) +pub proof fn lemma_contract_implies_proven( + src: Seq, + pos: usize, + goal: Goal, + html: bool, + r: Result, +) + requires + pos <= src.len(), + super::contract::next_input_element_ensures(src, pos, goal, html, r), + ensures + match r { + Err(_) => pos < src.len(), + Ok(Step::Eof) => pos == src.len(), + Ok(Step::Element(el)) => pos < src.len() && el.start == pos && pos < el.end && el.end + <= src.len() && kind_agrees_proven(goal, html, src, el), + }, +{ +} + +// --------------------------------------------------------------------- +// Reading code points. +// --------------------------------------------------------------------- + +/// What [`peek`] returns past the end of the source text. It is above +/// U+10FFFF, so it is no `SourceCharacter` and matches no class below, +/// just as a lookahead fails at the end (§11.1, `at` in the spec). +pub const EOF: u32 = 0xFFFF_FFFF; + +/// The code point at `k`, or [`EOF`]. +pub fn peek(src: &[u32], k: usize) -> (c: u32) + ensures + k < src.len() ==> c == src@[k as int], + k >= src.len() ==> c == EOF, +{ + match src.get(k) { + Some(c) => *c, + None => EOF, + } +} + +/// §11.1 `SourceCharacter`. +fn is_src(c: u32) -> (r: bool) + ensures + r == is_source_character(c), +{ + c <= 0x10FFFF +} + +/// `low <= c <= high`. +fn between(c: u32, low: u32, high: u32) -> (r: bool) + ensures + r == (low <= c <= high), +{ + low <= c && c <= high +} + +/// §12.3 `LineTerminator`: , , , . +fn is_lt(c: u32) -> (r: bool) + ensures + r == is_line_terminator(c), +{ + c == 0x0A || c == 0x0D || c == 0x2028 || c == 0x2029 +} + +/// `SourceCharacter but not LineTerminator` (§12.4 `SingleLineCommentChar`, +/// §12.9.5 `RegularExpressionNonTerminator`). +fn is_line_char(c: u32) -> (r: bool) + ensures + r == is_not_line_terminator(c), +{ + is_src(c) && !is_lt(c) +} + +/// §12.2 `WhiteSpace`: , , , , . +fn is_ws(c: u32) -> (r: bool) + ensures + r == (c == 0x0009 || c == 0x000B || c == 0x000C || c == 0xFEFF + || unicode_space_separator(c)), +{ + c == 0x0009 || c == 0x000B || c == 0x000C || c == 0xFEFF || c == 0x0020 || c == 0x00A0 || c + == 0x1680 || between(c, 0x2000, 0x200A) || c == 0x202F || c == 0x205F || c == 0x3000 +} + +/// §12.9.3 `DecimalDigit`. +fn is_dec(c: u32) -> (r: bool) + ensures + r == is_decimal_digit(c), +{ + between(c, 0x30, 0x39) +} + +/// §12.9.3 `OctalDigit`. +fn is_oct(c: u32) -> (r: bool) + ensures + r == is_octal_digit(c), +{ + between(c, 0x30, 0x37) +} + +/// §12.9.3 `HexDigit`. +fn is_hex(c: u32) -> (r: bool) + ensures + r == is_hex_digit(c), +{ + between(c, 0x30, 0x39) || between(c, 0x61, 0x66) || between(c, 0x41, 0x46) +} + +/// `BinaryDigit`, `OctalDigit`, `DecimalDigit` or `HexDigit` by radix. +fn is_radix(radix: u32, c: u32) -> (r: bool) + requires + radix == 2 || radix == 8 || radix == 10 || radix == 16, + ensures + r == is_radix_digit(radix as int, c), +{ + if radix == 2 { + c == 0x30 || c == 0x31 + } else if radix == 8 { + is_oct(c) + } else if radix == 10 { + is_dec(c) + } else { + is_hex(c) + } +} + +/// The MV of a `HexDigit` (§12.9.3.2). +fn hex_val(c: u32) -> (r: u32) + requires + is_hex_digit(c), + ensures + r as int == hex_digit_value(c), + r < 16, +{ + if between(c, 0x30, 0x39) { + c.saturating_sub(0x30) + } else if between(c, 0x61, 0x66) { + c.saturating_sub(0x61).saturating_add(10) + } else { + c.saturating_sub(0x41).saturating_add(10) + } +} + +/// §12.7 `IdentifierStartChar :: UnicodeIDStart | $ | _`. Above U+007F, +/// [`super::unicode`]'s table, about which nothing is claimed. +fn id_start_char(c: u32) -> (r: bool) + ensures + c < 0x80 ==> r == is_identifier_start_char(c), + r ==> c <= 0x10FFFF, + r ==> c != 0x5C, +{ + if c < 0x80 { + between(c, 0x61, 0x7A) || between(c, 0x41, 0x5A) || c == 0x24 || c == 0x5F + } else { + super::unicode::id_start_non_ascii(c) + } +} + +/// §12.7 `IdentifierPartChar :: UnicodeIDContinue | $`. +fn id_part_char(c: u32) -> (r: bool) + ensures + c < 0x80 ==> r == is_identifier_part_char(c), + r ==> c <= 0x10FFFF, + r ==> c != 0x5C, +{ + if c < 0x80 { + between(c, 0x61, 0x7A) || between(c, 0x41, 0x5A) || between(c, 0x30, 0x39) || c + == 0x24 || c == 0x5F + } else { + super::unicode::id_continue_non_ascii(c) + } +} + +/// Appends UTF16EncodeCodePoint(`cp`) (§11.1.1). +fn push_utf16(out: &mut Vec, cp: u32) + requires + cp <= 0x10FFFF, + ensures + final(out)@ == old(out)@ + utf16_encode_code_point(cp as int), +{ + if cp <= 0xFFFF { + // 2. "If cp ≤ 0xFFFF, return the String value consisting of the + // code unit whose numeric value is cp." + out.push(cp as u16); + assert(final(out)@ =~= old(out)@ + utf16_encode_code_point(cp as int)); + } else { + let v = cp.saturating_sub(0x10000); + // 3. cu1 is floor((cp - 0x10000) / 0x400) + 0xD800. + let cu1 = (v / 0x400).saturating_add(0xD800) as u16; + // 4. cu2 is ((cp - 0x10000) modulo 0x400) + 0xDC00. + let cu2 = (v % 0x400).saturating_add(0xDC00) as u16; + // 5. The string-concatenation of cu1 and cu2. + out.push(cu1); + out.push(cu2); + assert(final(out)@ =~= old(out)@ + utf16_encode_code_point(cp as int)); + } +} + +/// Appends the UTF-16 of `src[i..j)`, each code point a `SourceCharacter`. +fn push_range_utf16(out: &mut Vec, src: &[u32], i: usize, j: usize) + requires + i <= j <= src.len(), +{ + let mut k = i; + while k < j + invariant + i <= k <= j <= src.len(), + decreases j - k, + { + let c = peek(src, k); + if c <= 0x10FFFF { + push_utf16(out, c); + } + k = k.saturating_add(1); + } +} + +// --------------------------------------------------------------------- +// §12.4 Comments, §12.5 Hashbang Comments, Annex B.1.1 +// --------------------------------------------------------------------- + +/// §12.4 `SingleLineCommentChars?` from `k`: the first index at or after +/// `k` that is not a `SourceCharacter` other than a `LineTerminator`. +fn scan_line_chars(src: &[u32], k: usize) -> (r: usize) + requires + k <= src.len(), + ensures + k <= r <= src.len(), + all_of(src@, k as int, r as int, |c: u32| is_not_line_terminator(c)), + r < src.len() ==> !is_not_line_terminator(src@[r as int]), +{ + let mut j = k; + loop + invariant + k <= j <= src.len(), + all_of(src@, k as int, j as int, |c: u32| is_not_line_terminator(c)), + decreases src.len() - j, + { + let c = peek(src, j); + if is_line_char(c) { + j = j.saturating_add(1); + } else { + return j; + } + } +} + +/// §12.4 `MultiLineComment` (or, with `!multi_line`, Annex B's +/// `SingleLineDelimitedComment`) after its `/*` at `k`: the end of the +/// first `*/` and whether a `LineTerminator` came before it. `None` if the +/// source text ends, or a code point that is not a `SourceCharacter` (or, +/// with `!multi_line`, a `LineTerminator`) comes, first. +fn scan_block_comment(src: &[u32], k: usize, multi_line: bool) -> (r: Option<(usize, bool)>) + requires + k <= src.len(), + ensures + match r { + Some((e, _)) => k + 2 <= e <= src.len(), + None => true, + }, +{ + let mut j = k; + let mut lt = false; + loop + invariant + k <= j <= src.len(), + decreases src.len() - j, + { + let c = peek(src, j); + if c == 0x2A && peek(src, j.saturating_add(1)) == 0x2F { + return Some((j.saturating_add(2), lt)); + } + if !is_src(c) || (!multi_line && is_lt(c)) { + return None; + } + if is_lt(c) { + lt = true; + } + j = j.saturating_add(1); + } +} + +/// Annex B.1.1 `WhiteSpaceSequence?` from `k`. +fn scan_ws(src: &[u32], k: usize) -> (r: usize) + requires + k <= src.len(), + ensures + k <= r <= src.len(), +{ + let mut j = k; + loop + invariant + k <= j <= src.len(), + decreases src.len() - j, + { + if is_ws(peek(src, j)) { + j = j.saturating_add(1); + } else { + return j; + } + } +} + +/// Annex B.1.1 the end of `-->` SingleLineCommentChars? at `k`. +fn scan_html_arrow(src: &[u32], k: usize) -> (r: Option) + requires + k <= src.len(), + ensures + match r { + Some(e) => k + 3 <= e <= src.len(), + None => true, + }, +{ + if peek(src, k) == 0x2D && peek(src, k.saturating_add(1)) == 0x2D && peek( + src, + k.saturating_add(2), + ) == 0x3E { + Some(scan_line_chars(src, k.saturating_add(3))) + } else { + None + } +} + +/// Annex B.1.1 `HTMLCloseComment :: WhiteSpaceSequence? +/// SingleLineDelimitedCommentSequence? --> SingleLineCommentChars?` at `k`. +/// Each part is determined by the code point that starts it (white space, +/// `/*` or `-->`), so the parse is the only one, and it ends where the +/// line does. +fn scan_html_close(src: &[u32], k: usize) -> (r: Option) + requires + k <= src.len(), + ensures + match r { + Some(e) => k + 3 <= e <= src.len(), + None => true, + }, +{ + let mut j = scan_ws(src, k); + loop + invariant + k <= j <= src.len(), + decreases src.len() - j, + { + if peek(src, j) == 0x2F && peek(src, j.saturating_add(1)) == 0x2A { + // SingleLineDelimitedComment WhiteSpaceSequence? + match scan_block_comment(src, j.saturating_add(2), false) { + Some((e, _)) => { + j = scan_ws(src, e); + }, + None => { + return None; + }, + } + } else { + return scan_html_arrow(src, j); + } + } +} + +/// The `LineTerminatorSequence` at `k` (§12.3): is one. +fn line_terminator_sequence_end(src: &[u32], k: usize) -> (r: usize) + requires + k < src.len(), + is_line_terminator(src@[k as int]), + ensures + k < r <= src.len(), +{ + if peek(src, k) == 0x0D && peek(src, k.saturating_add(1)) == 0x0A { + k.saturating_add(2) + } else { + k.saturating_add(1) + } +} + +// --------------------------------------------------------------------- +// §12.7 Names +// --------------------------------------------------------------------- + +/// The MV of the `HexDigits[~Sep]` that start at `k` (all of them), capped +/// at 0x110000 (any value above U+10FFFF is `NotCodePoint`), and the end. +fn scan_hex_digits(src: &[u32], k: usize) -> (r: (u32, usize)) + requires + k <= src.len(), + ensures + r.0 <= 0x110000, + k <= r.1 <= src.len(), +{ + let mut j = k; + let mut v: u32 = 0; + loop + invariant + k <= j <= src.len(), + v <= 0x110000, + decreases src.len() - j, + { + let c = peek(src, j); + if is_hex(c) { + let d = hex_val(c); + if v <= 0x10FFFF { + let w = v.saturating_mul(16).saturating_add(d); + v = if w > 0x10FFFF { + 0x110000 + } else { + w + }; + } + j = j.saturating_add(1); + } else { + return (v, j); + } + } +} + +/// §12.9.4 `Hex4Digits` at `k`: its MV. +fn scan_hex4(src: &[u32], k: usize) -> (r: Option) + requires + k <= src.len(), + ensures + match r { + Some(v) => k + 4 <= src.len() && v <= 0xFFFF, + None => true, + }, +{ + let a = peek(src, k); + let b = peek(src, k.saturating_add(1)); + let c = peek(src, k.saturating_add(2)); + let d = peek(src, k.saturating_add(3)); + if is_hex(a) && is_hex(b) && is_hex(c) && is_hex(d) { + let v = hex_val(a).saturating_mul(0x1000).saturating_add( + hex_val(b).saturating_mul(0x100), + ).saturating_add(hex_val(c).saturating_mul(0x10)).saturating_add(hex_val(d)); + Some(v) + } else { + None + } +} + +/// §12.9.4 `UnicodeEscapeSequence :: u Hex4Digits | u{ CodePoint }` at `k` +/// (after the backslash): its code point (§12.7.1.3 IdentifierCodePoint) +/// and its end. +fn scan_unicode_escape(src: &[u32], k: usize) -> (r: Option<(u32, usize)>) + requires + k <= src.len(), + ensures + match r { + Some((cp, e)) => k < e <= src.len() && cp <= 0x10FFFF, + None => true, + }, +{ + if peek(src, k) != 0x75 { + return None; + } + if peek(src, k.saturating_add(1)) == 0x7B { + // 2. `u{` CodePoint `}`, with CodePoint's MV at most 0x10FFFF. + let (v, e) = scan_hex_digits(src, k.saturating_add(2)); + if e > k.saturating_add(2) && v <= 0x10FFFF && peek(src, e) == 0x7D { + Some((v, e.saturating_add(1))) + } else { + None + } + } else { + // 1. `u` Hex4Digits + let v = scan_hex4(src, k.saturating_add(1))?; + Some((v, k.saturating_add(5))) + } +} + +/// §12.7 `IdentifierName` at `k`, greedily (each `IdentifierPart` is +/// determined by its first code point, so the longest derivation is the +/// greedy one): IdentifierCodePoints (§12.7.1.2), whether an escape was +/// used, whether the §12.7.1.1 early errors pass, and the end. +fn scan_identifier(src: &[u32], k: usize) -> (r: Option<(Vec, bool, bool, usize)>) + requires + k <= src.len(), + ensures + match r { + Some((_, _, _, e)) => k < e <= src.len(), + None => true, + }, +{ + let mut cps: Vec = Vec::new(); + let mut escaped = false; + let mut valid = true; + let c = peek(src, k); + let mut j: usize; + if c == 0x5C { + // IdentifierStart :: \ UnicodeEscapeSequence; §12.7.1.1: "It is a + // Syntax Error if the IdentifierCodePoint of UnicodeEscapeSequence + // is not some Unicode code point matched by the IdentifierStartChar + // lexical grammar production." + match scan_unicode_escape(src, k.saturating_add(1)) { + Some((cp, e)) => { + cps.push(cp); + escaped = true; + valid = id_start_char(cp); + j = e; + }, + None => { + return None; + }, + } + } else if id_start_char(c) { + // IdentifierStart :: IdentifierStartChar + cps.push(c); + j = k.saturating_add(1); + } else { + return None; + } + loop + invariant + k < j <= src.len(), + decreases src.len() - j, + { + let c = peek(src, j); + if c == 0x5C { + // IdentifierPart :: \ UnicodeEscapeSequence, with §12.7.1.1's + // rule for IdentifierPartChar. + match scan_unicode_escape(src, j.saturating_add(1)) { + Some((cp, e)) => { + cps.push(cp); + escaped = true; + if !id_part_char(cp) { + valid = false; + } + j = e; + }, + None => { + return Some((cps, escaped, valid, j)); + }, + } + } else if id_part_char(c) { + // IdentifierPart :: IdentifierPartChar + cps.push(c); + j = j.saturating_add(1); + } else { + return Some((cps, escaped, valid, j)); + } + } +} + +// --------------------------------------------------------------------- +// §12.8 Punctuators +// --------------------------------------------------------------------- + +/// §12.8 `Punctuator` (`OptionalChainingPunctuator` and `OtherPunctuator`) +/// at `k`: the end of the longest one, or `k` if none starts there. +fn scan_punctuator(src: &[u32], k: usize) -> (r: usize) + requires + k < src.len(), + ensures + k <= r <= src.len(), +{ + let c0 = peek(src, k); + let c1 = peek(src, k.saturating_add(1)); + let c2 = peek(src, k.saturating_add(2)); + let c3 = peek(src, k.saturating_add(3)); + let n: usize = if c0 == 0x7B || c0 == 0x28 || c0 == 0x29 || c0 == 0x5B || c0 == 0x5D || c0 + == 0x3B || c0 == 0x2C || c0 == 0x7E || c0 == 0x3A { + // { ( ) [ ] ; , ~ : + 1 + } else if c0 == 0x2E { + // ... . + if c1 == 0x2E && c2 == 0x2E { + 3 + } else { + 1 + } + } else if c0 == 0x3C { + // <<= << <= < + if c1 == 0x3C && c2 == 0x3D { + 3 + } else if c1 == 0x3C || c1 == 0x3D { + 2 + } else { + 1 + } + } else if c0 == 0x3E { + // >>>= >>> >>= >> >= > + if c1 == 0x3E && c2 == 0x3E && c3 == 0x3D { + 4 + } else if c1 == 0x3E && (c2 == 0x3E || c2 == 0x3D) { + 3 + } else if c1 == 0x3E || c1 == 0x3D { + 2 + } else { + 1 + } + } else if c0 == 0x3D { + // === == => = + if c1 == 0x3D && c2 == 0x3D { + 3 + } else if c1 == 0x3D || c1 == 0x3E { + 2 + } else { + 1 + } + } else if c0 == 0x21 { + // !== != ! + if c1 == 0x3D && c2 == 0x3D { + 3 + } else if c1 == 0x3D { + 2 + } else { + 1 + } + } else if c0 == 0x2B || c0 == 0x2D { + // ++ += +, -- -= - + if c1 == c0 || c1 == 0x3D { + 2 + } else { + 1 + } + } else if c0 == 0x2A { + // **= ** *= * + if c1 == 0x2A && c2 == 0x3D { + 3 + } else if c1 == 0x2A || c1 == 0x3D { + 2 + } else { + 1 + } + } else if c0 == 0x25 || c0 == 0x5E { + // %= %, ^= ^ + if c1 == 0x3D { + 2 + } else { + 1 + } + } else if c0 == 0x26 || c0 == 0x7C { + // &&= && &= &, ||= || |= | + if c1 == c0 && c2 == 0x3D { + 3 + } else if c1 == c0 || c1 == 0x3D { + 2 + } else { + 1 + } + } else if c0 == 0x3F { + // ??= ?? ?. ?, with OptionalChainingPunctuator :: ?. + // [lookahead ∉ DecimalDigit] + if c1 == 0x3F && c2 == 0x3D { + 3 + } else if c1 == 0x3F || (c1 == 0x2E && !is_dec(c2)) { + 2 + } else { + 1 + } + } else { + 0 + }; + k.saturating_add(n) +} + +// --------------------------------------------------------------------- +// §12.9.3 Numeric Literals +// --------------------------------------------------------------------- + +/// §12.9.3 `Digits[+Sep]` of a radix, from a digit at `k`: digits, each +/// pair of them possibly separated by one `_`. Greedy, since a `_` is +/// taken only when a digit follows it. +fn scan_digits_sep(src: &[u32], k: usize, radix: u32) -> (r: usize) + requires + k < src.len(), + radix == 2 || radix == 8 || radix == 10 || radix == 16, + ensures + k < r <= src.len(), +{ + let mut j = k.saturating_add(1); + loop + invariant + k < j <= src.len(), + radix == 2 || radix == 8 || radix == 10 || radix == 16, + decreases src.len() - j, + { + let c = peek(src, j); + if is_radix(radix, c) { + j = j.saturating_add(1); + } else if c == 0x5F && is_radix(radix, peek(src, j.saturating_add(1))) { + j = j.saturating_add(2); + } else { + return j; + } + } +} + +/// §12.9.3 `ExponentPart[+Sep]?` at `k`: its end, or `k`. +fn scan_exponent(src: &[u32], k: usize) -> (r: usize) + requires + k <= src.len(), + ensures + k <= r <= src.len(), +{ + let c = peek(src, k); + if c == 0x65 || c == 0x45 { + let s = peek(src, k.saturating_add(1)); + let m = if s == 0x2B || s == 0x2D { + k.saturating_add(2) + } else { + k.saturating_add(1) + }; + if is_dec(peek(src, m)) { + scan_digits_sep(src, m, 10) + } else { + k + } + } else { + k + } +} + +/// The rest of a §12.9.3 `DecimalLiteral` after its `DecimalIntegerLiteral` +/// ends at `k`: `. DecimalDigits[+Sep]? ExponentPart[+Sep]?` or +/// `ExponentPart[+Sep]?`. +fn scan_decimal_tail(src: &[u32], k: usize) -> (r: usize) + requires + k <= src.len(), + ensures + k <= r <= src.len(), +{ + if peek(src, k) == 0x2E { + let m = k.saturating_add(1); + let n = if is_dec(peek(src, m)) { + scan_digits_sep(src, m, 10) + } else { + m + }; + scan_exponent(src, n) + } else { + scan_exponent(src, k) + } +} + +/// §12.9.3 `NumericLiteral` at `k`, the longest: its end, whether it is a +/// BigInt literal, and whether it is legacy (§12.9.3.1). +fn scan_numeric(src: &[u32], k: usize) -> (r: Option<(usize, bool, bool)>) + requires + k < src.len(), + ensures + match r { + Some((e, _, _)) => k < e <= src.len(), + None => true, + }, +{ + let c0 = peek(src, k); + let c1 = peek(src, k.saturating_add(1)); + if c0 == 0x2E { + // DecimalLiteral :: . DecimalDigits[+Sep] ExponentPart[+Sep]? + if is_dec(c1) { + let e = scan_digits_sep(src, k.saturating_add(1), 10); + Some((scan_exponent(src, e), false, false)) + } else { + None + } + } else if c0 == 0x30 { + let c2 = peek(src, k.saturating_add(2)); + let radix: u32 = if c1 == 0x62 || c1 == 0x42 { + 2 + } else if c1 == 0x6F || c1 == 0x4F { + 8 + } else if c1 == 0x78 || c1 == 0x58 { + 16 + } else { + 0 + }; + if radix != 0 && is_radix(radix, c2) { + // NonDecimalIntegerLiteral[+Sep] BigIntLiteralSuffix? + let e = scan_digits_sep(src, k.saturating_add(2), radix); + if peek(src, e) == 0x6E { + Some((e.saturating_add(1), true, false)) + } else { + Some((e, false, false)) + } + } else if c1 == 0x6E { + // DecimalBigIntegerLiteral :: 0 BigIntLiteralSuffix + Some((k.saturating_add(2), true, false)) + } else if is_dec(c1) { + // `0` and decimal digits, with no separator: a + // LegacyOctalIntegerLiteral if they are all octal, else a + // NonOctalDecimalIntegerLiteral, which a DecimalLiteral may + // continue. + let mut j = k.saturating_add(1); + let mut octal = true; + loop + invariant + k < j <= src.len(), + decreases src.len() - j, + { + let c = peek(src, j); + if is_dec(c) { + if !is_oct(c) { + octal = false; + } + j = j.saturating_add(1); + } else { + break; + } + } + if octal { + Some((j, false, true)) + } else { + Some((scan_decimal_tail(src, j), false, true)) + } + } else { + // DecimalIntegerLiteral :: 0 + Some((scan_decimal_tail(src, k.saturating_add(1)), false, false)) + } + } else if is_dec(c0) { + // DecimalIntegerLiteral :: NonZeroDigit NumericLiteralSeparator? + // DecimalDigits[+Sep], or a DecimalBigIntegerLiteral. + let e = scan_digits_sep(src, k, 10); + if peek(src, e) == 0x6E { + Some((e.saturating_add(1), true, false)) + } else { + Some((scan_decimal_tail(src, e), false, false)) + } + } else { + None + } +} + +/// §12.9.3: "The SourceCharacter immediately following a NumericLiteral +/// must not be an IdentifierStart or DecimalDigit." +fn numeric_follow_error(src: &[u32], j: usize) -> (r: bool) + requires + j <= src.len(), +{ + let c = peek(src, j); + if is_dec(c) || id_start_char(c) { + true + } else if c == 0x5C { + scan_unicode_escape(src, j.saturating_add(1)).is_some() + } else { + false + } +} + +// --------------------------------------------------------------------- +// §12.9.4 String Literals +// --------------------------------------------------------------------- + +/// §12.9.4 `LegacyOctalEscapeSequence` at `k`, whose first digit `d` is an +/// octal digit: its MV, pushed as one code unit, and its end. The +/// lookaheads make it the longest run its first digit allows: three digits +/// after `0` to `3`, two after `4` to `7`. +fn scan_legacy_octal_escape(src: &[u32], k: usize, units: &mut Vec) -> (r: usize) + requires + k < src.len(), + is_octal_digit(src@[k as int]), + ensures + k < r <= src.len(), +{ + let d = peek(src, k); + let mut v: u32 = d.saturating_sub(0x30); + let mut e = k.saturating_add(1); + let c1 = peek(src, e); + if is_oct(c1) { + v = v.saturating_mul(8).saturating_add(c1.saturating_sub(0x30)); + e = e.saturating_add(1); + let c2 = peek(src, e); + if d <= 0x33 && is_oct(c2) { + v = v.saturating_mul(8).saturating_add(c2.saturating_sub(0x30)); + e = e.saturating_add(1); + } + } + units.push(v as u16); + e +} + +/// §12.9.4 `\ EscapeSequence` or `LineContinuation`, the backslash at +/// `k`: appends the SV (§12.9.4.2) and returns the end. +fn scan_string_escape(src: &[u32], k: usize, units: &mut Vec) -> (r: Option) + requires + k < src.len(), + ensures + match r { + Some(e) => k + 1 < e <= src.len(), + None => true, + }, +{ + let m = k.saturating_add(1); + let d = peek(src, m); + if is_lt(d) { + // LineContinuation: the empty String. + Some(line_terminator_sequence_end(src, m)) + } else if d == 0x27 || d == 0x22 || d == 0x5C { + // SingleEscapeCharacter ' " \: the character itself. + units.push(d as u16); + Some(m.saturating_add(1)) + } else if d == 0x62 || d == 0x66 || d == 0x6E || d == 0x72 || d == 0x74 || d == 0x76 { + // SingleEscapeCharacter b f n r t v (Table "String Single Character + // Escape Sequences"). + let u: u16 = if d == 0x62 { + 0x08 + } else if d == 0x74 { + 0x09 + } else if d == 0x6E { + 0x0A + } else if d == 0x76 { + 0x0B + } else if d == 0x66 { + 0x0C + } else { + 0x0D + }; + units.push(u); + Some(m.saturating_add(1)) + } else if d == 0x30 && !is_dec(peek(src, m.saturating_add(1))) { + // `0` [lookahead ∉ DecimalDigit]: 0x0000. + units.push(0); + Some(m.saturating_add(1)) + } else if d == 0x30 && !is_oct(peek(src, m.saturating_add(1))) { + // LegacyOctalEscapeSequence :: `0` [lookahead ∈ { 8, 9 }]: 0. + units.push(0); + Some(m.saturating_add(1)) + } else if is_oct(d) { + // LegacyOctalEscapeSequence, the other alternatives. + Some(scan_legacy_octal_escape(src, m, units)) + } else if d == 0x38 || d == 0x39 { + // NonOctalDecimalEscapeSequence: 0x0038 or 0x0039. + units.push(d as u16); + Some(m.saturating_add(1)) + } else if d == 0x78 { + // HexEscapeSequence :: x HexDigit HexDigit + let a = peek(src, m.saturating_add(1)); + let b = peek(src, m.saturating_add(2)); + if is_hex(a) && is_hex(b) { + units.push(hex_val(a).saturating_mul(16).saturating_add(hex_val(b)) as u16); + Some(m.saturating_add(3)) + } else { + None + } + } else if d == 0x75 { + // UnicodeEscapeSequence: UTF16EncodeCodePoint of its MV. + match scan_unicode_escape(src, m) { + Some((cp, e)) => { + push_utf16(units, cp); + Some(e) + }, + None => None, + } + } else if is_src(d) { + // NonEscapeCharacter: the code point. + push_utf16(units, d); + Some(m.saturating_add(1)) + } else { + None + } +} + +/// §12.9.4 `StringLiteral` at `k` (a quote): its SV and its end. +fn scan_string(src: &[u32], k: usize) -> (r: Option<(Vec, usize)>) + requires + k < src.len(), + src@[k as int] == 0x22 || src@[k as int] == 0x27, + ensures + match r { + Some((_, e)) => k + 2 <= e <= src.len(), + None => true, + }, +{ + let quote = peek(src, k); + let mut units: Vec = Vec::new(); + let mut j = k.saturating_add(1); + loop + invariant + k < j <= src.len(), + quote == 0x22 || quote == 0x27, + decreases src.len() - j, + { + let c = peek(src, j); + if c == quote { + return Some((units, j.saturating_add(1))); + } else if c == 0x5C { + match scan_string_escape(src, j, &mut units) { + Some(e) => { + j = e; + }, + None => { + return None; + }, + } + } else if c == 0x2028 || c == 0x2029 { + // , : the code unit. + units.push(c as u16); + j = j.saturating_add(1); + } else if is_line_char(c) { + // SourceCharacter but not one of the quote or \ or + // LineTerminator. + push_utf16(&mut units, c); + j = j.saturating_add(1); + } else { + return None; + } + } +} + +// --------------------------------------------------------------------- +// §12.9.5 Regular Expression Literals +// --------------------------------------------------------------------- + +/// §12.9.5 `RegularExpressionClass` at `k` (a `[`): its end. +fn scan_regexp_class(src: &[u32], k: usize) -> (r: Option) + requires + k < src.len(), + ensures + match r { + Some(e) => k < e <= src.len(), + None => true, + }, +{ + let mut j = k.saturating_add(1); + loop + invariant + k < j <= src.len(), + decreases src.len() - j, + { + let c = peek(src, j); + if c == 0x5D { + return Some(j.saturating_add(1)); + } else if c == 0x5C { + // RegularExpressionBackslashSequence + if is_line_char(peek(src, j.saturating_add(1))) { + j = j.saturating_add(2); + } else { + return None; + } + } else if is_line_char(c) { + j = j.saturating_add(1); + } else { + return None; + } + } +} + +/// §12.9.5 `RegularExpressionLiteral` at `k` (a `/`): the index of the +/// closing `/` and the end of the flags. +fn scan_regexp(src: &[u32], k: usize) -> (r: Option<(usize, usize)>) + requires + k < src.len(), + ensures + match r { + Some((b, e)) => k + 1 < b < e <= src.len(), + None => true, + }, +{ + let first = peek(src, k.saturating_add(1)); + // RegularExpressionFirstChar is not `*` or `/`. + if first == 0x2A || first == 0x2F { + return None; + } + let mut j = k.saturating_add(1); + loop + invariant + k < j <= src.len(), + j == k + 1 ==> (k + 1 >= src.len() || src@[(k + 1) as int] != 0x2F), + decreases src.len() - j, + { + let c = peek(src, j); + if c == 0x2F { + // The closing `/`, then RegularExpressionFlags. + let mut f = j.saturating_add(1); + loop + invariant + k + 1 < j < f <= src.len(), + decreases src.len() - f, + { + if id_part_char(peek(src, f)) { + f = f.saturating_add(1); + } else { + return Some((j, f)); + } + } + } else if c == 0x5C { + if is_line_char(peek(src, j.saturating_add(1))) { + j = j.saturating_add(2); + } else { + return None; + } + } else if c == 0x5B { + match scan_regexp_class(src, j) { + Some(e) => { + j = e; + }, + None => { + return None; + }, + } + } else if is_line_char(c) { + j = j.saturating_add(1); + } else { + return None; + } + } +} + +// --------------------------------------------------------------------- +// §12.9.6 Template Literal Lexical Components +// --------------------------------------------------------------------- + +/// A template component's TV (`None` once a `NotEscapeSequence` makes it +/// *undefined*) and TRV, as they are built. +struct TemplateValues { + cooked: Vec, + defined: bool, + raw: Vec, +} + +/// §12.9.6 `\ TemplateEscapeSequence`, `\ NotEscapeSequence` or +/// `LineContinuation`, the backslash at `k`: appends the TV and TRV +/// (§12.9.6.1, §12.9.6.2) and returns the end. +fn scan_template_escape(src: &[u32], k: usize, values: &mut TemplateValues) -> (r: Option) + requires + k < src.len(), + ensures + match r { + Some(e) => k + 1 < e <= src.len(), + None => true, + }, +{ + let m = k.saturating_add(1); + let d = peek(src, m); + if is_lt(d) { + // LineContinuation: TV is empty; TRV is \ and the TRV of the + // LineTerminatorSequence ( and are ). + let e = line_terminator_sequence_end(src, m); + values.raw.push(0x5C); + if d == 0x2028 || d == 0x2029 { + values.raw.push(d as u16); + } else { + values.raw.push(0x0A); + } + return Some(e); + } + // In every other case the TRV is the UTF-16 of the matched code points. + let e: usize; + if d == 0x27 || d == 0x22 || d == 0x5C { + values.cooked.push(d as u16); + e = m.saturating_add(1); + } else if d == 0x62 || d == 0x66 || d == 0x6E || d == 0x72 || d == 0x74 || d == 0x76 { + let u: u16 = if d == 0x62 { + 0x08 + } else if d == 0x74 { + 0x09 + } else if d == 0x6E { + 0x0A + } else if d == 0x76 { + 0x0B + } else if d == 0x66 { + 0x0C + } else { + 0x0D + }; + values.cooked.push(u); + e = m.saturating_add(1); + } else if d == 0x30 { + if is_dec(peek(src, m.saturating_add(1))) { + // NotEscapeSequence :: 0 DecimalDigit + values.defined = false; + e = m.saturating_add(2); + } else { + // TemplateEscapeSequence :: 0 [lookahead ∉ DecimalDigit] + values.cooked.push(0); + e = m.saturating_add(1); + } + } else if is_dec(d) { + // NotEscapeSequence :: DecimalDigit but not 0 + values.defined = false; + e = m.saturating_add(1); + } else if d == 0x78 { + let a = peek(src, m.saturating_add(1)); + let b = peek(src, m.saturating_add(2)); + if is_hex(a) && is_hex(b) { + // HexEscapeSequence + values.cooked.push(hex_val(a).saturating_mul(16).saturating_add(hex_val(b)) as u16); + e = m.saturating_add(3); + } else if is_hex(a) { + // NotEscapeSequence :: x HexDigit [lookahead ∉ HexDigit] + values.defined = false; + e = m.saturating_add(2); + } else { + // NotEscapeSequence :: x [lookahead ∉ HexDigit] + values.defined = false; + e = m.saturating_add(1); + } + } else if d == 0x75 { + match scan_unicode_escape(src, m) { + Some((cp, end)) => { + // UnicodeEscapeSequence + push_utf16(&mut values.cooked, cp); + e = end; + }, + None => { + // NotEscapeSequence, alternatives 5 to 11: `u` and the hex + // digits that follow (up to three), or `u{` and all of + // them; each ends where no further hex digit follows. + values.defined = false; + if peek(src, m.saturating_add(1)) == 0x7B { + let (_, h) = scan_hex_digits(src, m.saturating_add(2)); + e = h; + } else { + let mut h = m.saturating_add(1); + while is_hex(peek(src, h)) && h < m.saturating_add(4) + invariant + m < h <= src.len(), + decreases src.len() - h, + { + h = h.saturating_add(1); + } + e = h; + } + }, + } + } else if is_src(d) { + // CharacterEscapeSequence :: NonEscapeCharacter + push_utf16(&mut values.cooked, d); + e = m.saturating_add(1); + } else { + return None; + } + push_range_utf16(&mut values.raw, src, k, e); + Some(e) +} + +/// What [`scan_template`] finds: whether the component closes with `` ` ``, +/// its TV (`None` for *undefined*), its TRV, and its end. +type TemplateScan = (bool, Option>, Vec, usize); + +/// §12.9.6 a template component at `k` (`` ` `` for `NoSubstitutionTemplate` +/// and `TemplateHead`, `}` for `TemplateMiddle` and `TemplateTail`): whether +/// it closes with `` ` `` (rather than `${`), its TV and TRV, and its end. +fn scan_template(src: &[u32], k: usize) -> (r: Option) + requires + k < src.len(), + ensures + match r { + Some((_, _, _, e)) => k + 2 <= e <= src.len(), + None => true, + }, +{ + let mut values = TemplateValues { cooked: Vec::new(), defined: true, raw: Vec::new() }; + let mut j = k.saturating_add(1); + loop + invariant + k < j <= src.len(), + decreases src.len() - j, + { + let c = peek(src, j); + if c == 0x60 || (c == 0x24 && peek(src, j.saturating_add(1)) == 0x7B) { + let closes = c == 0x60; + let end = if closes { + j.saturating_add(1) + } else { + j.saturating_add(2) + }; + let cooked = if values.defined { + Some(values.cooked) + } else { + None + }; + return Some((closes, cooked, values.raw, end)); + } else if c == 0x5C { + match scan_template_escape(src, j, &mut values) { + Some(e) => { + j = e; + }, + None => { + return None; + }, + } + } else if is_lt(c) { + // LineTerminatorSequence: TV and TRV are its TRV. + let e = line_terminator_sequence_end(src, j); + let u: u16 = if c == 0x2028 || c == 0x2029 { + c as u16 + } else { + 0x0A + }; + values.cooked.push(u); + values.raw.push(u); + j = e; + } else if is_src(c) { + // `$` [lookahead ≠ {], or SourceCharacter but not one of ` \ $ + // LineTerminator: UTF16EncodeCodePoint of it. + push_utf16(&mut values.cooked, c); + push_utf16(&mut values.raw, c); + j = j.saturating_add(1); + } else { + return None; + } + } +} + +// --------------------------------------------------------------------- +// §12 The goal symbols and the longest input element. +// --------------------------------------------------------------------- + +/// The goal has `RegularExpressionLiteral`. +fn goal_regexp(goal: Goal) -> (r: bool) + ensures + r == (goal == Goal::InputElementRegExp || goal == Goal::InputElementRegExpOrTemplateTail + || goal == Goal::InputElementHashbangOrRegExp), +{ + matches!( + goal, + Goal::InputElementRegExp | Goal::InputElementRegExpOrTemplateTail + | Goal::InputElementHashbangOrRegExp + ) +} + +/// The goal has `DivPunctuator`. +fn goal_div(goal: Goal) -> (r: bool) + ensures + r == (goal == Goal::InputElementDiv || goal == Goal::InputElementTemplateTail), +{ + matches!(goal, Goal::InputElementDiv | Goal::InputElementTemplateTail) +} + +/// The goal has `RightBracePunctuator`. +fn goal_right_brace(goal: Goal) -> (r: bool) + ensures + r == (goal == Goal::InputElementDiv || goal == Goal::InputElementRegExp), +{ + matches!(goal, Goal::InputElementDiv | Goal::InputElementRegExp) +} + +/// The goal has `TemplateSubstitutionTail`. +fn goal_template_tail(goal: Goal) -> (r: bool) + ensures + r == (goal == Goal::InputElementRegExpOrTemplateTail || goal + == Goal::InputElementTemplateTail), +{ + matches!(goal, Goal::InputElementRegExpOrTemplateTail | Goal::InputElementTemplateTail) +} + +/// An element with no value. +fn plain(kind: ElementKind, start: usize, end: usize) -> (r: Element) + requires + !(kind == ElementKind::IdentifierName || kind == ElementKind::PrivateIdentifier || kind + == ElementKind::StringLiteral || kind == ElementKind::NumericLiteral || kind + == ElementKind::RegularExpressionLiteral || kind == ElementKind::Template( + TemplateKind::NoSubstitution, + ) || kind == ElementKind::Template(TemplateKind::Head) || kind == ElementKind::Template( + TemplateKind::Middle, + ) || kind == ElementKind::Template(TemplateKind::Tail)), + ensures + r.kind == kind, + r.start == start, + r.end == end, + value_shape_agrees(r), +{ + Element { kind, start, end, value: Value::None } +} + +/// An element and the error that the checks after the longest element is +/// chosen raise for it, if any (§12.9.3, §12.7.1.1; [`lex_error`]). +type Candidate = (Element, Option); + +/// The candidate's element is a non-empty range at `pos`. +pub open spec fn candidate_ok(src: Seq, pos: usize, c: (Element, Option)) -> bool { + c.0.start == pos && pos < c.0.end <= src.len() && value_shape_agrees(c.0) +} + +/// [`candidate_ok`], and [`kind_agrees_proven`] in the goal. +pub open spec fn candidate_agrees( + goal: Goal, + html: bool, + src: Seq, + pos: usize, + c: (Element, Option), +) -> bool { + candidate_ok(src, pos, c) && kind_agrees_proven(goal, html, src, c.0) +} + +/// The longer of two candidates (the first on a tie). +fn longer(a: Option, b: Option) -> (r: Option) + ensures + match a { + Some(x) => match b { + Some(y) => r == Some(x) || r == Some(y), + None => r == Some(x), + }, + None => r == b, + }, +{ + match (a, b) { + (Some(x), Some(y)) => { + if y.0.end > x.0.end { + Some(y) + } else { + Some(x) + } + }, + (Some(x), None) => Some(x), + (None, y) => y, + } +} + +/// A comment that starts with `/` at `pos`: `//` (§12.4 +/// `SingleLineComment`) or `/*` (§12.4 `MultiLineComment`; with `html`, +/// Annex B.1.1's `SingleLineDelimitedComment`, or its `MultiLineComment` +/// with an optional `HTMLCloseComment` after the `*/` when the comment +/// holds a `LineTerminator`). +fn lex_slash_comment(src: &[u32], pos: usize, html: bool) -> (r: Option) + requires + pos < src.len(), + ensures + match r { + Some(c) => candidate_ok(src@, pos, c) && c.0.kind == ElementKind::Comment, + None => true, + }, +{ + let c1 = peek(src, pos.saturating_add(1)); + if c1 == 0x2F { + let e = scan_line_chars(src, pos.saturating_add(2)); + Some((plain(ElementKind::Comment, pos, e), None)) + } else if c1 == 0x2A { + match scan_block_comment(src, pos.saturating_add(2), true) { + Some((e, lt)) => { + let end = if html && lt { + match scan_html_close(src, e) { + Some(f) => f, + None => e, + } + } else { + e + }; + Some((plain(ElementKind::Comment, pos, end), None)) + }, + None => None, + } + } else { + None + } +} + +/// Every alternative of `goal` that can start with the code point at +/// `pos`, the longest of them. +fn lex_common(src: &[u32], pos: usize, goal: Goal, html: bool) -> (r: Option) + requires + pos < src.len(), + ensures + match r { + Some(c) => candidate_agrees(goal, html, src@, pos, c), + None => true, + }, +{ + let c = peek(src, pos); + let c1 = peek(src, pos.saturating_add(1)); + if is_ws(c) { + // WhiteSpace + assert(white_space(src@, pos as int, pos + 1)); + Some((plain(ElementKind::WhiteSpace, pos, pos.saturating_add(1)), None)) + } else if is_lt(c) { + // LineTerminator; with `html`, Annex B.1.1's + // SingleLineHTMLCloseComment :: LineTerminatorSequence + // HTMLCloseComment, which is longer when it matches. + let lt = Some((plain(ElementKind::LineTerminator, pos, pos.saturating_add(1)), None)); + if html { + let s = line_terminator_sequence_end(src, pos); + longer(lt, html_close_candidate(src, pos, s)) + } else { + lt + } + } else if c == 0x2F { + // Comment, RegularExpressionLiteral or DivPunctuator. + let comment = lex_slash_comment(src, pos, html); + let regexp = if goal_regexp(goal) { + regexp_candidate(src, pos) + } else { + None + }; + let div = if goal_div(goal) { + let e = if c1 == 0x3D { + // DivPunctuator :: `/=` + assert(src@.subrange(pos as int, pos + 2) =~= seq!['/', '='].map_values( + |c: char| c as u32, + )); + pos.saturating_add(2) + } else { + // DivPunctuator :: `/` + assert(src@.subrange(pos as int, pos + 1) =~= seq!['/'].map_values( + |c: char| c as u32, + )); + pos.saturating_add(1) + }; + assert(div_punctuator(src@, pos as int, e as int)); + Some((plain(ElementKind::DivPunctuator, pos, e), None)) + } else { + None + }; + longer(longer(comment, regexp), div) + } else if c == 0x7D { + // TemplateMiddle / TemplateTail or RightBracePunctuator. + if goal_template_tail(goal) { + lex_template(src, pos, true) + } else if goal_right_brace(goal) { + // RightBracePunctuator :: `}` + assert(src@.subrange(pos as int, pos + 1) =~= seq!['}'].map_values( + |c: char| c as u32, + )); + assert(right_brace_punctuator(src@, pos as int, pos + 1)); + Some((plain(ElementKind::RightBracePunctuator, pos, pos.saturating_add(1)), None)) + } else { + None + } + } else if c == 0x60 { + // NoSubstitutionTemplate / TemplateHead + lex_template(src, pos, false) + } else if c == 0x22 || c == 0x27 { + // StringLiteral + let (units, e) = scan_string(src, pos)?; + Some( + ( + Element { + kind: ElementKind::StringLiteral, + start: pos, + end: e, + value: Value::String { units }, + }, + None, + ), + ) + } else if c == 0x23 { + // PrivateIdentifier :: # IdentifierName + match scan_identifier(src, pos.saturating_add(1)) { + Some((cps, escaped, valid, e)) => { + let err = if valid { + None + } else { + Some(LexError::InvalidIdentifierEscape) + }; + Some( + ( + Element { + kind: ElementKind::PrivateIdentifier, + start: pos, + end: e, + value: Value::Name { code_points: cps, escaped }, + }, + err, + ), + ) + }, + None => None, + } + } else if is_dec(c) || (c == 0x2E && is_dec(c1)) { + // NumericLiteral + match scan_numeric(src, pos) { + Some((e, bigint, legacy)) => { + let err = if numeric_follow_error(src, e) { + Some(LexError::IdentifierAfterNumericLiteral) + } else { + None + }; + Some( + ( + Element { + kind: ElementKind::NumericLiteral, + start: pos, + end: e, + value: Value::Numeric { bigint, legacy }, + }, + err, + ), + ) + }, + None => None, + } + } else if c == 0x5C || id_start_char(c) { + // IdentifierName + match scan_identifier(src, pos) { + Some((cps, escaped, valid, e)) => { + let err = if valid { + None + } else { + Some(LexError::InvalidIdentifierEscape) + }; + Some( + ( + Element { + kind: ElementKind::IdentifierName, + start: pos, + end: e, + value: Value::Name { code_points: cps, escaped }, + }, + err, + ), + ) + }, + None => None, + } + } else if html && c == 0x3C && c1 == 0x21 && peek(src, pos.saturating_add(2)) == 0x2D && peek( + src, + pos.saturating_add(3), + ) == 0x2D { + // Annex B.1.1 SingleLineHTMLOpenComment :: y\nz", DIV, true)?, + vec![ + ("IdentifierName", 0, 1), + ("Comment", 1, 7), + ("LineTerminator", 7, 8), + ("IdentifierName", 8, 9) + ] + ); + expect("\r\n /* a */ /*b*/ --> c", DIV, true, "Comment", 23)?; + expect("\n--> c", DIV, false, "LineTerminator", 1)?; + // A delimited comment holding a line terminator stops it. + expect("\n/*\n*/-->", DIV, true, "LineTerminator", 1)?; + // Annex B MultiLineComment: one holding a line terminator may be + // followed by an HTMLCloseComment. + expect("/*\n*/ --> c\nd", DIV, true, "Comment", 11)?; + expect("/* */ --> c", DIV, true, "Comment", 5)?; + // HTMLCloseComment at the start of InputElementHashbangOrRegExp. + expect("--> c\nd", START, true, "Comment", 5)?; + expect(" /**/ --> c", START, true, "Comment", 12)?; + expect("--> c", DIV, true, "Punctuator", 2)?; + expect("--> c", START, false, "Punctuator", 2)?; + Ok(()) +} + +#[test] +fn identifiers() -> Result<(), String> { + let el = expect("abc d", DIV, true, "IdentifierName", 3)?; + assert_eq!(name(&el)?, (cps("abc"), false)); + let el = expect("$_a1", DIV, true, "IdentifierName", 4)?; + assert_eq!(name(&el)?, (cps("$_a1"), false)); + let el = expect("\\u0061b", DIV, true, "IdentifierName", 7)?; + assert_eq!(name(&el)?, (cps("ab"), true)); + let el = expect("a\\u{62}\\u{0000063}", DIV, true, "IdentifierName", 18)?; + assert_eq!(name(&el)?, (cps("abc"), true)); + let el = expect("a\\u0030", DIV, true, "IdentifierName", 7)?; + assert_eq!(name(&el)?, (cps("a0"), true)); + // §12.7.1.1: an escape must name an IdentifierStartChar (first) or an + // IdentifierPartChar. + expect_err("\\u0030a", DIV, true, LexError::InvalidIdentifierEscape)?; + expect_err("a\\u002D", DIV, true, LexError::InvalidIdentifierEscape)?; + expect_err("\\u{1F600}", DIV, true, LexError::InvalidIdentifierEscape)?; + expect_err("\\uD800", DIV, true, LexError::InvalidIdentifierEscape)?; + // Malformed escapes are not part of the name. + expect("a\\u00", DIV, true, "IdentifierName", 1)?; + expect("a\\u{110000}", DIV, true, "IdentifierName", 1)?; + expect_err("\\u{}", DIV, true, LexError::NoInputElement)?; + expect_err("\\x41", DIV, true, LexError::NoInputElement)?; + // Reserved words are names; the parser decides (§12.7.2). + expect("while", DIV, true, "IdentifierName", 5)?; + // Private names. + let el = expect("#foo.x", DIV, true, "PrivateIdentifier", 4)?; + assert_eq!(name(&el)?, (cps("foo"), false)); + let el = expect("#\\u0061", DIV, true, "PrivateIdentifier", 7)?; + assert_eq!(name(&el)?, (cps("a"), true)); + expect_err("#", DIV, true, LexError::NoInputElement)?; + expect_err("# a", DIV, true, LexError::NoInputElement)?; + expect_err("#\\u0030", DIV, true, LexError::InvalidIdentifierEscape)?; + Ok(()) +} + +#[test] +fn unicode_identifiers() -> Result<(), String> { + let el = expect("café x", DIV, true, "IdentifierName", 4)?; + assert_eq!(name(&el)?, (cps("café"), false)); + // U+212E is Other_ID_Start. + expect("\u{212E}", DIV, true, "IdentifierName", 1)?; + expect("日本語", DIV, true, "IdentifierName", 3)?; + // U+10400, outside the BMP. + expect("\u{10400}", DIV, true, "IdentifierName", 1)?; + // U+0300 (a combining mark) is ID_Continue but not ID_Start. + expect("a\u{300}", DIV, true, "IdentifierName", 2)?; + expect_err("\u{300}", DIV, true, LexError::NoInputElement)?; + // ZWNJ and ZWJ are ID_Continue (since Unicode 15.1). + expect("a\u{200C}\u{200D}", DIV, true, "IdentifierName", 3)?; + expect("a\\u200D", DIV, true, "IdentifierName", 7)?; + // Arabic-Indic digits continue a name but do not start one. + expect("a\u{661}", DIV, true, "IdentifierName", 2)?; + expect_err("\u{661}", DIV, true, LexError::NoInputElement)?; + // Emoji are neither. + expect_err("\u{1F600}", DIV, true, LexError::NoInputElement)?; + expect("a\u{1F600}", DIV, true, "IdentifierName", 1)?; + Ok(()) +} + +#[test] +fn punctuators() -> Result<(), String> { + for (s, n) in [ + ("{", 1), + ("(", 1), + (")", 1), + ("[", 1), + ("]", 1), + (".", 1), + ("...", 3), + ("..", 1), + (";", 1), + (",", 1), + ("<", 1), + ("<=", 2), + ("<<", 2), + ("<<=", 3), + (">", 1), + (">=", 2), + (">>", 2), + (">>=", 3), + (">>>", 3), + (">>>=", 4), + ("=", 1), + ("==", 2), + ("===", 3), + ("====", 3), + ("=>", 2), + ("!", 1), + ("!=", 2), + ("!==", 3), + ("+", 1), + ("++", 2), + ("+=", 2), + ("+++", 2), + ("-", 1), + ("--", 2), + ("-=", 2), + ("*", 1), + ("*=", 2), + ("**", 2), + ("**=", 3), + ("%", 1), + ("%=", 2), + ("&", 1), + ("&&", 2), + ("&=", 2), + ("&&=", 3), + ("|", 1), + ("||", 2), + ("|=", 2), + ("||=", 3), + ("^", 1), + ("^=", 2), + ("~", 1), + ("?", 1), + ("??", 2), + ("??=", 3), + ("?.", 2), + ("?.a", 2), + (":", 1), + ] { + expect(s, DIV, true, "Punctuator", n)?; + } + // OptionalChainingPunctuator :: ?. [lookahead ∉ DecimalDigit] + expect("?.5:1", DIV, true, "Punctuator", 1)?; + expect("/", DIV, true, "DivPunctuator", 1)?; + expect("/=", DIV, true, "DivPunctuator", 2)?; + expect("/=", TAIL, true, "DivPunctuator", 2)?; + expect("}", DIV, true, "RightBracePunctuator", 1)?; + expect("}", RE, true, "RightBracePunctuator", 1)?; + expect_err("@", DIV, true, LexError::NoInputElement)?; + Ok(()) +} + +#[test] +fn numeric_literals() -> Result<(), String> { + for (s, n, bigint, legacy) in [ + ("0", 1, false, false), + ("0;", 1, false, false), + ("123", 3, false, false), + ("1_000_000", 9, false, false), + ("1.5", 3, false, false), + ("1.", 2, false, false), + ("1.e3", 4, false, false), + ("0.e3", 4, false, false), + (".5", 2, false, false), + (".5e-3", 5, false, false), + ("1E+10", 5, false, false), + ("1_0.0_1e1_0", 11, false, false), + ("0.5", 3, false, false), + ("0x1F", 4, false, false), + ("0XdEaD_bEeF", 11, false, false), + ("0b1010", 6, false, false), + ("0o17", 4, false, false), + ("1n", 2, true, false), + ("0n", 2, true, false), + ("1_0n", 4, true, false), + ("0x1Fn", 5, true, false), + ("0b1n", 4, true, false), + ("07", 2, false, true), + ("0777", 4, false, true), + ("08", 2, false, true), + ("019", 3, false, true), + ("08.5", 4, false, true), + ("09e1", 4, false, true), + // A LegacyOctalIntegerLiteral is not a DecimalIntegerLiteral: `07.5` + // is `07` then `.5`. + ("07.5", 2, false, true), + ("1..toString", 2, false, false), + ("1 in", 1, false, false), + ] { + let el = expect(s, DIV, true, "NumericLiteral", n)?; + assert_eq!(numeric(&el)?, (bigint, legacy), "{s:?}"); + } + // §12.9.3: no IdentifierStart or DecimalDigit right after. + for s in [ + "3in", "1n2", "0x", "0b2", "0b12", "0xg", "1e", "1e+", "1_", "1__0", "0_1", "1._5", "08n", + "07n", "00n", "1.5n", "07e1", "1$", "1\\u0061", "1é", + ] { + expect_err(s, DIV, true, LexError::IdentifierAfterNumericLiteral)?; + } + // A malformed escape after a number is not an IdentifierStart. + expect("1\\u00", DIV, true, "NumericLiteral", 1)?; + Ok(()) +} + +#[test] +fn string_literals() -> Result<(), String> { + let el = expect("\"abc\"x", DIV, true, "StringLiteral", 5)?; + assert_eq!(string(&el)?, units("abc")); + let el = expect("''", DIV, true, "StringLiteral", 2)?; + assert_eq!(string(&el)?, units("")); + for (s, v) in [ + ("'a\\nb'", "a\nb"), + ("'\\b\\f\\n\\r\\t\\v'", "\u{8}\u{c}\n\r\t\u{b}"), + ("'\\'\\\"\\\\'", "'\"\\"), + ("'\\x41'", "A"), + ("'\\u0041'", "A"), + ("'\\u{1F600}'", "\u{1F600}"), + ("'\u{1F600}'", "\u{1F600}"), + ("'\\0'", "\0"), + ("'\\08'", "\08"), + ("'\\101'", "A"), + ("'\\1010'", "A0"), + ("'\\377'", "\u{ff}"), + ("'\\400'", " 0"), + ("'\\47'", "'"), + ("'\\7'", "\u{7}"), + ("'\\8\\9'", "89"), + ("'\\a\\q'", "aq"), + ("'a\\\nb'", "ab"), + ("'a\\\r\nb'", "ab"), + ("'a\\\u{2028}b'", "ab"), + ("'a\u{2028}b'", "a\u{2028}b"), + ("\"'\"", "'"), + ] { + let el = expect(s, DIV, true, "StringLiteral", cps(s).len())?; + assert_eq!(string(&el)?, units(v), "{s:?}"); + } + let el = expect("'\\uD800'", DIV, true, "StringLiteral", 8)?; + assert_eq!(string(&el)?, vec![0xD800]); + for s in [ + "'abc", + "'a\nb'", + "'a\rb'", + "'\\x4'", + "'\\xg0'", + "'\\u004'", + "'\\u{110000}'", + "'\\u{}'", + "'\\", + "\"a'", + ] { + expect_err(s, DIV, true, LexError::NoInputElement)?; + } + Ok(()) +} + +#[test] +fn templates() -> Result<(), String> { + let el = expect("`abc`", DIV, true, "NoSubstitutionTemplate", 5)?; + assert_eq!(template(&el)?, (Some(units("abc")), units("abc"))); + let el = expect("`a${b}`", DIV, true, "TemplateHead", 4)?; + assert_eq!(template(&el)?, (Some(units("a")), units("a"))); + let el = expect("``", DIV, true, "NoSubstitutionTemplate", 2)?; + assert_eq!(template(&el)?, (Some(units("")), units(""))); + let el = expect("`$`", DIV, true, "NoSubstitutionTemplate", 3)?; + assert_eq!(template(&el)?, (Some(units("$")), units("$"))); + let el = expect("`a\\n`", DIV, true, "NoSubstitutionTemplate", 5)?; + assert_eq!(template(&el)?, (Some(units("a\n")), units("a\\n"))); + let el = expect("`\r\n\r`", DIV, true, "NoSubstitutionTemplate", 5)?; + assert_eq!(template(&el)?, (Some(units("\n\n")), units("\n\n"))); + let el = expect("`\\\r\n`", DIV, true, "NoSubstitutionTemplate", 5)?; + assert_eq!(template(&el)?, (Some(units("")), units("\\\n"))); + let el = expect("`\\u{1F600}`", DIV, true, "NoSubstitutionTemplate", 11)?; + assert_eq!( + template(&el)?, + (Some(units("\u{1F600}")), units("\\u{1F600}")) + ); + // NotEscapeSequence: the TV is undefined, the TRV is the source. + for (s, raw) in [ + ("`\\unicode`", "\\unicode"), + ("`\\u{`", "\\u{"), + ("`\\u{}`", "\\u{}"), + ("`\\u{110000}`", "\\u{110000}"), + ("`\\u{12`", "\\u{12"), + ("`\\u12`", "\\u12"), + ("`\\x`", "\\x"), + ("`\\x1`", "\\x1"), + ("`\\xg`", "\\xg"), + ("`\\1`", "\\1"), + ("`\\01`", "\\01"), + ("`\\8`", "\\8"), + ] { + let el = expect(s, DIV, true, "NoSubstitutionTemplate", cps(s).len())?; + assert_eq!(template(&el)?, (None, units(raw)), "{s:?}"); + } + let el = expect("`\\0`", DIV, true, "NoSubstitutionTemplate", 4)?; + assert_eq!(template(&el)?, (Some(units("\0")), units("\\0"))); + let el = expect("}a`", TAIL, true, "TemplateTail", 3)?; + assert_eq!(template(&el)?, (Some(units("a")), units("a"))); + expect("}a${", RE_TAIL, true, "TemplateMiddle", 4)?; + expect("}", DIV, true, "RightBracePunctuator", 1)?; + expect_err("}", TAIL, true, LexError::NoInputElement)?; + expect_err("`abc", DIV, true, LexError::NoInputElement)?; + expect_err("`\\", DIV, true, LexError::NoInputElement)?; + Ok(()) +} + +#[test] +fn regular_expressions() -> Result<(), String> { + let el = expect("/ab+c/gi.x", RE, true, "RegularExpressionLiteral", 8)?; + assert!(matches!(el.value, Value::RegExp { body_end: 5 })); + expect("/[/]/", RE, true, "RegularExpressionLiteral", 5)?; + expect("/[\\]/]/", RE, true, "RegularExpressionLiteral", 7)?; + expect("/a\\/b/", RE, true, "RegularExpressionLiteral", 6)?; + expect("/=/", RE, true, "RegularExpressionLiteral", 3)?; + expect("/=/", DIV, true, "DivPunctuator", 2)?; + expect("/a/", START, true, "RegularExpressionLiteral", 3)?; + expect("/a/", RE_TAIL, true, "RegularExpressionLiteral", 3)?; + expect("/a/\\u0067", RE, true, "RegularExpressionLiteral", 3)?; + expect("/a/ü", RE, true, "RegularExpressionLiteral", 4)?; + for s in ["/a", "/a\n/", "/[a/", "/a\\\n/", "/*a/", "/"] { + expect_err(s, RE, true, LexError::NoInputElement)?; + } + Ok(()) +} + +/// The tokens of `src` in `goal` from 0, as (start, newline_before), and +/// the final `Eof`'s flag. +fn tokens(src: &[u32], goal: Goal) -> Result<(Vec<(usize, bool)>, bool), String> { + let mut pos = 0; + let mut seen = Vec::new(); + loop { + match next_token(src, pos, goal, true) { + Ok(Next::Token(tok)) => { + assert!(tok.element.start >= pos); + seen.push((tok.element.start, tok.newline_before)); + pos = tok.element.end; + } + Ok(Next::Eof { newline_before }) => return Ok((seen, newline_before)), + Err(e) => return Err(format!("at {pos}: {}", error_name(e))), + } + } +} + +#[test] +fn tokens_and_line_terminators() -> Result<(), String> { + let src = cps("a /* x */ b\n/*\n*/c // d\n\u{2028}e"); + assert_eq!( + tokens(&src, DIV)?, + (vec![(0, false), (10, false), (17, true), (25, true)], false) + ); + assert_eq!(tokens(&cps("x /*\n*/"), DIV)?, (vec![(0, false)], true)); + assert_eq!(tokens(&cps(""), DIV)?, (vec![], false)); + assert!(matches!( + next_token(&cps(" @"), 0, DIV, true), + Err(LexError::NoInputElement) + )); + Ok(()) +} + +/// Every input of up to three code points from an alphabet that reaches +/// every branch's first code point gives a result, at every position, and +/// an element is a non-empty range at the position (what Verus proves, run +/// here as well so that Miri sees each branch execute). +#[test] +fn small_inputs_are_total() { + const ALPHABET: &[u32] = &[ + 0x09, + 0x0A, + 0x0D, + 0x20, + 0x21, + 0x22, + 0x23, + 0x24, + 0x27, + 0x2A, + 0x2B, + 0x2D, + 0x2E, + 0x2F, + 0x30, + 0x31, + 0x38, + 0x3C, + 0x3D, + 0x3E, + 0x3F, + 0x5B, + 0x5C, + 0x5D, + 0x5F, + 0x60, + 0x61, + 0x6E, + 0x75, + 0x78, + 0x7B, + 0x7D, + 0xE9, + 0x2028, + 0xFEFF, + 0x1_F600, + 0x11_0000, + u32::MAX, + ]; + const GOALS: [Goal; 5] = [DIV, RE, RE_TAIL, TAIL, START]; + // Under Miri a stride of the inputs (it interprets every step); the + // full set natively. + let stride = if cfg!(miri) { 331 } else { 1 }; + let mut inputs: Vec> = ALPHABET.iter().map(|&a| vec![a]).collect(); + for &a in ALPHABET { + for &b in ALPHABET { + inputs.push(vec![a, b]); + for &c in ALPHABET { + inputs.push(vec![a, b, c]); + } + } + } + let mut checked = 0; + for src in inputs.iter().step_by(stride) { + for goal in GOALS { + for html in [false, true] { + for pos in 0..=src.len() { + match next_input_element(src, pos, goal, html) { + Ok(Step::Eof) => assert_eq!(pos, src.len()), + Ok(Step::Element(el)) => { + assert!(el.start == pos && pos < el.end && el.end <= src.len()); + } + Err(_) => assert!(pos < src.len()), + } + if let Ok(Next::Token(tok)) = next_token(src, pos, goal, html) { + assert!(pos <= tok.element.start && tok.element.end <= src.len()); + } + } + } + } + checked += 1; + } + assert!(checked > 0); +} diff --git a/crates/clazel-js-lexer/tests/test262.rs b/crates/clazel-js-lexer/tests/test262.rs new file mode 100644 index 000000000..9e6fe5b3d --- /dev/null +++ b/crates/clazel-js-lexer/tests/test262.rs @@ -0,0 +1,231 @@ +//! The lexer over the vendored Test262 subset (`tests/data/test262/`, its +//! table `tests/generated/test262.rs`). +//! +//! This is not yet Test262's conformance run: that goes through the parser +//! (phase-7.md 7.3), which chooses each token's goal symbol. Without a +//! parser the goal is guessed from the previous token (below), so these +//! tests check what does not depend on the guess: +//! +//! - **Totality and span coverage**, on every file: from every position the +//! lexer reaches, `next_input_element` returns, and the elements tile the +//! file (after an error it resumes one code point on). +//! - **No spurious error**: every test that must parse (no `negative:`, not +//! an expected failure) lexes to the end with the guessed goals. (Were +//! the guess wrong for one, a parser's goal would differ; none is.) + +use clazel_js_lexer::api::{Element, Next, Step, Value}; +use clazel_js_lexer::exec::{next_input_element, next_token}; +use clazel_js_lexer::spec::{ElementKind, Goal, TemplateKind}; +use std::collections::BTreeSet; +use std::path::PathBuf; + +/// A test's row in the generated table. +struct Case { + path: &'static str, + flags: &'static [&'static str], + features: &'static [&'static str], + includes: &'static [&'static str], + negative: Option<(&'static str, &'static str)>, + expect: Expect, +} + +/// Whether a test is expected to pass, and if not why. +enum Expect { + Run, + Fail(&'static str), +} + +include!("generated/test262.rs"); + +fn root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data/test262/test") +} + +fn read(case: &Case) -> Result, String> { + let path = root().join(case.path); + let text = std::fs::read_to_string(&path).map_err(|e| format!("{}: {e}", case.path))?; + Ok(text.chars().map(u32::from).collect()) +} + +/// Annex B.1.1 applies to scripts, not modules. +fn html(case: &Case) -> bool { + !case.flags.contains(&"module") +} + +/// Every element from 0 in `goal`, resuming one code point after an error: +/// the elements and the skipped code points tile the source text. +fn tile(src: &[u32], goal: Goal, html: bool) { + let mut pos: usize = 0; + loop { + match next_input_element(src, pos, goal, html) { + Ok(Step::Eof) => { + assert_eq!(pos, src.len()); + return; + } + Ok(Step::Element(el)) => { + assert!(el.start == pos && pos < el.end && el.end <= src.len()); + pos = el.end; + } + Err(_) => { + assert!(pos < src.len()); + pos = pos.saturating_add(1); + } + } + } +} + +/// The names after which a `/` starts a regular expression. +const BEFORE_EXPRESSION: &[&str] = &[ + "return", + "typeof", + "instanceof", + "in", + "of", + "new", + "delete", + "void", + "throw", + "case", + "do", + "else", + "yield", + "await", + "extends", +]; + +/// Whether a `/` after `el` starts a regular expression, as a guess. +fn regexp_after(el: &Element, src: &[u32]) -> bool { + let text: String = src + .get(el.start..el.end) + .unwrap_or(&[]) + .iter() + .filter_map(|&c| char::from_u32(c)) + .collect(); + match el.kind { + ElementKind::Punctuator => !matches!(text.as_str(), ")" | "]"), + ElementKind::RightBracePunctuator + | ElementKind::DivPunctuator + | ElementKind::Template(TemplateKind::Head | TemplateKind::Middle) => true, + ElementKind::IdentifierName => match &el.value { + Value::Name { + escaped: false, + code_points, + } => { + let name: String = code_points + .iter() + .filter_map(|&c| char::from_u32(c)) + .collect(); + BEFORE_EXPRESSION.contains(&name.as_str()) + } + _ => false, + }, + _ => false, + } +} + +/// Lexes `src` to the end with `next_token`, guessing each goal symbol from +/// the token before (and a stack of open braces, to know when a `}` closes +/// a template substitution). Returns the error's position, if any. +fn lex_tokens(src: &[u32], html: bool) -> Result<(), usize> { + let mut pos = 0; + let mut regexp = true; + let mut braces: Vec = Vec::new(); + loop { + let in_template = braces.last() == Some(&true); + let goal = if pos == 0 { + Goal::InputElementHashbangOrRegExp + } else { + match (regexp, in_template) { + (true, true) => Goal::InputElementRegExpOrTemplateTail, + (false, true) => Goal::InputElementTemplateTail, + (true, false) => Goal::InputElementRegExp, + (false, false) => Goal::InputElementDiv, + } + }; + match next_token(src, pos, goal, html) { + Ok(Next::Eof { .. }) => return Ok(()), + Ok(Next::Token(tok)) => { + let el = &tok.element; + assert!(pos <= el.start && el.start < el.end && el.end <= src.len()); + match el.kind { + ElementKind::Punctuator if src.get(el.start) == Some(&u32::from('{')) => { + braces.push(false); + } + ElementKind::Template(TemplateKind::Head) => braces.push(true), + ElementKind::RightBracePunctuator + | ElementKind::Template(TemplateKind::Tail) => { + braces.pop(); + } + _ => {} + } + regexp = regexp_after(el, src); + pos = el.end; + } + Err(_) => { + // The position of the token that failed: after its trivia. + let mut p = pos; + while let Ok(Step::Element(el)) = next_input_element(src, p, goal, html) { + p = el.end; + } + return Err(p); + } + } + } +} + +#[test] +#[cfg_attr(miri, ignore = "reads files")] +fn test262_lexes_total_and_tiled() -> Result<(), String> { + assert_eq!(TREE_SHA256.len(), 64); + let mut spurious = Vec::new(); + let mut rejected_negative = 0; + let mut negative = 0; + let mut features = BTreeSet::new(); + let mut with_includes = 0; + let mut expected_failures = 0; + let mut reasons = BTreeSet::new(); + for case in TEST262 { + let src = read(case)?; + let html = html(case); + features.extend(case.features.iter().copied()); + if !case.includes.is_empty() { + with_includes += 1; + } + for goal in [Goal::InputElementDiv, Goal::InputElementRegExp] { + tile(&src, goal, html); + } + let result = lex_tokens(&src, html); + if let Expect::Fail(reason) = case.expect { + reasons.insert(reason); + } + match (case.negative, &case.expect) { + (Some(_), _) => { + negative += 1; + if result.is_err() { + rejected_negative += 1; + } + } + (None, Expect::Run) => { + if let Err(at) = result { + spurious.push(format!("{} at {at}", case.path)); + } + } + (None, Expect::Fail(_)) => expected_failures += 1, + } + } + println!( + "{} tests ({} features, {with_includes} with includes); {negative} negative, \ + {rejected_negative} of them rejected by the lexer; {expected_failures} positive \ + expected failures skipped; {} reasons for expected failures", + TEST262.len(), + features.len(), + reasons.len() + ); + assert!( + spurious.is_empty(), + "{} tests that must parse do not lex:\n{}", + spurious.len(), + spurious.join("\n") + ); + Ok(()) +} diff --git a/crates/clazel-js-lexer/tests/unicode_tables.rs b/crates/clazel-js-lexer/tests/unicode_tables.rs new file mode 100644 index 000000000..eae4018a9 --- /dev/null +++ b/crates/clazel-js-lexer/tests/unicode_tables.rs @@ -0,0 +1,78 @@ +//! The `ID_Start` and `ID_Continue` tables (`src/unicode_data.rs`, generated +//! from Unicode 18.0.0 by `xtask/src/js_idents.rs`) and their lookups: the +//! shape the bisection relies on, a property Unicode guarantees, and known +//! code points. Nothing about the tables is proven (`src/unicode.rs`). + +use clazel_js_lexer::unicode::{Range, id_continue_non_ascii, id_start_non_ascii}; +use clazel_js_lexer::unicode_data::{ID_CONTINUE, ID_START}; + +/// Sorted, disjoint, non-empty rows, all within U+0080..U+10FFFF. +fn well_formed(table: &[Range]) { + let mut previous: Option = None; + for row in table { + assert!(0x80 <= row.first && row.first <= row.last && row.last <= 0x10_FFFF); + if let Some(p) = previous { + // Maximal ranges: not even adjacent. + assert!(p.saturating_add(1) < row.first); + } + previous = Some(row.last); + } +} + +#[test] +fn tables_are_sorted_and_disjoint() { + well_formed(&ID_START); + well_formed(&ID_CONTINUE); +} + +/// Every lookup agrees with a linear scan of its table, and `ID_Start` is a +/// subset of `ID_Continue` (Unicode's stability policy for identifiers). +#[test] +fn lookups_agree_with_the_tables() { + let in_table = |table: &[Range], c: u32| table.iter().any(|r| r.first <= c && c <= r.last); + // Each linear scan reads about 800 rows, so natively every 7th code + // point (all of them would take minutes in a debug build) plus every + // row's ends; under Miri, which interprets each step, a wider stride of + // both. + let (stride, row_stride) = if cfg!(miri) { (40_009, 97) } else { (7, 1) }; + let mut points: Vec = (0..=0x11_0000u32).step_by(stride).collect(); + for row in ID_START + .iter() + .chain(ID_CONTINUE.iter()) + .step_by(row_stride) + { + points.extend([ + row.first.saturating_sub(1), + row.first, + row.last, + row.last.saturating_add(1), + ]); + } + for c in points { + let start = id_start_non_ascii(c); + let cont = id_continue_non_ascii(c); + assert_eq!(start, c >= 0x80 && in_table(&ID_START, c), "U+{c:04X}"); + assert_eq!(cont, c >= 0x80 && in_table(&ID_CONTINUE, c), "U+{c:04X}"); + assert!(!start || cont, "U+{c:04X} is ID_Start but not ID_Continue"); + } +} + +#[test] +fn known_code_points() { + for c in [0xE9, 0xAA, 0x212E, 0x65E5, 0x1_0400, 0x2_0000, 0x3_134A] { + assert!(id_start_non_ascii(c), "U+{c:04X} is ID_Start"); + } + for c in [0x300, 0x661, 0x200C, 0x200D, 0xB7, 0x1369] { + assert!(!id_start_non_ascii(c), "U+{c:04X} is not ID_Start"); + assert!(id_continue_non_ascii(c), "U+{c:04X} is ID_Continue"); + } + for c in [ + 0x7F, 0x80, 0xA0, 0xD7, 0x2028, 0x2E2F, 0xD800, 0xFEFF, 0x1_F600, 0x10_FFFF, 0x11_0000, + ] { + assert!(!id_continue_non_ascii(c), "U+{c:04X} is not ID_Continue"); + } + // ASCII is not in the tables (the spec defines it). + for c in 0..0x80 { + assert!(!id_start_non_ascii(c) && !id_continue_non_ascii(c)); + } +} diff --git a/xtask/src/js_idents.rs b/xtask/src/js_idents.rs new file mode 100644 index 000000000..2253d90c8 --- /dev/null +++ b/xtask/src/js_idents.rs @@ -0,0 +1,191 @@ +//! The `ID_Start` and `ID_Continue` tables of `clazel-js-lexer` (Phase 7 +//! item 7.2). +//! +//! ECMA-262 §12.7 defines `UnicodeIDStart` and `UnicodeIDContinue` as the +//! code points with the Unicode properties `ID_Start` and `ID_Continue`. +//! The lexer's spec defines them for ASCII and leaves the rest +//! uninterpreted; the executable lexer reads them from +//! `crates/clazel-js-lexer/src/unicode_data.rs`, which this module writes +//! from `DerivedCoreProperties.txt` of Unicode 18.0.0, the copy vendored for +//! `clazel-js-regexp` (`crates/clazel-js-regexp/tests/data/ucd-regexp/`; +//! `cargo xtask testdata` checks that directory's inventory). `cargo xtask +//! tables` writes it; `check` and `verify` fail if the file's SHA-256 +//! differs from that `SOURCE.md` or the committed table differs from what +//! this produces. +//! +//! What it derives (also listed in the generated file's header): for each +//! of the two properties, the code points above U+007F listed with it, as +//! maximal ranges, sorted and disjoint. + +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; + +use crate::reviews; +use crate::unicode::{data_lines, range, read}; + +/// The vendored data directory, relative to the repository root. +const DATA_DIR: &str = "crates/clazel-js-regexp/tests/data/ucd-regexp"; +/// The one file read. +const FILE: &str = "DerivedCoreProperties.txt"; +/// The generated file, relative to the repository root. +const OUTPUT: &str = "crates/clazel-js-lexer/src/unicode_data.rs"; + +const MAX_CODE_POINT: u32 = 0x10_FFFF; + +/// Compare (or, with `write`, rewrite) the generated table. Returns +/// (problems, notes). +pub fn run(write: bool) -> (Vec, Vec) { + let dir = PathBuf::from(DATA_DIR); + let mut notes = Vec::new(); + if let Err(e) = check_hash(&dir) { + return (vec![e], notes); + } + let fresh = match generate(&dir) { + Ok(f) => f, + Err(e) => return (vec![format!("{DATA_DIR}: {e}")], notes), + }; + let mut problems = Vec::new(); + let current = std::fs::read_to_string(OUTPUT).ok(); + if current.as_deref() == Some(fresh.as_str()) { + notes.push(format!("{OUTPUT}: up to date")); + } else if write { + match std::fs::write(OUTPUT, &fresh) { + Ok(()) => notes.push(format!("{OUTPUT}: written")), + Err(e) => problems.push(format!("{OUTPUT}: {e}")), + } + } else if current.is_none() { + problems.push(format!("{OUTPUT}: missing; run `cargo xtask tables`")); + } else { + problems.push(format!( + "{OUTPUT}: stale (differs from what xtask/src/js_idents.rs produces); \ + run `cargo xtask tables` and commit the result" + )); + } + (problems, notes) +} + +/// The file's SHA-256 is the one its `SOURCE.md` lists, and its header +/// names Unicode 18.0.0. +fn check_hash(dir: &Path) -> Result<(), String> { + let source = std::fs::read_to_string(dir.join("SOURCE.md")) + .map_err(|e| format!("{DATA_DIR}/SOURCE.md: {e}"))?; + let listed = source + .lines() + .find_map(|l| { + let rest = l.trim().strip_prefix("SHA-256:")?.trim(); + let (hash, name) = rest.split_once(char::is_whitespace)?; + (name.trim().trim_matches('`') == FILE).then(|| hash.to_ascii_lowercase()) + }) + .ok_or(format!( + "{DATA_DIR}/SOURCE.md: no \"SHA-256: {FILE}\" line" + ))?; + let actual = reviews::sha256(&dir.join(FILE))?; + if actual != listed { + return Err(format!( + "{DATA_DIR}/{FILE}: SHA-256 {actual} differs from SOURCE.md ({listed})" + )); + } + let text = read(dir, FILE)?; + if !text + .lines() + .next() + .is_some_and(|l| l.contains("DerivedCoreProperties-18.0.0.txt")) + { + return Err(format!("{DATA_DIR}/{FILE}: not Unicode 18.0.0")); + } + Ok(()) +} + +/// The maximal ranges above U+007F of the code points `FILE` lists with +/// property `name`. +fn ranges(text: &str, name: &str) -> Result, String> { + let len = usize::try_from(MAX_CODE_POINT) + .map_err(|e| e.to_string())? + .saturating_add(1); + let mut has = vec![false; len]; + let mut found = false; + for (line, f) in data_lines(text) { + if f.get(1).copied() != Some(name) { + continue; + } + found = true; + let (a, b) = range(f.first().copied().unwrap_or(""), FILE, line)?; + if a > b || b > MAX_CODE_POINT { + return Err(format!("{FILE}:{line}: bad range")); + } + for c in a..=b { + let i = usize::try_from(c).map_err(|e| e.to_string())?; + if let Some(slot) = has.get_mut(i) { + *slot = true; + } + } + } + if !found { + return Err(format!("{FILE}: no {name} lines")); + } + let mut out: Vec<(u32, u32)> = Vec::new(); + for (c, &v) in (0u32..).zip(&has) { + if !v || c < 0x80 { + continue; + } + match out.last_mut() { + Some((_, last)) if last.checked_add(1) == Some(c) => *last = c, + _ => out.push((c, c)), + } + } + Ok(out) +} + +fn generate(dir: &Path) -> Result { + let text = read(dir, FILE)?; + let id_start = ranges(&text, "ID_Start")?; + let id_continue = ranges(&text, "ID_Continue")?; + let mut out = String::new(); + out.push_str(HEADER); + table( + &mut out, + "ID_START", + "`ID_Start` above U+007F (DerivedCoreProperties.txt), as maximal ranges.", + &id_start, + ); + table( + &mut out, + "ID_CONTINUE", + "`ID_Continue` above U+007F (DerivedCoreProperties.txt), as maximal ranges.", + &id_continue, + ); + out.push_str("\n} // verus!\n"); + Ok(out) +} + +const HEADER: &str = "\ +//! The Unicode `ID_Start` and `ID_Continue` tables of the lexer (§12.7). +//! +// @generated by `cargo xtask tables` (xtask/src/js_idents.rs) from +// crates/clazel-js-regexp/tests/data/ucd-regexp/DerivedCoreProperties.txt +// (Unicode 18.0.0; URL and SHA-256 in SOURCE.md there). Do not edit: +// `cargo xtask check` fails if this differs from what the generator +// produces. +// +// What the generator derives: for each of ID_Start and ID_Continue, the +// code points above U+007F that DerivedCoreProperties.txt lists with it, as +// maximal ranges (first, last), sorted and disjoint. + +use super::unicode::Range; +#[allow(unused_imports)] +use vstd::prelude::*; + +verus! { +"; + +fn table(out: &mut String, name: &str, doc: &str, rows: &[(u32, u32)]) { + let _ = write!( + out, + "\n/// {doc}\n#[verifier::opaque]\npub const {name}: [Range; {}] = [\n", + rows.len() + ); + for (a, b) in rows { + let _ = writeln!(out, " Range {{ first: 0x{a:X}, last: 0x{b:X} }},"); + } + out.push_str("];\n"); +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index b54ca9e96..0feed1485 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -22,7 +22,8 @@ //! `verify` fail if a committed table is stale. //! - `cargo xtask tables`: regenerate the Unicode tables generated from //! vendored Unicode data (`crates/clazel-idna/src/uts46/data.rs` and -//! `crates/clazel-layout/src/text/data.rs`), the +//! `crates/clazel-layout/src/text/data.rs` and +//! `crates/clazel-js-lexer/src/unicode_data.rs`), the //! Public Suffix List, and the root certificate table from NSS's //! `certdata.txt` (`crates/clazel-x509/src/roots/data.rs`), and the //! user agent style sheet from HTML §15 (`crates/clazel-style/data/*.css`). @@ -39,6 +40,7 @@ mod conformance; mod deps; mod encoding_indexes; mod entities; +mod js_idents; mod linebreak; mod mutate; mod parts; @@ -367,6 +369,8 @@ fn static_checks(r: &mut Runner, crates: &[Crate], pins: &Pins) { r.step("generated test data", outcome(problems, notes)); let (problems, notes) = unicode::run(false); r.step("generated Unicode tables", outcome(problems, notes)); + let (problems, notes) = js_idents::run(false); + r.step("generated JS identifier tables", outcome(problems, notes)); let (problems, notes) = linebreak::run(false); r.step("generated line breaking tables", outcome(problems, notes)); let (problems, notes) = psl::run(false); @@ -1029,6 +1033,8 @@ fn generate_tables() -> ExitCode { } let (problems, notes) = unicode::run(true); r.step("generate Unicode tables", outcome(problems, notes)); + let (problems, notes) = js_idents::run(true); + r.step("generate JS identifier tables", outcome(problems, notes)); let (problems, notes) = linebreak::run(true); r.step("generate line breaking tables", outcome(problems, notes)); let (problems, notes) = psl::run(true);