From b5edfcccc3d4d720bfc6419ed8a8837cac8c1897 Mon Sep 17 00:00:00 2001 From: Alex Tiernan-Berry Date: Mon, 16 Nov 2026 14:55:54 +0000 Subject: [PATCH 1/5] Transcribe the ECMAScript lexical grammar as a Verus spec (7.2, spec first) ECMA-262 es2026 at 0248456c: section 11.1, section 12 (goal symbols, white space, comments, tokens, names, punctuators, numeric, string, regular expression and template literals with SV, TV and TRV) and Annex B.1.1 HTML-like comments, with the longest-element rule and the two lexical early errors. Facts proven about it (longest-element search, trivia skip bounds) and evaluated examples. Records one erratum (the missing MV of the escape 0 before 8 or 9). Tier stays 0: no implementation. Co-Authored-By: Claude Sonnet 5.5 --- Cargo.lock | 3 + crates/clazel-js-lexer/Cargo.toml | 12 +- crates/clazel-js-lexer/src/examples.rs | 245 ++++ crates/clazel-js-lexer/src/lemmas.rs | 144 ++ crates/clazel-js-lexer/src/lib.rs | 17 +- crates/clazel-js-lexer/src/spec.rs | 1789 ++++++++++++++++++++++++ docs/ERRATA.md | 22 + 7 files changed, 2225 insertions(+), 7 deletions(-) create mode 100644 crates/clazel-js-lexer/src/examples.rs create mode 100644 crates/clazel-js-lexer/src/lemmas.rs create mode 100644 crates/clazel-js-lexer/src/spec.rs diff --git a/Cargo.lock b/Cargo.lock index 4819e7bfa..34f5389e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -362,6 +362,9 @@ dependencies = [ [[package]] name = "clazel-js-lexer" version = "0.0.0" +dependencies = [ + "vstd", +] [[package]] name = "clazel-js-regexp" diff --git a/crates/clazel-js-lexer/Cargo.toml b/crates/clazel-js-lexer/Cargo.toml index 05dadcf62..e2dd9eb22 100644 --- a/crates/clazel-js-lexer/Cargo.toml +++ b/crates/clazel-js-lexer/Cargo.toml @@ -6,14 +6,20 @@ license.workspace = true publish.workspace = true description = "The JavaScript lexer for Clazel's Phase 7 browser (phase-7.md item 7.2): ECMA-262 section 12 source text to tokens, the goal symbol an input." +[dependencies] +vstd = "=0.0.0-2026-10-04-0306" + # Verification tier (docs/VERIFICATION.md). `tier` is what is proven about the # shipped code today; `target-tier` is what the policy requires, since this -# lexes scripts a page chose (phase-7.md 7.0(a)). Tier 0: this is only the -# skeleton that holds the test data of item 7.1. The contracts come first -# (rule 3), in item 7.2. +# lexes scripts a page chose (phase-7.md 7.0(a)). Tier 0: the spec and the +# contracts are written (item 7.2, first half, CLAUDE.md rule 3) and check +# with Verus, but no implementation exists, so nothing shipped is proven. [package.metadata.clazel] tier = 0 target-tier = 2 +[package.metadata.verus] +verify = true + [lints] workspace = true diff --git a/crates/clazel-js-lexer/src/examples.rs b/crates/clazel-js-lexer/src/examples.rs new file mode 100644 index 000000000..7b5144d25 --- /dev/null +++ b/crates/clazel-js-lexer/src/examples.rs @@ -0,0 +1,245 @@ +//! The standard's own examples and a few small inputs, evaluated against +//! the spec: evidence that the transcription accepts what the standard says +//! it accepts, and computes the values it says it computes. Positive facts +//! only; the negative ones (that no alternative derives a range) are for +//! the conformance suite and, in the second half of 7.2, the proofs. +//! +//! This is a module of its own for the reason `clazel-percent`'s is. + +#[allow(unused_imports)] +use super::spec::*; +#[allow(unused_imports)] +use vstd::prelude::*; + +verus! { + +/// §12.7 and §12.7.1.2: `ab` is an `IdentifierName` whose +/// IdentifierCodePoints are `ab` (an escape contributes the code point it +/// names), and it has no early error. +pub proof fn example_identifier_with_escape() + ensures + identifier_name_cps( + seq![0x61u32, 0x5C, 0x75, 0x30, 0x30, 0x36, 0x32], + 0, + 7, + seq![0x61u32, 0x62], + ), + identifier_early_errors_ok(seq![0x61u32, 0x62]), +{ + let s = seq![0x61u32, 0x5C, 0x75, 0x30, 0x30, 0x36, 0x32]; + reveal_with_fuel(hex_digits_mv, 5); + assert(identifier_name_cps(s, 0, 1, seq![0x61u32])); + assert(identifier_part(s, 1, 7)) by { + assert(hex4_digits(s, 3, 7)); + assert(unicode_escape_sequence(s, 2, 7)); + } + assert(unicode_escape_code_point(s, 2, 7) == 0x62); + assert(identifier_code_point(s, 1, 7) == 0x62u32); + assert(seq![0x61u32, 0x62].drop_last() =~= seq![0x61u32]); + assert(identifier_name_cps(s, 0, 7, seq![0x61u32, 0x62])); + let cps = seq![0x61u32, 0x62]; + assert(identifier_early_errors_ok(cps)) by { + assert(cps[0] == 0x61u32); + assert(cps[1] == 0x62u32); + } +} + +/// §12.9.3: "`3in` is an error and not the two input elements `3` and +/// `in`": the code point after `3` starts an `IdentifierStart`. +pub proof fn example_numeric_literal_then_identifier() + ensures + numeric_literal(seq![0x33u32, 0x69, 0x6E], 0, 1), + numeric_literal_follow_error(seq![0x33u32, 0x69, 0x6E], 1), +{ + let s = seq![0x33u32, 0x69, 0x6E]; + assert(decimal_integer_literal(s, 0, 1)); + assert(decimal_literal(s, 0, 1)) by { + assert(decimal_integer_literal(s, 0, 1)); + } + assert(identifier_start(s, 1, 2)); +} + +/// §12.9.3: numeric separators and the BigInt suffix. `0b1_0n` is a +/// `NumericLiteral`, a BigInt; `1_000` is a decimal literal. +pub proof fn example_numeric_separators() + ensures + numeric_literal(seq![0x30u32, 0x62, 0x31, 0x5F, 0x30, 0x6E], 0, 6), + numeric_literal_is_bigint(seq![0x30u32, 0x62, 0x31, 0x5F, 0x30, 0x6E], 0, 6), + numeric_literal(seq![0x31u32, 0x5F, 0x30, 0x30, 0x30], 0, 5), +{ + let s = seq![0x30u32, 0x62, 0x31, 0x5F, 0x30, 0x6E]; + assert(binary_digits(true, s, 2, 3)); + assert(binary_digits(true, s, 2, 5)); + assert(non_decimal_integer_literal(true, s, 0, 5)); + let t = seq![0x31u32, 0x5F, 0x30, 0x30, 0x30]; + assert(decimal_digits(true, t, 2, 3)); + assert(decimal_digits(true, t, 2, 4)); + assert(decimal_digits(true, t, 2, 5)); + assert(decimal_integer_literal(t, 0, 5)); + assert(decimal_literal(t, 0, 5)) by { + assert(decimal_integer_literal(t, 0, 5)); + } +} + +/// §12.9.3 with the legacy forms: `010` is a `LegacyOctalIntegerLiteral` and +/// `08` a `NonOctalDecimalIntegerLiteral`, both flagged legacy. +pub proof fn example_legacy_numeric_literals() + ensures + numeric_literal(seq![0x30u32, 0x31, 0x30], 0, 3), + numeric_literal_is_legacy(seq![0x30u32, 0x31, 0x30], 0, 3), + numeric_literal(seq![0x30u32, 0x38], 0, 2), + numeric_literal_is_legacy(seq![0x30u32, 0x38], 0, 2), +{ + let s = seq![0x30u32, 0x31, 0x30]; + assert(legacy_octal_integer_literal(s, 0, 2)); + assert(legacy_octal_integer_literal(s, 0, 3)); + let t = seq![0x30u32, 0x38]; + assert(non_octal_decimal_integer_literal(t, 0, 2)); + assert(decimal_integer_literal(t, 0, 2)); + assert(decimal_literal(t, 0, 2)) by { + assert(decimal_integer_literal(t, 0, 2)); + } + assert(numeric_literal_is_legacy(t, 0, 2)) by { + assert(non_octal_decimal_integer_literal(t, 0, 2)); + } +} + + +/// §12.9.4.2: `"a\n\u{1F600}\ud800"` has the string value `a`, LF, the +/// surrogate pair of U+1F600, and a lone high surrogate. +pub proof fn example_string_values() + ensures + string_literal_sv( + seq![ + 0x22u32, 0x61, 0x5C, 0x6E, 0x5C, 0x75, 0x7B, 0x31, 0x46, 0x36, 0x30, 0x30, 0x7D, + 0x5C, 0x75, 0x64, 0x38, 0x30, 0x30, 0x22, + ], + 0, + 20, + seq![0x61u16, 0x0A, 0xD83D, 0xDE00, 0xD800], + ), +{ + let s = seq![ + 0x22u32, 0x61, 0x5C, 0x6E, 0x5C, 0x75, 0x7B, 0x31, 0x46, 0x36, 0x30, 0x30, 0x7D, 0x5C, + 0x75, 0x64, 0x38, 0x30, 0x30, 0x22, + ]; + let q = 0x22u32; + reveal_with_fuel(hex_digits_mv, 6); + // The characters, from the right: `\ud800` is [13, 19), `\u{1F600}` is + // [4, 13), `\n` is [2, 4) and `a` is [1, 2). + let v5 = seq![0xD800u16]; + assert(string_character_sv(s, 13, 19, q, v5)) by { + assert(hex4_digits(s, 15, 19)); + assert(unicode_escape_sequence(s, 14, 19)); + assert(unicode_escape_sv(s, 14, 19) =~= v5); + assert(template_escape_sequence_sv(s, 14, 19, v5)); + assert(escape_sequence_sv(s, 14, 19, v5)); + } + super::lemmas::lemma_string_characters_one(s, 13, 19, q, v5); + let pair = utf16_encode_code_point(0x1F600); + assert(pair =~= seq![0xD83Du16, 0xDE00]); + assert(string_character_sv(s, 4, 13, q, pair)) by { + assert(digits(16, false, s, 7, 8)); + assert(digits(16, false, s, 7, 9)); + assert(digits(16, false, s, 7, 10)); + assert(digits(16, false, s, 7, 11)); + assert(digits(16, false, s, 7, 12)); + assert(code_point(s, 7, 12)); + assert(unicode_escape_sequence(s, 5, 13)); + assert(unicode_escape_sv(s, 5, 13) =~= pair); + assert(template_escape_sequence_sv(s, 5, 13, pair)); + assert(escape_sequence_sv(s, 5, 13, pair)); + } + super::lemmas::lemma_string_characters_cons(s, 4, 13, 19, q, pair, v5); + let lf = seq![0x0Au16]; + assert(string_character_sv(s, 2, 4, q, lf)) by { + assert(character_escape_sequence_sv(s, 3, 4, lf)); + assert(template_escape_sequence_sv(s, 3, 4, lf)); + assert(escape_sequence_sv(s, 3, 4, lf)); + } + super::lemmas::lemma_string_characters_cons(s, 2, 4, 19, q, lf, pair + v5); + let a = seq![0x61u16]; + assert(string_character_sv(s, 1, 2, q, a)); + super::lemmas::lemma_string_characters_cons(s, 1, 2, 19, q, a, lf + (pair + v5)); + assert(a + (lf + (pair + v5)) =~= seq![0x61u16, 0x0A, 0xD83D, 0xDE00, 0xD800]); + assert(string_literal_sv(s, 0, 20, a + (lf + (pair + v5)))); +} + +/// §12.9.6: `` `a${ `` is a `TemplateHead`, with TV and TRV `a`; and +/// `` `\u{` `` (an invalid escape) has TV *undefined* and TRV `\u{`. +pub proof fn example_template_values() + ensures + template_part( + seq![0x60u32, 0x61, 0x24, 0x7B], + 0, + 4, + TemplateKind::Head, + Some(seq![0x61u16]), + seq![0x61u16], + ), + template_part( + seq![0x60u32, 0x5C, 0x75, 0x7B, 0x60], + 0, + 5, + TemplateKind::NoSubstitution, + None, + seq![0x5Cu16, 0x75, 0x7B], + ), +{ + let s = seq![0x60u32, 0x61, 0x24, 0x7B]; + let a = seq![0x61u16]; + assert(template_character(s, 1, 2, Some(a), a)); + super::lemmas::lemma_template_characters_one(s, 1, 2, Some(a), a); + assert(is_lit(s, 2, 4, seq!['$', '{'])); + let t = seq![0x60u32, 0x5C, 0x75, 0x7B, 0x60]; + let raw = seq![0x5Cu16, 0x75, 0x7B]; + assert(not_escape_sequence(t, 2, 4)); + reveal_with_fuel(utf16_of, 5); + assert(utf16_of(t, 1, 4) =~= raw); + assert(template_character(t, 1, 4, None, raw)); + super::lemmas::lemma_template_characters_one(t, 1, 4, None, raw); +} + +/// §12.4: `/**/` and `/* a */` are comments; and a multi-line comment holds a +/// line terminator, which the parser sees as one. +pub proof fn example_comments() + ensures + comment(seq![0x2Fu32, 0x2A, 0x2A, 0x2F], 0, 4, false), + comment(seq![0x2Fu32, 0x2A, 0x0A, 0x2A, 0x2F], 0, 5, false), + contains_line_terminator(seq![0x2Fu32, 0x2A, 0x0A, 0x2A, 0x2F], 0, 5), +{ + let s = seq![0x2Fu32, 0x2A, 0x2A, 0x2F]; + assert(multi_line_comment(s, 0, 4)); + let t = seq![0x2Fu32, 0x2A, 0x0A, 0x2A, 0x2F]; + assert(multi_line_comment_chars(t, 2, 3)); + assert(multi_line_comment(t, 0, 5)); + assert(is_line_terminator(t[2])); +} + +/// §12.9.5: `/[/]\//gu` is a `RegularExpressionLiteral`: a `/` inside a +/// class or escaped does not end the body. +pub proof fn example_regexp_literal() + ensures + regexp_literal(seq![0x2Fu32, 0x5B, 0x2F, 0x5D, 0x5C, 0x2F, 0x2F, 0x67, 0x75], 0, 9, 6), +{ + let s = seq![0x2Fu32, 0x5B, 0x2F, 0x5D, 0x5C, 0x2F, 0x2F, 0x67, 0x75]; + assert(regexp_class_chars(s, 2, 2)); + assert(regexp_class_chars(s, 2, 3)) by { + assert(regexp_class_char(s, 2, 3)); + } + assert(regexp_class(s, 1, 4)); + assert(regexp_first_char(s, 1, 4)); + assert(regexp_chars(s, 4, 4)); + assert(regexp_backslash_sequence(s, 4, 6)); + assert(regexp_char(s, 4, 6)); + assert(regexp_chars(s, 4, 6)) by { + assert(regexp_chars(s, 4, 4)); + } + assert(regexp_body(s, 1, 6)); + assert(regexp_flags(s, 7, 9)) by { + assert(is_identifier_part_char(s[7])); + assert(is_identifier_part_char(s[8])); + } +} + +} // verus! diff --git a/crates/clazel-js-lexer/src/lemmas.rs b/crates/clazel-js-lexer/src/lemmas.rs new file mode 100644 index 000000000..c0693055c --- /dev/null +++ b/crates/clazel-js-lexer/src/lemmas.rs @@ -0,0 +1,144 @@ +//! Facts proven about the spec: evidence that the transcription is +//! well-behaved, not part of what the implementation must meet. +//! +//! - The longest-element search returns what it claims to, and the trivia +//! skip stays inside the source text: together with the contracts, these +//! are the span-coverage property of phase-7.md 7.2 (a lexical step +//! starts where the last one ended, goes forward, and does not leave the +//! source text). + +#[allow(unused_imports)] +use super::spec::*; +#[allow(unused_imports)] +use vstd::prelude::*; + +verus! { + +/// [`longest_len_upto`] is between 0 and `m`; when it is positive some +/// alternative derives that range, and no longer range up to `m` is derived. +pub proof fn lemma_longest_len_upto(goal: Goal, html: bool, s: Src, i: int, m: int) + requires + 0 <= i <= s.len(), + ensures + 0 <= longest_len_upto(goal, html, s, i, m) <= (if m > 0 { + m + } else { + 0 + }), + longest_len_upto(goal, html, s, i, m) > 0 ==> { + &&& i + longest_len_upto(goal, html, s, i, m) <= s.len() + &&& any_element_matches( + goal, + html, + s, + i, + i + longest_len_upto(goal, html, s, i, m), + ) + }, + forall|n: int| + longest_len_upto(goal, html, s, i, m) < n <= m && i + n <= s.len() ==> !( + #[trigger] any_element_matches(goal, html, s, i, i + n)), + decreases m, +{ + if m > 0 && !(i + m <= s.len() && any_element_matches(goal, html, s, i, i + m)) { + lemma_longest_len_upto(goal, html, s, i, m - 1); + } +} + +/// [`longest_len`] is the length of the longest input element: an element +/// of that length exists if it is positive, it fits in the source text, and +/// no longer one exists. +pub proof fn lemma_longest_len(goal: Goal, html: bool, s: Src, i: int) + requires + 0 <= i <= s.len(), + ensures + 0 <= longest_len(goal, html, s, i) <= s.len() - i, + longest_len(goal, html, s, i) > 0 ==> any_element_matches( + goal, + html, + s, + i, + i + longest_len(goal, html, s, i), + ), + forall|n: int| + longest_len(goal, html, s, i) < n <= s.len() - i ==> !(#[trigger] any_element_matches( + goal, + html, + s, + i, + i + n, + )), +{ + lemma_longest_len_upto(goal, html, s, i, s.len() - i); +} + +/// [`skip_trivia`] goes forward and stays inside the source text. +pub proof fn lemma_skip_trivia_bounds(goal: Goal, html: bool, s: Src, i: int) + requires + 0 <= i <= s.len(), + ensures + i <= skip_trivia(goal, html, s, i).0 <= s.len(), + decreases s.len() - i, +{ + let n = trivia_len(goal, html, s, i); + if 0 < n && i + n <= s.len() { + lemma_skip_trivia_bounds(goal, html, s, i + n); + } +} + +/// Introduction rules for [`string_characters_sv`], for the single-character +/// and the character-then-characters alternatives. +pub proof fn lemma_string_characters_one(s: Src, i: int, j: int, quote: u32, v: Seq) + requires + nonempty(s, i, j), + string_character_sv(s, i, j, quote, v), + ensures + string_characters_sv(s, i, j, quote, v), +{ +} + +/// See [`lemma_string_characters_one`]. +pub proof fn lemma_string_characters_cons( + s: Src, + i: int, + k: int, + j: int, + quote: u32, + v1: Seq, + v2: Seq, +) + requires + nonempty(s, i, j), + i < k < j, + string_character_sv(s, i, k, quote, v1), + string_characters_sv(s, k, j, quote, v2), + ensures + string_characters_sv(s, i, j, quote, v1 + v2), +{ +} + +/// Introduction rules for [`template_characters`]. +pub proof fn lemma_template_characters_one(s: Src, i: int, j: int, tv: Tv, trv: Seq) + requires + nonempty(s, i, j), + template_character(s, i, j, tv, trv), + ensures + template_characters(s, i, j, tv, trv), +{ +} + +/// A step that finds no error and is not at the end of the source text +/// consumes a non-empty range inside the source text: the element starts at +/// `i` and ends after it (the span-coverage property of one step). +pub proof fn lemma_step_progress(goal: Goal, html: bool, s: Src, i: int) + requires + 0 <= i < s.len(), + lex_error(goal, html, s, i) is None, + ensures + 0 < longest_len(goal, html, s, i) <= s.len() - i, + any_element_matches(goal, html, s, i, i + longest_len(goal, html, s, i)), +{ + lemma_longest_len(goal, html, s, i); +} + +} // verus! diff --git a/crates/clazel-js-lexer/src/lib.rs b/crates/clazel-js-lexer/src/lib.rs index 971d16068..c4bc5f8d7 100644 --- a/crates/clazel-js-lexer/src/lib.rs +++ b/crates/clazel-js-lexer/src/lib.rs @@ -1,7 +1,16 @@ //! The JavaScript lexer for Clazel's Phase 7 browser (phase-7.md item 7.2). //! -//! Empty for now: item 7.1 only creates the crate, so that the test data -//! (`tests/data/`) has a home. The spec and its contracts come first (CLAUDE.md -//! rule 3), then the implementation. +//! Spec: ECMA-262, 2026 edition, §11.1, §12 and Annex B.1.1, at commit +//! `0248456c758431e4bb8e5d26333ff1865123c9cd` (`SPEC-PIN.md`). +//! Tier: see `[package.metadata.clazel]` in `Cargo.toml` and +//! `docs/VERIFICATION.md`. //! -//! Tier: see `[package.metadata.clazel]` in `Cargo.toml` and `docs/VERIFICATION.md`. +//! This is the first half of item 7.2: the spec and the contracts, with no +//! implementation (CLAUDE.md rule 3). +//! +//! - [`spec`]: a transcription of the lexical grammar, the goal symbol a +//! parameter, and the longest-input-element rule. + +pub mod examples; +pub mod lemmas; +pub mod spec; diff --git a/crates/clazel-js-lexer/src/spec.rs b/crates/clazel-js-lexer/src/spec.rs new file mode 100644 index 000000000..9b3b0a2c5 --- /dev/null +++ b/crates/clazel-js-lexer/src/spec.rs @@ -0,0 +1,1789 @@ +//! Executable-style specification of the ECMAScript lexical grammar, +//! transcribed from ECMA-262, 2026 edition (`es2026`), §11.1 "Source Text", +//! §12 "ECMAScript Language: Lexical Grammar" and Annex B.1.1 "HTML-like +//! Comments", at commit `0248456c758431e4bb8e5d26333ff1865123c9cd` of +//! (31 March 2026; `SPEC-PIN.md`). +//! Section numbers below are the ones that `spec.html` numbers at that +//! commit. Nothing here runs; it defines what "correct" means. Every +//! definition cites the standard, and each `// N.` comment is the step or +//! alternative number in the cited production or algorithm (a production's +//! alternatives are numbered from 1 in the order the standard lists them). +//! +//! # What this file covers +//! +//! The five lexical goal symbols (§12), `WhiteSpace`, `LineTerminator`, +//! `Comment` (§12.4, and Annex B.1.1's HTML-like comments, which are required +//! of a web browser), `HashbangComment` (§12.5), `CommonToken` (§12.6), +//! `IdentifierName`, `PrivateIdentifier` and `ReservedWord` (§12.7), +//! `Punctuator` (§12.8), `NumericLiteral` with separators, BigInt and legacy +//! forms (§12.9.3), `StringLiteral` and the string value SV (§12.9.4), +//! `RegularExpressionLiteral`, lexical part only (§12.9.5), the template +//! literal lexical components with TV and TRV (§12.9.6), and the rule that +//! the lexer takes the longest input element (§12 intro). +//! +//! # What it leaves out +//! +//! - The MV and NumericValue of numeric literals (§12.9.3.2, §12.9.3.3). A +//! token carries its span; the number's value is computed from the span by +//! the numbers item (phase-7.md 7.11, with StringToNumber), not by the +//! lexer, so it is not part of what the lexer is proven to produce. +//! - Automatic semicolon insertion (§12.10), strict-mode early errors, the +//! parse of a regular expression's body, and `NullLiteral` and +//! `BooleanLiteral` (§12.9.1, §12.9.2), which are the keywords `null`, +//! `true` and `false`, tokens the lexer returns as `IdentifierName`. +//! - Annex B.1.2 (regular expression patterns): a body is not parsed here. +//! +//! # Modelling choices +//! +//! Each one is a deliberate difference in form, never in meaning. +//! +//! - **Source text** is a `Seq` of code points (§11.1), the type +//! `clazel-encoding` decodes to. `SourceCharacter` is `c <= 0x10FFFF` +//! ([`is_source_character`]), surrogate code points included, so every +//! `Seq` is a legal argument: a value above U+10FFFF matches no +//! production and is a lexical error, rather than a precondition. +//! - **A production is a relation on a range.** `X(s, i, j)` says that the +//! code points `s[i..j)` are derived by nonterminal `X` in the source text +//! `s`. All nonterminals derive a non-empty range except those the +//! standard writes with an `[empty]` alternative; an optional symbol +//! `Y?` is `a == b || Y(s, a, b)` over its range `[a, b)`. A right-hand +//! side that is a sequence `A B` is an existential over the split point. +//! This is the grammar as the standard gives it, so every production can +//! be compared with the standard's line by line, and it is the only place +//! where "derived by" is defined; no deterministic scanner appears here. +//! - **Lookahead** `[lookahead != x]`, `[lookahead ∈ S]` reads `s[j]`, the code +//! point after the derived range, in the whole source text (the +//! "immediately following input"); it fails to match at the end of the +//! source text. +//! - **Parameters** `[Sep]` on the digit productions are the Boolean `sep` +//! (`[+Sep]` is `true`). `BinaryDigits`, `OctalDigits`, `DecimalDigits` and +//! `HexDigits` have the same three alternatives over different digit +//! classes, and are one function [`digits`] with the radix as a parameter; +//! [`binary_digits`] and the others name the instances the standard names. +//! `DoubleStringCharacter` and `SingleStringCharacter` differ only in the +//! excluded quote, so they are one function with a `quote` parameter. +//! - **Values** (`IdentifierCodePoints`, SV, TV, TRV) are attributes of a +//! derivation. They are written as relations too: `identifier_name_cps(s, +//! i, j, cps)` says "`IdentifierName` derives `s[i..j)` and its +//! IdentifierCodePoints is `cps`", `string_characters_sv(.., v)` says the +//! same for SV, and so on. Each clause is the standard's rule for that +//! production. The relation holds for exactly one value for each range +//! the grammar derives; that is not assumed, and it is not needed for the +//! contracts, which only say that the value returned stands in the +//! relation to the range. +//! - **Template TRV of an escape sequence.** The TRV rules for +//! `TemplateEscapeSequence` and `NotEscapeSequence` (§12.9.6.2) list, for +//! each alternative, the string-concatenation of the code units of the +//! code points it matches (a digit's TRV is UTF16EncodeCodePoint of that +//! digit, and so on through `Hex4Digits`, `HexDigits`, `CodePoint`, +//! `NotCodePoint`, `SingleEscapeCharacter` and `NonEscapeCharacter`). The +//! twenty-odd rules therefore fold into [`utf16_of`], the UTF-16 encoding +//! of the matched range, used at the one step that holds them +//! ([`template_character`] alternatives 2 and 3). +//! - **Strings are UTF-16.** String values are `Seq` code units, built +//! with UTF16EncodeCodePoint (§11.1.1) and never decoded, so a lone +//! surrogate (from `"\uD800"`, or a lone surrogate code point in the +//! source) is an ordinary element. +//! - **Unicode properties.** The spec needs three properties of code points: +//! `Space_Separator` (Zs), `ID_Start` and `ID_Continue`. Zs is a closed +//! list of 17 code points that has not changed since Unicode 6.3, and is +//! written out ([`unicode_space_separator`]). `ID_Start` and `ID_Continue` +//! are large and change with every Unicode version; the standard says "the +//! latest version of the Unicode Standard", with later additions +//! permitted, so they are uninterpreted functions of the non-ASCII code +//! points ([`unicode_id_start_non_ascii`] and +//! [`unicode_id_continue_non_ascii`]), and the ASCII part, which does not +//! change, is defined. The implementation (phase-7.md 7.2, second half) +//! fixes a Unicode version in a generated table and has to tie it to +//! these two functions; until then nothing is claimed about them. +//! - **The longest input element** (§12 intro: "repeatedly taking the +//! longest possible sequence of code points as the next input element") is +//! [`longest_len`], the largest length for which some alternative of the +//! goal symbol derives the range, and the two checks that the standard +//! words as constraints on a chosen element rather than as grammar: +//! §12.9.3 "The SourceCharacter immediately following a NumericLiteral +//! must not be an IdentifierStart or DecimalDigit" and §12.7.1.1's early +//! errors for `IdentifierStart :: \ UnicodeEscapeSequence` and +//! `IdentifierPart :: \ UnicodeEscapeSequence`. They are checked after the +//! longest element is chosen, not used to pick a shorter one: `1.e` is an +//! error, not `1` `.` `e`, as in every engine. A failure is a +//! [`LexError`]; those two rules are the only early errors the lexer +//! raises, because they depend on one token alone (the others, such as +//! strict-mode octal, depend on context the parser has). +//! - **"IdentifierStart" after a numeric literal** is read as the +//! nonterminal: the following code points must not start a derivation of +//! `IdentifierStart`, so `3a` is an error as well as `3a`. +//! - **Goal symbols are alternatives, per the standard.** The goal is a +//! parameter of every function. `HashbangComment` and Annex B's +//! `HTMLCloseComment` alternative of `InputElementHashbangOrRegExp` match +//! only at index 0: the standard says they are "location-sensitive" and +//! the goal "is used at the start of a Script or Module". +//! - **Annex B.1.1 is a mode.** The boolean `html` says the source is parsed +//! as a `Script`, where Annex B's `Comment` replaces §12.4's; it is +//! `false` for a `Module`, which does not allow HTML-like comments. +//! - **A line terminator for the parser** (§12.4: "if a MultiLineComment +//! contains a line terminator code point, then the entire comment is +//! considered to be a LineTerminator", and Annex B: a +//! `SingleLineHTMLCloseComment` is too) is [`contains_line_terminator`] on +//! the range of a trivia element: the other trivia elements contain no +//! line terminator code point except the `LineTerminator` itself. +//! - **Trivia** (white space, line terminators, comments) are input elements +//! the syntactic grammar never sees. [`skip_trivia`] repeats the same goal +//! symbol over them; that is sound because the goal depends on the +//! syntactic context of the next token, which trivia do not change. +//! `ReservedWord` is [`is_reserved_word`], a function of an identifier's +//! code points; whether a given escape-free name is one is for the parser +//! (§12.7.2: the early errors of §13.1.1). + +#[allow(unused_imports)] +use vstd::prelude::*; + +verus! { + +/// §11.1 `ECMAScript source text` is a sequence of code points. +pub type Src = Seq; + +// --------------------------------------------------------------------- +// Small helpers for reading the standard's notation. +// --------------------------------------------------------------------- + +/// `s[k]` exists and is the code point `c`: a terminal symbol at `k`, and +/// the test of a `[lookahead = c]` (false at the end of the source text). +pub open spec fn at(s: Src, k: int, c: char) -> bool { + 0 <= k < s.len() && s[k] == c as u32 +} + +/// `s[i..j)` is exactly the code points of `lit`: a sequence of terminal +/// symbols such as `` `//` `` (and `[lookahead = ...]` of a string). +pub open spec fn is_lit(s: Src, i: int, j: int, lit: Seq) -> bool { + 0 <= i && j <= s.len() && j - i == lit.len() + && s.subrange(i, j) =~= lit.map_values(|c: char| c as u32) +} + +/// `Y?` over the range `[a, b)` of a symbol that derives a non-empty range: +/// the range is empty, or `y` holds. +pub open spec fn opt(a: int, b: int, y: bool) -> bool { + a == b || y +} + +/// The range `[i, j)` is non-empty and inside the source text. +pub open spec fn nonempty(s: Src, i: int, j: int) -> bool { + 0 <= i && i < j && j <= s.len() +} + +/// Every code point of `s[i..j)` satisfies `p`. +pub open spec fn all_of(s: Src, i: int, j: int, p: spec_fn(u32) -> bool) -> bool { + forall|k: int| i <= k < j ==> p(#[trigger] s[k]) +} + +// --------------------------------------------------------------------- +// §11.1 Source Text +// --------------------------------------------------------------------- + +/// §11.1 `SourceCharacter :: any Unicode code point`: U+0000 to U+10FFFF, +/// surrogate code points included. +pub open spec fn is_source_character(c: u32) -> bool { + c <= 0x10FFFF +} + +/// §11.1.1 UTF16EncodeCodePoint (_cp_). +pub open spec fn utf16_encode_code_point(cp: int) -> Seq { + // 1. Assert: 0 ≤ cp ≤ 0x10FFFF. + // 2. "If cp ≤ 0xFFFF, return the String value consisting of the code unit + // whose numeric value is cp." + if cp <= 0xFFFF { + seq![cp as u16] + } else { + // 3. Let cu1 be the code unit whose numeric value is + // floor((cp - 0x10000) / 0x400) + 0xD800. + let cu1 = ((cp - 0x10000) / 0x400 + 0xD800) as u16; + // 4. Let cu2 be the code unit whose numeric value is + // ((cp - 0x10000) modulo 0x400) + 0xDC00. + let cu2 = ((cp - 0x10000) % 0x400 + 0xDC00) as u16; + // 5. Return the string-concatenation of cu1 and cu2. + seq![cu1, cu2] + } +} + +/// The UTF-16 encoding of `s[i..j)`: the string-concatenation of +/// UTF16EncodeCodePoint of each code point, as in §11.1.2 CodePointsToString. +pub open spec fn utf16_of(s: Src, i: int, j: int) -> Seq + decreases j - i, +{ + if i >= j { + seq![] + } else { + utf16_encode_code_point(s[i] as int) + utf16_of(s, i + 1, j) + } +} + +// --------------------------------------------------------------------- +// §12 Lexical grammar: the goal symbols. +// --------------------------------------------------------------------- + +/// The lexical goal symbols (§12): which input elements the next element may +/// be. The syntactic grammar chooses one for every token. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Goal { + /// `InputElementDiv`: the default. + InputElementDiv, + /// `InputElementRegExp`: a `RegularExpressionLiteral` is permitted. + InputElementRegExp, + /// `InputElementRegExpOrTemplateTail`: a `RegularExpressionLiteral`, + /// `TemplateMiddle` or `TemplateTail` is permitted. + InputElementRegExpOrTemplateTail, + /// `InputElementTemplateTail`: a `TemplateMiddle` or `TemplateTail` is + /// permitted. + InputElementTemplateTail, + /// `InputElementHashbangOrRegExp`: the start of a `Script` or `Module`. + InputElementHashbangOrRegExp, +} + +/// The kind of a template literal lexical component (§12.9.6). +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum TemplateKind { + /// `NoSubstitutionTemplate :: ` TemplateCharacters? ```. + NoSubstitution, + /// `TemplateHead :: ` TemplateCharacters? ${`. + Head, + /// `TemplateMiddle :: } TemplateCharacters? ${`. + Middle, + /// `TemplateTail :: } TemplateCharacters? ```. + Tail, +} + +/// The input elements a goal symbol derives (§12). The standard's +/// alternatives that are tokens named by `CommonToken` are listed +/// separately, so that a result says which one it is. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum ElementKind { + /// `WhiteSpace`. + WhiteSpace, + /// `LineTerminator`. + LineTerminator, + /// `Comment`, and Annex B's `HTMLCloseComment` where the goal has it. + Comment, + /// `HashbangComment`. + HashbangComment, + /// `IdentifierName` (`CommonToken`). + IdentifierName, + /// `PrivateIdentifier` (`CommonToken`). + PrivateIdentifier, + /// `Punctuator` (`CommonToken`). + Punctuator, + /// `NumericLiteral` (`CommonToken`). + NumericLiteral, + /// `StringLiteral` (`CommonToken`). + StringLiteral, + /// `Template` (`CommonToken`): `NoSubstitution` or `Head`; or + /// `TemplateSubstitutionTail`: `Middle` or `Tail`. + Template(TemplateKind), + /// `DivPunctuator`. + DivPunctuator, + /// `RightBracePunctuator`. + RightBracePunctuator, + /// `RegularExpressionLiteral`. + RegularExpressionLiteral, +} + +// --------------------------------------------------------------------- +// §12.2 White Space; §12.3 Line Terminators +// --------------------------------------------------------------------- + +/// Table "White Space Code Points" (§12.2), the `` row: any code point +/// in general category "Space_Separator". The category is the closed list +/// of Zs code points (Unicode 6.3 and later). +pub open spec fn unicode_space_separator(c: u32) -> bool { + c == 0x0020 || c == 0x00A0 || c == 0x1680 || (0x2000 <= c <= 0x200A) || c == 0x202F + || c == 0x205F || c == 0x3000 +} + +/// §12.2 `WhiteSpace :: | | | | `. +pub open spec fn white_space(s: Src, i: int, j: int) -> bool { + nonempty(s, i, j) && j == i + 1 && ( + // 1. + s[i] == 0x0009 + // 2. + || s[i] == 0x000B + // 3. + || s[i] == 0x000C + // 4. + || s[i] == 0xFEFF + // 5. + || unicode_space_separator(s[i])) +} + +/// §12.3 `LineTerminator :: | | | ` as a code point. +pub open spec fn is_line_terminator(c: u32) -> bool { + // , , , + c == 0x000A || c == 0x000D || c == 0x2028 || c == 0x2029 +} + +/// §12.3 `LineTerminator`. +pub open spec fn line_terminator(s: Src, i: int, j: int) -> bool { + nonempty(s, i, j) && j == i + 1 && is_line_terminator(s[i]) +} + +/// §12.3 `LineTerminatorSequence`. +pub open spec fn line_terminator_sequence(s: Src, i: int, j: int) -> bool { + nonempty(s, i, j) && ( + // 1. + (j == i + 1 && s[i] == 0x000A) + // 2. [lookahead != ] + || (j == i + 1 && s[i] == 0x000D && !at(s, j, '\n')) + // 3. + || (j == i + 1 && s[i] == 0x2028) + // 4. + || (j == i + 1 && s[i] == 0x2029) + // 5. + || (j == i + 2 && s[i] == 0x000D && s[i + 1] == 0x000A)) +} + +/// `SourceCharacter but not LineTerminator`. +pub open spec fn is_not_line_terminator(c: u32) -> bool { + is_source_character(c) && !is_line_terminator(c) +} + +// --------------------------------------------------------------------- +// §12.4 Comments, §12.5 Hashbang Comments +// --------------------------------------------------------------------- + +/// §12.4 `SingleLineCommentChars :: SingleLineCommentChar +/// SingleLineCommentChars?`, with `SingleLineCommentChar :: SourceCharacter +/// but not LineTerminator`: one or more such code points (the right +/// recursion of a list of a one-code-point class is that class, repeated). +pub open spec fn single_line_comment_chars(s: Src, i: int, j: int) -> bool { + nonempty(s, i, j) && all_of(s, i, j, |c: u32| is_not_line_terminator(c)) +} + +/// §12.4 `SingleLineComment :: `//` SingleLineCommentChars?`. +pub open spec fn single_line_comment(s: Src, i: int, j: int) -> bool { + is_lit(s, i, i + 2, seq!['/', '/']) && j >= i + 2 + && opt(i + 2, j, single_line_comment_chars(s, i + 2, j)) +} + +/// §12.4 `MultiLineCommentChars`. +pub open spec fn multi_line_comment_chars(s: Src, i: int, j: int) -> bool + decreases j - i, 0int, +{ + nonempty(s, i, j) && ( + // 1. MultiLineNotAsteriskChar MultiLineCommentChars? + (is_source_character(s[i]) && s[i] != '*' as u32 + && (j == i + 1 || multi_line_comment_chars(s, i + 1, j))) + // 2. `*` PostAsteriskCommentChars? + || (s[i] == '*' as u32 && (j == i + 1 || post_asterisk_comment_chars(s, i + 1, j)))) +} + +/// §12.4 `PostAsteriskCommentChars`. +pub open spec fn post_asterisk_comment_chars(s: Src, i: int, j: int) -> bool + decreases j - i, 1int, +{ + nonempty(s, i, j) && ( + // 1. MultiLineNotForwardSlashOrAsteriskChar MultiLineCommentChars? + (is_source_character(s[i]) && s[i] != '/' as u32 && s[i] != '*' as u32 + && (j == i + 1 || multi_line_comment_chars(s, i + 1, j))) + // 2. `*` PostAsteriskCommentChars? + || (s[i] == '*' as u32 && (j == i + 1 || post_asterisk_comment_chars(s, i + 1, j)))) +} + +/// §12.4 `MultiLineComment :: `/*` MultiLineCommentChars? `*/``. +pub open spec fn multi_line_comment(s: Src, i: int, j: int) -> bool { + is_lit(s, i, i + 2, seq!['/', '*']) && is_lit(s, j - 2, j, seq!['*', '/']) && i + 2 <= j - 2 + && opt(i + 2, j - 2, multi_line_comment_chars(s, i + 2, j - 2)) +} + +/// §12.5 `HashbangComment :: `#!` SingleLineCommentChars?`. The goal symbols +/// that have it apply it at index 0 only (see [`element_matches`]). +pub open spec fn hashbang_comment(s: Src, i: int, j: int) -> bool { + is_lit(s, i, i + 2, seq!['#', '!']) && j >= i + 2 + && opt(i + 2, j, single_line_comment_chars(s, i + 2, j)) +} + +// Annex B.1.1 HTML-like Comments (the grammar that extends §12.4). + +/// Annex B.1.1 `SingleLineDelimitedCommentChars`. +pub open spec fn single_line_delimited_comment_chars(s: Src, i: int, j: int) -> bool + decreases j - i, 0int, +{ + nonempty(s, i, j) && ( + // 1. SingleLineNotAsteriskChar SingleLineDelimitedCommentChars? + // with SingleLineNotAsteriskChar :: SourceCharacter but not one of + // `*` or LineTerminator + (is_not_line_terminator(s[i]) && s[i] != '*' as u32 + && (j == i + 1 || single_line_delimited_comment_chars(s, i + 1, j))) + // 2. `*` SingleLinePostAsteriskCommentChars? + || (s[i] == '*' as u32 + && (j == i + 1 || single_line_post_asterisk_comment_chars(s, i + 1, j)))) +} + +/// Annex B.1.1 `SingleLinePostAsteriskCommentChars`. +pub open spec fn single_line_post_asterisk_comment_chars(s: Src, i: int, j: int) -> bool + decreases j - i, 1int, +{ + nonempty(s, i, j) && ( + // 1. SingleLineNotForwardSlashOrAsteriskChar + // SingleLineDelimitedCommentChars? + // with SingleLineNotForwardSlashOrAsteriskChar :: SourceCharacter + // but not one of `/` or `*` or LineTerminator + (is_not_line_terminator(s[i]) && s[i] != '/' as u32 && s[i] != '*' as u32 + && (j == i + 1 || single_line_delimited_comment_chars(s, i + 1, j))) + // 2. `*` SingleLinePostAsteriskCommentChars? + || (s[i] == '*' as u32 + && (j == i + 1 || single_line_post_asterisk_comment_chars(s, i + 1, j)))) +} + +/// Annex B.1.1 `SingleLineDelimitedComment :: `/*` +/// SingleLineDelimitedCommentChars? `*/``. +pub open spec fn single_line_delimited_comment(s: Src, i: int, j: int) -> bool { + is_lit(s, i, i + 2, seq!['/', '*']) && is_lit(s, j - 2, j, seq!['*', '/']) && i + 2 <= j - 2 + && opt(i + 2, j - 2, single_line_delimited_comment_chars(s, i + 2, j - 2)) +} + +/// Annex B.1.1 `WhiteSpaceSequence :: WhiteSpace WhiteSpaceSequence?`. +pub open spec fn white_space_sequence(s: Src, i: int, j: int) -> bool + decreases j - i, +{ + nonempty(s, i, j) && white_space(s, i, i + 1) + && (j == i + 1 || white_space_sequence(s, i + 1, j)) +} + +/// Annex B.1.1 `SingleLineDelimitedCommentSequence :: +/// SingleLineDelimitedComment WhiteSpaceSequence? +/// SingleLineDelimitedCommentSequence?`: a delimited comment on `[i, a)`, +/// optional white space on `[a, b)`, and the rest on `[b, j)` (optional). +pub open spec fn single_line_delimited_comment_sequence(s: Src, i: int, j: int) -> bool + decreases j - i, +{ + nonempty(s, i, j) + && exists|a: int, b: int| #![auto] + i < a && a <= b && b <= j && single_line_delimited_comment(s, i, a) + && opt(a, b, white_space_sequence(s, a, b)) + && opt(b, j, single_line_delimited_comment_sequence(s, b, j)) +} + +/// Annex B.1.1 `HTMLCloseComment :: WhiteSpaceSequence? +/// SingleLineDelimitedCommentSequence? `-->` SingleLineCommentChars?`. +pub open spec fn html_close_comment_split(s: Src, i: int, a: int, b: int, j: int) -> bool { + i <= a && a <= b && b + 3 <= j && j <= s.len() + && opt(i, a, white_space_sequence(s, i, a)) + && opt(a, b, single_line_delimited_comment_sequence(s, a, b)) + && is_lit(s, b, b + 3, seq!['-', '-', '>']) + && opt(b + 3, j, single_line_comment_chars(s, b + 3, j)) +} + +/// Annex B.1.1 `HTMLCloseComment`. +pub open spec fn html_close_comment(s: Src, i: int, j: int) -> bool { + nonempty(s, i, j) && exists|a: int, b: int| html_close_comment_split(s, i, a, b, j) +} + +/// Annex B.1.1 `SingleLineHTMLOpenComment :: `