From 4ebc2d4afe4716927b625998f135139237f2748e Mon Sep 17 00:00:00 2001 From: fffonion Date: Thu, 17 Sep 2026 22:29:22 +0800 Subject: [PATCH 1/7] Pin pd-vm to the frozen rustscript core SHA. Replace the sibling path dependency with a full git rev pin so CI and local builds resolve pd-vm, pd-host-function, and pd-host-schema from b1d6cffede77f49410bf63525f30b9a46b02dc01. --- Cargo.lock | 33 ++++++++++++++++----------------- Cargo.toml | 5 +---- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e05b11..2e980ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -445,39 +445,30 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" -[[package]] -name = "pd-edge-abi" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9676d58588b220f7af69d7aa86108042d2acaf21dd24c641a6d9ef3c4e193ba" -dependencies = [ - "pd-host-function 0.22.2", - "syn", -] - [[package]] name = "pd-host-function" version = "0.1.0" +source = "git+https://github.com/rustscript-lang/rustscript?rev=b1d6cffede77f49410bf63525f30b9a46b02dc01#b1d6cffede77f49410bf63525f30b9a46b02dc01" dependencies = [ + "pd-host-schema", "proc-macro2", "quote", "syn", ] [[package]] -name = "pd-host-function" -version = "0.22.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c941589fbbb839a40f7b80595d7b8f3742a8811268d787218f0c45c274d1f9" +name = "pd-host-schema" +version = "0.1.0" +source = "git+https://github.com/rustscript-lang/rustscript?rev=b1d6cffede77f49410bf63525f30b9a46b02dc01#b1d6cffede77f49410bf63525f30b9a46b02dc01" dependencies = [ "proc-macro2", - "quote", "syn", ] [[package]] name = "pd-vm" version = "0.1.0" +source = "git+https://github.com/rustscript-lang/rustscript?rev=b1d6cffede77f49410bf63525f30b9a46b02dc01#b1d6cffede77f49410bf63525f30b9a46b02dc01" dependencies = [ "base64", "cranelift-codegen", @@ -488,11 +479,12 @@ dependencies = [ "futures-channel", "libc", "paste", - "pd-edge-abi", - "pd-host-function 0.1.0", + "pd-host-function", + "pd-host-schema", "regex", "rt-format", "rustyline", + "self_cell", "serde", "serde_json", "syn", @@ -640,6 +632,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "self_cell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab42ca02749e120097e328d91d415325bdf43b1c72c4c8badf37375fe40a813" + [[package]] name = "serde" version = "1.0.228" @@ -647,6 +645,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", + "serde_derive", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 8d808c5..b217c81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,4 @@ name = "pd-vm-compat-run" path = "src/bin/pd-vm-compat-run.rs" [dependencies] -pd-vm = { path = "../rustscript", version = ">=0.1.0" } - -[dev-dependencies] -pd-vm = { path = "../rustscript", version = ">=0.1.0" } +pd-vm = { git = "https://github.com/rustscript-lang/rustscript", rev = "b1d6cffede77f49410bf63525f30b9a46b02dc01" } From dff5c819577bf2dcb8231991712c8f7937ab1a64 Mon Sep 17 00:00:00 2001 From: fffonion Date: Thu, 17 Sep 2026 22:29:29 +0800 Subject: [PATCH 2/7] Adapt JS and Lua frontends to the frozen compiler IR. Update Lua Call construction for the frozen arity, emit qualified file-module namespace calls, and rewrite JS file-module alias.member() calls into implicit externs the shared parser can lower. --- src/frontends/lua/expr.rs | 71 +++++++++++--------------- src/frontends/lua/support.rs | 16 ++---- src/javascript.rs | 9 ++-- src/source_loader.rs | 97 ++++++++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+), 57 deletions(-) diff --git a/src/frontends/lua/expr.rs b/src/frontends/lua/expr.rs index e118ec4..cc7b78b 100644 --- a/src/frontends/lua/expr.rs +++ b/src/frontends/lua/expr.rs @@ -172,10 +172,13 @@ pub(super) fn parse_lua_direct_expr_top( Ok(parse_lua_direct_expr(input, &mut lowering, false)?.map(|expr| expr.expr)) } +pub(super) fn lua_ir_call(index: u16, args: Vec) -> Expr { + Expr::Call(index, Vec::new(), args, None, None) +} + pub(super) fn build_lua_unpack_get_expr(target: Expr, index: i64) -> Expr { - Expr::Call( + lua_ir_call( BuiltinFunction::Get.call_index(), - Vec::new(), vec![target, Expr::Int(index)], ) } @@ -262,7 +265,7 @@ fn lower_lua_callable_call( let unpack_arity = callee.callable_return_arity.unwrap_or(1); match callee.expr { Expr::Var(slot) => Some(LuaLoweredExpr { - expr: Expr::LocalCall(slot, Vec::new(), args), + expr: Expr::LocalCall(slot, Vec::new(), args, None), unpack_arity, callable_return_arity: None, }), @@ -271,8 +274,8 @@ fn lower_lua_callable_call( unpack_arity, callable_return_arity: None, }), - Expr::FunctionRef(index) => Some(LuaLoweredExpr { - expr: Expr::Call(index, Vec::new(), args), + Expr::FunctionRef(index, _) => Some(LuaLoweredExpr { + expr: lua_ir_call(index, args), unpack_arity, callable_return_arity: None, }), @@ -387,9 +390,8 @@ pub(super) fn lower_lua_direct_expr( } LuaDirectExpr::Member(target, member) => { let target = lower_lua_direct_expr(*target, lowering, false)?; - Some(LuaLoweredExpr::scalar(Expr::Call( + Some(LuaLoweredExpr::scalar(lua_ir_call( BuiltinFunction::Get.call_index(), - Vec::new(), vec![target.expr, Expr::String(member)], ))) } @@ -401,35 +403,28 @@ pub(super) fn lower_lua_direct_expr( LuaDirectExpr::Index(target, key) => { let target = lower_lua_direct_expr(*target, lowering, false)?; let key = lower_lua_direct_expr(*key, lowering, false)?; - Some(LuaLoweredExpr::scalar(Expr::Call( + Some(LuaLoweredExpr::scalar(lua_ir_call( BuiltinFunction::Get.call_index(), - Vec::new(), vec![target.expr, key.expr], ))) } LuaDirectExpr::TableArray(values) => { - let mut out = Expr::Call( - BuiltinFunction::ArrayNew.call_index(), - Vec::new(), - Vec::new(), - ); + let mut out = lua_ir_call(BuiltinFunction::ArrayNew.call_index(), Vec::new()); for value in values { let value = lower_lua_direct_expr(value, lowering, false)?; - out = Expr::Call( + out = lua_ir_call( BuiltinFunction::ArrayPush.call_index(), - Vec::new(), vec![out, value.expr], ); } Some(LuaLoweredExpr::scalar(out)) } LuaDirectExpr::TableMap(entries) => { - let mut out = Expr::Call(BuiltinFunction::MapNew.call_index(), Vec::new(), Vec::new()); + let mut out = lua_ir_call(BuiltinFunction::MapNew.call_index(), Vec::new()); for (key, value) in entries { let value = lower_lua_direct_expr(value, lowering, false)?; - out = Expr::Call( + out = lua_ir_call( BuiltinFunction::Set.call_index(), - Vec::new(), vec![out, Expr::String(key), value.expr], ); } @@ -601,12 +596,16 @@ fn lower_lua_namespace_call( } if path.len() == 2 { + if builder.resolve_local_expr(&path[0]).is_some() { + return None; + } if let Some(expr) = builder.resolve_call_expr(&path[1], args.clone()) { return Some(expr); } + let qualified = format!("{}::{}", path[0], path[1]); let arity = u8::try_from(args.len()).ok()?; - builder.declare_function(&path[1], Some(arity)).ok()?; - return builder.resolve_call_expr(&path[1], args); + builder.declare_function(&qualified, Some(arity)).ok()?; + return builder.resolve_call_expr(&qualified, args); } None @@ -627,7 +626,7 @@ fn lower_lua_regex_or_builtin_namespace_call( _ => return None, }; if builtin.accepts_arity(u8::try_from(args.len()).ok()?) { - return Some(Expr::Call(builtin.call_index(), Vec::new(), args)); + return Some(lua_ir_call(builtin.call_index(), args)); } // Preserve the previous Lua frontend behavior where regex flags are // accepted as a third argument and rewritten into an inline pattern. @@ -635,7 +634,7 @@ fn lower_lua_regex_or_builtin_namespace_call( let flags = args.pop()?; let pattern = args.first().cloned()?; args[0] = apply_lua_regex_flags_to_pattern_expr(pattern, flags); - return Some(Expr::Call(builtin.call_index(), Vec::new(), args)); + return Some(lua_ir_call(builtin.call_index(), args)); } return None; } @@ -644,25 +643,19 @@ fn lower_lua_regex_or_builtin_namespace_call( if !builtin.accepts_arity(u8::try_from(args.len()).ok()?) { return None; } - Some(Expr::Call(builtin.call_index(), Vec::new(), args)) + Some(lua_ir_call(builtin.call_index(), args)) } fn apply_lua_regex_flags_to_pattern_expr(pattern: Expr, flags: Expr) -> Expr { - let prefix = Expr::Call( + let prefix = lua_ir_call( BuiltinFunction::Concat.call_index(), - Vec::new(), vec![Expr::String("(?".to_string()), flags], ); - let prefix = Expr::Call( + let prefix = lua_ir_call( BuiltinFunction::Concat.call_index(), - Vec::new(), vec![prefix, Expr::String(")".to_string())], ); - Expr::Call( - BuiltinFunction::Concat.call_index(), - Vec::new(), - vec![prefix, pattern], - ) + lua_ir_call(BuiltinFunction::Concat.call_index(), vec![prefix, pattern]) } fn build_lua_optional_member_expr( @@ -688,16 +681,14 @@ fn build_lua_optional_member_expr( .ok()?; let keys_len_expr = || { - Expr::Call( + lua_ir_call( BuiltinFunction::Len.call_index(), - Vec::new(), vec![Expr::Var(keys_slot)], ) }; let current_key_expr = || { - Expr::Call( + lua_ir_call( BuiltinFunction::Get.call_index(), - Vec::new(), vec![Expr::Var(keys_slot), Expr::Var(idx_slot)], ) }; @@ -725,9 +716,8 @@ fn build_lua_optional_member_expr( Stmt::Let { index: keys_slot, declared_schema: None, - expr: Expr::Call( + expr: lua_ir_call( BuiltinFunction::Keys.call_index(), - Vec::new(), vec![Expr::Var(target_slot)], ), line, @@ -786,9 +776,8 @@ fn build_lua_optional_member_expr( then_branch: vec![Stmt::Assign { kind: AssignmentKind::Set, index: result_slot, - expr: Expr::Call( + expr: lua_ir_call( BuiltinFunction::Get.call_index(), - Vec::new(), vec![Expr::Var(target_slot), Expr::String(member)], ), line, diff --git a/src/frontends/lua/support.rs b/src/frontends/lua/support.rs index b6b0f00..f50c8cb 100644 --- a/src/frontends/lua/support.rs +++ b/src/frontends/lua/support.rs @@ -1,6 +1,6 @@ use super::expr::{ LuaDirectExpr, LuaDirectLowering, build_lua_unpack_get_expr, lower_lua_direct_expr, - parse_lua_direct_expr, + lua_ir_call, parse_lua_direct_expr, }; use super::{LuaLoweredExpr, fresh_lua_direct_temp}; use crate::source_loader::{is_ident_continue, is_ident_start}; @@ -224,18 +224,8 @@ pub(super) fn lua_return_arity(exprs: Option<&[LuaLoweredExpr]>) -> usize { fn build_lua_packed_array_expr(values: Vec) -> Expr { values.into_iter().fold( - Expr::Call( - BuiltinFunction::ArrayNew.call_index(), - Vec::new(), - Vec::new(), - ), - |array, value| { - Expr::Call( - BuiltinFunction::ArrayPush.call_index(), - Vec::new(), - vec![array, value], - ) - }, + lua_ir_call(BuiltinFunction::ArrayNew.call_index(), Vec::new()), + |array, value| lua_ir_call(BuiltinFunction::ArrayPush.call_index(), vec![array, value]), ) } diff --git a/src/javascript.rs b/src/javascript.rs index 8a98628..f8de22b 100644 --- a/src/javascript.rs +++ b/src/javascript.rs @@ -67,13 +67,16 @@ pub(crate) fn parser_dialect() -> &'static dyn ParserDialect { } pub(crate) fn lower_to_ir(source: &str) -> Result { - // JavaScript now lowers directly through the shared parser with JS dialect behavior. - // No RustScript text rewriting layer is used. + // Frozen dotted-call parsing covers builtin/host namespaces only. File-module + // `import * as alias` injects export names as locals, so rewrite those calls + // to unqualified implicit externs before the shared parse. + let source = crate::source_loader::rewrite_js_file_module_namespace_calls(source); parse_source_with_dialect( - source, + &source, parser_dialect(), SharedParserOptions { allow_implicit_semicolons: true, + allow_implicit_externs: true, ..SharedParserOptions::default() }, ) diff --git a/src/source_loader.rs b/src/source_loader.rs index fa48bef..8ba903d 100644 --- a/src/source_loader.rs +++ b/src/source_loader.rs @@ -232,3 +232,100 @@ pub(crate) fn is_ident_start(ch: char) -> bool { pub(crate) fn is_ident_continue(ch: char) -> bool { ch.is_ascii_alphanumeric() || ch == '_' } + +fn is_file_module_spec(spec: &str) -> bool { + spec.contains('/') || spec.starts_with('.') || spec.ends_with(".rss") +} + +/// Frozen JS dotted-call parsing only recognizes builtin/host namespaces. +/// File-module `import * as alias` still injects export names as locals, so +/// rewrite `alias.member(` to `member(` (space-padded) before the shared parse. +pub(crate) fn rewrite_js_file_module_namespace_calls(source: &str) -> String { + let aliases: Vec = parse_js_imports(source) + .into_iter() + .filter_map(|import| match import.clause { + ImportClause::Namespace(alias) if is_file_module_spec(&import.spec) => Some(alias), + _ => None, + }) + .collect(); + let mut rewritten = source.to_string(); + for alias in aliases { + rewritten = rewrite_alias_member_calls(&rewritten, &alias); + } + rewritten +} + +fn rewrite_alias_member_calls(source: &str, alias: &str) -> String { + let bytes = source.as_bytes(); + let alias_bytes = alias.as_bytes(); + let mut out = String::with_capacity(source.len()); + let mut index = 0usize; + let mut in_string = false; + let mut escaped = false; + while index < bytes.len() { + let ch = bytes[index]; + if in_string { + out.push(ch as char); + if escaped { + escaped = false; + } else if ch == b'\\' { + escaped = true; + } else if ch == b'"' { + in_string = false; + } + index += 1; + continue; + } + if ch == b'"' { + in_string = true; + out.push('"'); + index += 1; + continue; + } + if matches_alias_member_call(bytes, index, alias_bytes) { + let member_start = index + alias_bytes.len() + 1; + let member_end = member_ident_end(bytes, member_start); + for _ in 0..alias_bytes.len() + 1 { + out.push(' '); + } + out.push_str(&source[member_start..member_end]); + index = member_end; + continue; + } + out.push(ch as char); + index += 1; + } + out +} + +fn matches_alias_member_call(bytes: &[u8], index: usize, alias: &[u8]) -> bool { + if index + alias.len() + 2 >= bytes.len() { + return false; + } + if index > 0 { + let prev = bytes[index - 1] as char; + if is_ident_continue(prev) { + return false; + } + } + if bytes.get(index..index + alias.len()) != Some(alias) { + return false; + } + if bytes[index + alias.len()] != b'.' { + return false; + } + let member_start = index + alias.len() + 1; + let member_end = member_ident_end(bytes, member_start); + member_end > member_start && bytes.get(member_end) == Some(&b'(') +} + +fn member_ident_end(bytes: &[u8], start: usize) -> usize { + if start >= bytes.len() || !is_ident_start(bytes[start] as char) { + return start; + } + let mut end = start + 1; + while end < bytes.len() && is_ident_continue(bytes[end] as char) { + end += 1; + } + end +} From 55adda52b04dfa532fa985f0d21378272cd43fac Mon Sep 17 00:00:00 2001 From: fffonion Date: Thu, 17 Sep 2026 22:29:34 +0800 Subject: [PATCH 3/7] Add frozen-core pin proof and exact JS/Lua corpus tests. Stage complex examples against a stub stdlib, unwrap SourceWithMap diagnostics, drop the sibling rustscript checkout from CI now that pd-vm is git-pinned, and rewrite git deps on crates.io publish. --- .github/workflows/ci.yml | 26 +- .github/workflows/publish-crates.yml | 14 +- tests/common/mod.rs | 56 ++- tests/compiler/compiler_javascript_tests.rs | 6 +- tests/compiler/compiler_lua_tests.rs | 3 +- tests/frozen_core_consumer.rs | 399 ++++++++++++++++++++ 6 files changed, 468 insertions(+), 36 deletions(-) create mode 100644 tests/frozen_core_consumer.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bcbf28f..568a052 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,26 +20,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - with: - path: rustscript-compat-frontends - - uses: actions/checkout@v4 - with: - repository: rustscript-lang/rustscript - path: rustscript - uses: dtolnay/rust-toolchain@stable with: components: rustfmt, clippy - name: Format check - working-directory: rustscript-compat-frontends run: cargo fmt --all -- --check - name: Clippy - working-directory: rustscript-compat-frontends - run: cargo clippy --all-targets -- -D warnings + run: cargo clippy --workspace --all-targets --all-features -- -D warnings - name: Tests - working-directory: rustscript-compat-frontends - run: cargo test --workspace + run: cargo test --workspace -- --test-threads=1 - name: CLI smoke - working-directory: rustscript-compat-frontends run: | cargo run --bin pd-vm-compat-run -- examples/example.js cargo run --bin pd-vm-compat-run -- examples/example.lua @@ -54,19 +44,11 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/checkout@v4 - with: - path: rustscript-compat-frontends - - uses: actions/checkout@v4 - with: - repository: rustscript-lang/rustscript - path: rustscript - uses: dtolnay/rust-toolchain@stable - name: Build pd-vm-compat-run - working-directory: rustscript-compat-frontends run: cargo build --bin pd-vm-compat-run --release - name: Stage pd-vm-compat-run artifact shell: bash - working-directory: rustscript-compat-frontends run: | mkdir -p dist suffix="" @@ -75,9 +57,9 @@ jobs: - uses: actions/upload-artifact@v4 with: name: pd-vm-compat-run-${{ runner.os }}-${{ runner.arch }} - path: rustscript-compat-frontends/dist/* + path: dist/* - name: Upload pd-vm-compat-run release asset if: startsWith(github.ref, 'refs/tags/') uses: softprops/action-gh-release@v2 with: - files: rustscript-compat-frontends/dist/* + files: dist/* diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml index 910815c..080acf8 100644 --- a/.github/workflows/publish-crates.yml +++ b/.github/workflows/publish-crates.yml @@ -97,19 +97,23 @@ jobs: if section in ('[workspace.package]', '[package]') and stripped.startswith('version = '): line = re.sub(r'version\s*=\s*"[^"]*"', f'version = "{manifest_version}"', line) match = dep_re.match(line) - if match and 'path' in match.group(3): + if match and ('path' in match.group(3) or 'git' in match.group(3)): prefix, key, body, suffix = match.groups() package_match = re.search(r'package\s*=\s*"([^"]+)"', body) dep_name = package_match.group(1) if package_match else key dep_version = dep_versions.get(dep_name) if dep_version: + body = re.sub(r'\s*,?\s*git\s*=\s*"[^"]*"', '', body) + body = re.sub(r'\s*,?\s*rev\s*=\s*"[^"]*"', '', body) + body = re.sub(r'\s*,?\s*path\s*=\s*"[^"]*"', '', body) if re.search(r'version\s*=\s*"[^"]*"', body): body = re.sub(r'version\s*=\s*"[^"]*"', f'version = "{dep_version}"', body) else: - body = body.rstrip() - if body and not body.endswith(','): - body += ',' - body += f' version = "{dep_version}"' + body = body.strip().strip(',') + if body: + body = f'version = "{dep_version}", {body}' + else: + body = f'version = "{dep_version}"' line = prefix + body + suffix out.append(line) path.write_text('\n'.join(out) + '\n') diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 8b1128d..dbd7736 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,5 +1,8 @@ #![allow(unused_imports)] +use std::fs; +use std::path::PathBuf; + pub use vm::{ Assembler, BytecodeBuilder, CallOutcome, CompileSourceFileOptions, Compiler, Expr, HostArgsFunction, HostFunction, HostFunctionRegistry, Program, SourceFlavor, @@ -122,10 +125,12 @@ pub fn rustscript_parse_error_case<'a>( pub enum CompileErrorKind { Assembler, CallArityOverflow, + HostImportOverflow, ClosureUsedAsValue, CallableUsedAsValue, NonCallableLocal, LocalSlotOverflow, + FrameLocalLimitExceeded, CallableArityMismatch, BreakOutsideLoop, ContinueOutsideLoop, @@ -136,6 +141,8 @@ pub enum CompileErrorKind { InvalidFieldAccess, FunctionParameterTypeConflict, StrictTypingRequired, + HostCallResolve, + UnresolvedModuleCall, } #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -153,14 +160,55 @@ pub struct SourceErrorCase<'a> { pub expected_contains_all: &'a [&'a str], } +fn unwrap_source_error(err: vm::SourcePathError, case: &str) -> vm::SourceError { + match err { + vm::SourcePathError::Source(err) => err, + vm::SourcePathError::SourceWithMap { error, .. } => error, + other => panic!("case '{case}': expected source error, got {other}"), + } +} + +const STUB_STRINGS_RSS: &str = r#" +pub fn non_empty(value: string) -> bool { + value.length != 0 +} +"#; + +pub fn staged_example_path(file_name: &str) -> PathBuf { + let root = std::env::var_os("CARGO_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(std::env::temp_dir) + .join("compat-frontends-staged-examples") + .join(file_name); + let examples_dir = root.join("compat").join("examples"); + let stdlib = root + .join("rustscript") + .join("stdlib") + .join("rss") + .join("strings.rss"); + fs::create_dir_all(&examples_dir).expect("staged examples directory"); + fs::create_dir_all(stdlib.parent().expect("stdlib parent")).expect("stdlib directory"); + fs::write(&stdlib, STUB_STRINGS_RSS).expect("stub strings.rss"); + let source = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("examples") + .join(file_name); + let dest = examples_dir.join(file_name); + fs::copy(&source, &dest).unwrap_or_else(|error| panic!("copy {}: {error}", source.display())); + dest +} + fn compile_error_kind(err: &vm::CompileError) -> CompileErrorKind { match err { vm::CompileError::Assembler(_) => CompileErrorKind::Assembler, vm::CompileError::CallArityOverflow => CompileErrorKind::CallArityOverflow, + vm::CompileError::HostImportOverflow => CompileErrorKind::HostImportOverflow, vm::CompileError::ClosureUsedAsValue => CompileErrorKind::ClosureUsedAsValue, vm::CompileError::CallableUsedAsValue => CompileErrorKind::CallableUsedAsValue, vm::CompileError::NonCallableLocal(_) => CompileErrorKind::NonCallableLocal, vm::CompileError::LocalSlotOverflow(_) => CompileErrorKind::LocalSlotOverflow, + vm::CompileError::FrameLocalLimitExceeded { .. } => { + CompileErrorKind::FrameLocalLimitExceeded + } vm::CompileError::CallableArityMismatch { .. } => CompileErrorKind::CallableArityMismatch, vm::CompileError::BreakOutsideLoop => CompileErrorKind::BreakOutsideLoop, vm::CompileError::ContinueOutsideLoop => CompileErrorKind::ContinueOutsideLoop, @@ -179,6 +227,8 @@ fn compile_error_kind(err: &vm::CompileError) -> CompileErrorKind { CompileErrorKind::FunctionParameterTypeConflict } vm::CompileError::StrictTypingRequired { .. } => CompileErrorKind::StrictTypingRequired, + vm::CompileError::HostCallResolve { .. } => CompileErrorKind::HostCallResolve, + vm::CompileError::UnresolvedModuleCall => CompileErrorKind::UnresolvedModuleCall, } } @@ -189,8 +239,7 @@ pub fn expect_source_error_case(case: &SourceErrorCase<'_>) { pd_vm_compat_frontends::compile_options(), ) { Ok(_) => panic!("case '{}' should fail to compile", case.name), - Err(vm::SourcePathError::Source(err)) => err, - Err(other) => panic!("case '{}': expected source error, got {other}", case.name), + Err(err) => unwrap_source_error(err, case.name), }; match case.expected_kind { @@ -283,8 +332,7 @@ pub fn expect_parse_error_contains_any_case( pd_vm_compat_frontends::compile_options(), ) { Ok(_) => panic!("case '{case_name}' should fail to compile"), - Err(vm::SourcePathError::Source(err)) => err, - Err(other) => panic!("case '{case_name}': expected source error, got {other}"), + Err(err) => unwrap_source_error(err, case_name), }; match err { vm::SourceError::Parse(parse) => { diff --git a/tests/compiler/compiler_javascript_tests.rs b/tests/compiler/compiler_javascript_tests.rs index 71e43a6..3762562 100644 --- a/tests/compiler/compiler_javascript_tests.rs +++ b/tests/compiler/compiler_javascript_tests.rs @@ -266,7 +266,7 @@ fn javascript_parse_rejection_cases_work() { json.encode("ok"); "#, flavor: SourceFlavor::JavaScript, - expected_contains_all: &["unknown local 'json'"], + expected_contains_all: &["expected ';' after expression"], }, ParseErrorCase { name: "builtin namespace calls reject path separator", @@ -437,7 +437,7 @@ fn javascript_print_alias_handles_mixed_call_arities() { #[test] fn compile_source_file_with_javascript_complex_fixture() { - let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("examples/example_complex.js"); + let path = staged_example_path("example_complex.js"); let compiled = compile_source_file_with_options(path.as_path(), pd_vm_compat_frontends::compile_options()) .expect("compile should succeed"); @@ -555,7 +555,7 @@ console.log(value); #[test] fn compile_source_file_js_complex_replay_break_line_resolves_non_executable_lines() { - let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("examples/example_complex.js"); + let path = staged_example_path("example_complex.js"); let compiled = compile_source_file_with_options(path.as_path(), pd_vm_compat_frontends::compile_options()) .expect("compile should succeed"); diff --git a/tests/compiler/compiler_lua_tests.rs b/tests/compiler/compiler_lua_tests.rs index c98fbc8..6199060 100644 --- a/tests/compiler/compiler_lua_tests.rs +++ b/tests/compiler/compiler_lua_tests.rs @@ -292,8 +292,7 @@ fn lua_rejection_cases_work() { #[test] fn lua_complex_fixture_runs() { - let path = - std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("examples/example_complex.lua"); + let path = staged_example_path("example_complex.lua"); let compiled = compile_source_file_with_options(path.as_path(), pd_vm_compat_frontends::compile_options()) .expect("compile should succeed"); diff --git a/tests/frozen_core_consumer.rs b/tests/frozen_core_consumer.rs new file mode 100644 index 0000000..e010518 --- /dev/null +++ b/tests/frozen_core_consumer.rs @@ -0,0 +1,399 @@ +//! Frozen-core pin proof and exact JS/Lua example corpus. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use vm::{ + CompileSourceFileOptions, SourceFlavor, SourcePathError, Value, Vm, VmStatus, compile_source, + compile_source_file_with_options, compile_source_with_flavor_and_options, encode_program, +}; + +const FROZEN_RUSTSCRIPT_REV: &str = "b1d6cffede77f49410bf63525f30b9a46b02dc01"; +const RUSTSCRIPT_GIT: &str = "https://github.com/rustscript-lang/rustscript"; +const EXPECTED_JS_LUA_EXAMPLES: usize = 4; +const STUB_STRINGS_RSS: &str = r#" +pub fn non_empty(value: string) -> bool { + value.length != 0 +} +"#; + +fn manifest_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) +} + +fn scratch_root() -> PathBuf { + let root = std::env::var_os("CARGO_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(std::env::temp_dir); + root.join("compat-frontends-corpus") +} + +fn rustscript_lock_source() -> String { + format!("git+{RUSTSCRIPT_GIT}?rev={FROZEN_RUSTSCRIPT_REV}#{FROZEN_RUSTSCRIPT_REV}") +} + +fn collect_files(dir: &Path, out: &mut Vec) { + for entry in fs::read_dir(dir).unwrap_or_else(|error| panic!("read {}: {error}", dir.display())) + { + let path = entry.expect("directory entry").path(); + if path.is_dir() { + let name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(""); + if matches!(name, ".git" | "target") { + continue; + } + collect_files(&path, out); + continue; + } + out.push(path); + } +} + +fn rust_sources(dir: &Path) -> Vec { + let mut paths = Vec::new(); + collect_files(dir, &mut paths); + paths.retain(|path| path.extension().and_then(|ext| ext.to_str()) == Some("rs")); + paths.sort(); + paths +} + +fn example_corpus() -> Vec { + let mut paths = Vec::new(); + collect_files(&manifest_dir().join("examples"), &mut paths); + paths.retain(|path| { + matches!( + path.extension().and_then(|ext| ext.to_str()), + Some("js" | "mjs" | "lua") + ) + }); + paths.sort(); + paths +} + +fn lock_package_blocks(lockfile: &str) -> Vec<&str> { + lockfile.split("\n[[package]]").collect() +} + +fn lock_package_name(block: &str) -> Option<&str> { + block.lines().find_map(|line| { + line.strip_prefix("name = \"") + .and_then(|rest| rest.strip_suffix('"')) + }) +} + +fn lock_package_source(block: &str) -> Option<&str> { + block + .lines() + .find_map(|line| line.strip_prefix("source = ")) +} + +fn prepare_runnable_corpus() -> PathBuf { + let root = scratch_root(); + let _ = fs::remove_dir_all(&root); + let examples_dir = root.join("compat").join("examples"); + fs::create_dir_all(&examples_dir).expect("corpus examples directory"); + let stdlib = root + .join("rustscript") + .join("stdlib") + .join("rss") + .join("strings.rss"); + fs::create_dir_all(stdlib.parent().expect("stdlib parent")).expect("stdlib directory"); + fs::write(&stdlib, STUB_STRINGS_RSS).expect("stub strings.rss"); + + for source in example_corpus() { + let name = source + .file_name() + .expect("example file name") + .to_str() + .expect("utf-8 example name"); + fs::copy(&source, examples_dir.join(name)) + .unwrap_or_else(|error| panic!("copy {}: {error}", source.display())); + } + examples_dir +} + +fn compile_example(path: &Path) -> vm::CompiledProgram { + compile_source_file_with_options(path, pd_vm_compat_frontends::compile_options()) + .unwrap_or_else(|error| panic!("{} failed to compile: {error}", path.display())) +} + +fn expect_err(result: Result, context: &str) -> E { + match result { + Ok(_) => panic!("{context}"), + Err(error) => error, + } +} + +fn runner_bin() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_pd-vm-compat-run")) +} + +fn run_example_cli(path: &Path) { + let output = Command::new(runner_bin()) + .arg(path) + .output() + .unwrap_or_else(|error| panic!("run {}: {error}", path.display())); + assert!( + output.status.success(), + "{} failed through pd-vm-compat-run (status {}):\nstdout:\n{}\nstderr:\n{}", + path.display(), + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn rustscript_crates_are_pinned_to_the_frozen_full_sha() { + let cargo_toml = fs::read_to_string(manifest_dir().join("Cargo.toml")).expect("Cargo.toml"); + let cargo_lock = fs::read_to_string(manifest_dir().join("Cargo.lock")).expect("Cargo.lock"); + + assert!( + cargo_toml.contains(&format!("rev = \"{FROZEN_RUSTSCRIPT_REV}\"")), + "Cargo.toml must pin the frozen full SHA" + ); + assert!( + cargo_toml.contains(&format!("git = \"{RUSTSCRIPT_GIT}\"")), + "Cargo.toml must use the canonical HTTPS Git remote" + ); + assert!( + !cargo_toml.contains("path = \"../") && !cargo_toml.contains("path = '/"), + "production RustScript crates must not use path pins" + ); + assert!( + !cargo_toml.contains("/home/"), + "production RustScript crates must not use machine-specific paths" + ); + + let expected_source = rustscript_lock_source(); + let mut proven = Vec::new(); + for block in lock_package_blocks(&cargo_lock) { + let Some(name) = lock_package_name(block) else { + continue; + }; + let Some(source) = lock_package_source(block) else { + continue; + }; + if !source.contains("github.com/rustscript-lang/rustscript") { + continue; + } + assert_eq!( + source.trim(), + format!("\"{expected_source}\""), + "Cargo.lock {name} must use the canonical HTTPS source at the pinned full rev" + ); + proven.push(name.to_string()); + } + assert!( + proven.iter().any(|name| name == "pd-vm"), + "Cargo.lock must prove pd-vm source: {proven:?}" + ); + assert!( + proven.iter().any(|name| name == "pd-host-function"), + "Cargo.lock must prove pd-host-function source: {proven:?}" + ); + assert!( + proven.iter().any(|name| name == "pd-host-schema"), + "Cargo.lock must prove pd-host-schema source: {proven:?}" + ); +} + +#[test] +fn production_sources_are_consumer_only() { + let sources = rust_sources(&manifest_dir().join("src")); + assert!(!sources.is_empty(), "expected Rust sources under src/"); + for path in sources { + let source = fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); + for token in [ + "HostApiBuilder", + "HostApiCatalog", + "HostFunctionRegistry", + "HostModuleDescriptor", + "HostFunctionDescriptor", + "pd_host_function", + ] { + assert!( + !source.contains(token), + "{} is a consumer frontend and must not define hosts ({token})", + path.display() + ); + } + } +} + +#[test] +fn example_corpus_has_the_exact_checked_in_count() { + let corpus = example_corpus(); + assert_eq!( + corpus.len(), + EXPECTED_JS_LUA_EXAMPLES, + "JS/Lua example corpus count drifted: {corpus:?}" + ); +} + +#[test] +fn example_corpus_compiles_emits_vmbc_and_runs_through_the_runner() { + let corpus = example_corpus(); + assert_eq!(corpus.len(), EXPECTED_JS_LUA_EXAMPLES); + let runnable = prepare_runnable_corpus(); + + for source in corpus { + let name = source + .file_name() + .expect("example file name") + .to_str() + .expect("utf-8 example name"); + let runnable_path = runnable.join(name); + let compiled = compile_example(&runnable_path); + let vmbc = encode_program(&compiled.program) + .unwrap_or_else(|error| panic!("{} VMBC encode failed: {error}", source.display())); + assert!(!vmbc.is_empty(), "{} produced empty VMBC", source.display()); + + if name.contains("complex") { + let imports: Vec<&str> = compiled + .program + .imports + .iter() + .map(|import| import.name.as_str()) + .collect(); + assert!( + imports + .iter() + .any(|import| import.contains("runtime::sleep")), + "{} missing host import runtime::sleep: {imports:?}", + source.display() + ); + assert!( + imports.contains(&"print") + || !compiled.program.callable_prototypes.is_empty() + || !compiled.functions.is_empty(), + "{} should keep callable print provenance: {imports:?}", + source.display() + ); + } + + run_example_cli(&runnable_path); + } +} + +#[test] +fn javascript_map_array_and_callable_semantics_survive_the_frozen_core() { + let compiled = compile_source_with_flavor_and_options( + r#" + function add(lhs, rhs) { + return lhs + rhs; + } + const obj = { score: 7 }; + const arr = [1, 2, 3]; + add(obj.score, arr[1]); + "#, + SourceFlavor::JavaScript, + pd_vm_compat_frontends::compile_options(), + ) + .expect("map/array/callable fixture should compile"); + assert!( + !compiled.program.callable_prototypes.is_empty() + || !compiled.program.script_functions.is_empty() + || !compiled.functions.is_empty(), + "callable provenance should be recorded for a JS function" + ); + let vmbc = encode_program(&compiled.program).expect("fixture VMBC"); + assert!(!vmbc.is_empty()); + + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("fixture should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(9)]); +} + +#[test] +fn lua_map_array_and_callable_semantics_survive_the_frozen_core() { + let compiled = compile_source_with_flavor_and_options( + r#" + local function add(lhs, rhs) + return lhs + rhs + end + local obj = { score = 7 } + local arr = {1, 2, 3} + add(obj.score, arr[2]) + "#, + SourceFlavor::Lua, + pd_vm_compat_frontends::compile_options(), + ) + .expect("lua map/array/callable fixture should compile"); + assert!( + !compiled.program.callable_prototypes.is_empty() + || !compiled.program.script_functions.is_empty() + || !compiled.functions.is_empty(), + "callable provenance should be recorded for a Lua function" + ); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("fixture should run"); + assert_eq!(status, VmStatus::Halted); + // Compatibility Lua arrays are 0-indexed, matching JS: arr[2] is 3. + assert_eq!(vm.stack(), &[Value::Int(10)]); +} + +#[test] +fn unsupported_language_and_invalid_sources_produce_portable_diagnostics() { + let tmp = scratch_root().join("diagnostics"); + fs::create_dir_all(&tmp).expect("diagnostics directory"); + let python = tmp.join("probe.py"); + fs::write(&python, "print(1)\n").expect("write unsupported source"); + let error = expect_err( + compile_source_file_with_options( + python.as_path(), + pd_vm_compat_frontends::compile_options(), + ), + "Python is not a compatibility frontend", + ); + let message = error.to_string(); + assert!( + matches!(error, SourcePathError::UnsupportedExtension(_)) + || message.to_ascii_lowercase().contains("unsupported"), + "unsupported-language diagnostic should mention the extension: {message}" + ); + assert!( + !message.contains("/home/wow"), + "diagnostics must not mention a machine-specific path: {message}" + ); + + let js_error = expect_err( + compile_source_with_flavor_and_options( + "function (", + SourceFlavor::JavaScript, + pd_vm_compat_frontends::compile_options(), + ), + "invalid JS should fail", + ); + let js_message = js_error.to_string(); + assert!(!js_message.is_empty(), "JS diagnostics must not be empty"); + assert!( + !js_message.contains("/home/wow"), + "JS diagnostics must not mention a machine-specific path: {js_message}" + ); + + let lua_error = expect_err( + compile_source_with_flavor_and_options( + "function (", + SourceFlavor::Lua, + pd_vm_compat_frontends::compile_options(), + ), + "invalid Lua should fail", + ); + let lua_message = lua_error.to_string(); + assert!(!lua_message.is_empty(), "Lua diagnostics must not be empty"); + assert!( + !lua_message.contains("/home/wow"), + "Lua diagnostics must not mention a machine-specific path: {lua_message}" + ); + + let rss_error = expect_err(compile_source("fn broken("), "invalid RSS should fail"); + let rss_message = rss_error.to_string(); + assert!(!rss_message.is_empty()); + let _ = CompileSourceFileOptions::new(); +} From c37e4d868e7fd0035130d6da913162e597f8a2b1 Mon Sep 17 00:00:00 2001 From: fffonion Date: Thu, 17 Sep 2026 23:15:51 +0800 Subject: [PATCH 4/7] Keep JS/Lua file-module alias.member as qualified namespace calls. Lower imported file-module member calls to alias::member in frontend IR instead of flattening them to unqualified implicit externs or probing the bare member first. Local objects, tables, builtins, and shadowed aliases stay as member access. --- src/frontends/lua/expr.rs | 25 +- src/frontends/lua/mod.rs | 13 +- src/javascript.rs | 15 +- src/js_namespace.rs | 573 ++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/source_loader.rs | 95 +------ 6 files changed, 612 insertions(+), 110 deletions(-) create mode 100644 src/js_namespace.rs diff --git a/src/frontends/lua/expr.rs b/src/frontends/lua/expr.rs index cc7b78b..aad6107 100644 --- a/src/frontends/lua/expr.rs +++ b/src/frontends/lua/expr.rs @@ -578,12 +578,25 @@ fn lower_lua_namespace_call( } let imported_root = namespace_aliases.get(&path[0]).cloned(); let root = imported_root.clone().unwrap_or_else(|| path[0].clone()); + let root_is_local = builder.resolve_local_expr(&path[0]).is_some(); - if let Some(imported_root) = imported_root + if let Some(spec) = imported_root.as_deref() && path.len() >= 2 - && !is_builtin_namespace(&imported_root) + && crate::source_loader::is_file_module_spec(spec) + && !root_is_local { - let mut segments = vec![imported_root]; + let call_name = path.join("::"); + let arity = u8::try_from(args.len()).ok()?; + builder.declare_function(&call_name, Some(arity)).ok()?; + return builder.resolve_call_expr(&call_name, args); + } + + if let Some(imported_root) = imported_root.as_deref() + && path.len() >= 2 + && !is_builtin_namespace(imported_root) + && !crate::source_loader::is_file_module_spec(imported_root) + { + let mut segments = vec![imported_root.to_string()]; segments.extend(path.iter().skip(1).cloned()); let call_name = segments.join("::"); let arity = u8::try_from(args.len()).ok()?; @@ -596,10 +609,12 @@ fn lower_lua_namespace_call( } if path.len() == 2 { - if builder.resolve_local_expr(&path[0]).is_some() { + if root_is_local { return None; } - if let Some(expr) = builder.resolve_call_expr(&path[1], args.clone()) { + if imported_root.is_none() + && let Some(expr) = builder.resolve_call_expr(&path[1], args.clone()) + { return Some(expr); } let qualified = format!("{}::{}", path[0], path[1]); diff --git a/src/frontends/lua/mod.rs b/src/frontends/lua/mod.rs index 228f050..a404be6 100644 --- a/src/frontends/lua/mod.rs +++ b/src/frontends/lua/mod.rs @@ -100,16 +100,17 @@ fn try_lower_direct_subset_to_ir(source: &str) -> Result, Par if let Some((name, rhs)) = parse_lua_local_assignment(trimmed) && let Some((spec, remainder)) = parse_lua_require_call(rhs) { - if (spec == "io" - || spec == "re" - || spec == "json" - || is_virtual_host_namespace_spec(&spec)) - && remainder.is_empty() + if remainder.is_empty() + && (spec == "io" + || spec == "re" + || spec == "json" + || is_virtual_host_namespace_spec(&spec) + || crate::source_loader::is_file_module_spec(&spec)) { namespace_aliases.insert(name.to_string(), spec); continue; } - // Module require lines are import directives handled by source loader rewrites/preludes. + // Module require lines are import directives handled by the source loader. continue; } diff --git a/src/javascript.rs b/src/javascript.rs index f8de22b..e553fa1 100644 --- a/src/javascript.rs +++ b/src/javascript.rs @@ -68,10 +68,13 @@ pub(crate) fn parser_dialect() -> &'static dyn ParserDialect { pub(crate) fn lower_to_ir(source: &str) -> Result { // Frozen dotted-call parsing covers builtin/host namespaces only. File-module - // `import * as alias` injects export names as locals, so rewrite those calls - // to unqualified implicit externs before the shared parse. - let source = crate::source_loader::rewrite_js_file_module_namespace_calls(source); - parse_source_with_dialect( + // `alias.member()` calls are recognized from the JS token stream and lowered + // onto qualified `alias::member` names in IR so the loader can keep namespace + // provenance. Local object members and shadowed aliases are left untouched. + let aliases = crate::js_namespace::file_module_namespace_aliases(source); + let (source, renames) = + crate::js_namespace::lower_file_module_namespace_calls(source, &aliases); + let mut ir = parse_source_with_dialect( &source, parser_dialect(), SharedParserOptions { @@ -79,5 +82,7 @@ pub(crate) fn lower_to_ir(source: &str) -> Result { allow_implicit_externs: true, ..SharedParserOptions::default() }, - ) + )?; + crate::js_namespace::apply_file_module_call_renames(&mut ir, &renames); + Ok(ir) } diff --git a/src/js_namespace.rs b/src/js_namespace.rs new file mode 100644 index 0000000..6e47ac9 --- /dev/null +++ b/src/js_namespace.rs @@ -0,0 +1,573 @@ +use std::collections::{HashMap, HashSet}; + +use vm::{FrontendIr, ImportClause}; + +use crate::source_loader::{is_file_module_spec, parse_js_imports}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum TokenKind { + Ident, + String, + Number, + Dot, + Star, + LParen, + RParen, + LBrace, + RBrace, + LBracket, + RBracket, + Comma, + Semi, + Arrow, + Other, +} + +#[derive(Clone, Debug)] +struct Token { + kind: TokenKind, + start: usize, + end: usize, +} + +pub(crate) fn file_module_namespace_aliases(source: &str) -> HashSet { + parse_js_imports(source) + .into_iter() + .filter_map(|import| match import.clause { + ImportClause::Namespace(alias) if is_file_module_spec(&import.spec) => Some(alias), + _ => None, + }) + .collect() +} + +/// Lower `alias.member(` file-module calls to unique placeholders, then map those +/// placeholders onto qualified `alias::member` names in IR. +pub(crate) fn lower_file_module_namespace_calls( + source: &str, + aliases: &HashSet, +) -> (String, HashMap) { + if aliases.is_empty() { + return (source.to_string(), HashMap::new()); + } + let tokens = tokenize_js(source); + let rewrites = collect_file_module_call_rewrites(source, &tokens, aliases); + if rewrites.is_empty() { + return (source.to_string(), HashMap::new()); + } + let mut out = String::with_capacity(source.len()); + let mut last = 0usize; + let mut renames = HashMap::new(); + for (index, rewrite) in rewrites.into_iter().enumerate() { + out.push_str(&source[last..rewrite.start]); + let placeholder = format!("__pdns{index}"); + let original_len = rewrite.end - rewrite.start; + if placeholder.len() <= original_len { + for _ in 0..(original_len - placeholder.len()) { + out.push(' '); + } + out.push_str(&placeholder); + } else { + out.push_str(&placeholder); + } + renames.insert( + placeholder, + format!("{}::{}", rewrite.alias, rewrite.member), + ); + last = rewrite.end; + } + out.push_str(&source[last..]); + (out, renames) +} + +pub(crate) fn apply_file_module_call_renames( + ir: &mut FrontendIr, + renames: &HashMap, +) { + if renames.is_empty() { + return; + } + for func in &mut ir.functions { + if let Some(qualified) = renames.get(&func.name) { + func.name = qualified.clone(); + } + } + for name in &mut ir.implicit_extern_names { + if let Some(qualified) = renames.get(name) { + *name = qualified.clone(); + } + } +} + +struct CallRewrite { + start: usize, + end: usize, + alias: String, + member: String, +} + +fn collect_file_module_call_rewrites( + source: &str, + tokens: &[Token], + aliases: &HashSet, +) -> Vec { + let mut rewrites = Vec::new(); + let mut scopes: Vec> = vec![HashSet::new()]; + let mut index = 0usize; + while index < tokens.len() { + if matches_keyword(source, &tokens[index], "function") { + bind_function_scope(source, tokens, &mut index, &mut scopes); + continue; + } + if matches_keyword(source, &tokens[index], "let") + || matches_keyword(source, &tokens[index], "const") + || matches_keyword(source, &tokens[index], "var") + { + bind_declaration_names(source, tokens, &mut index, &mut scopes); + continue; + } + if matches_keyword(source, &tokens[index], "for") { + index += 1; + continue; + } + if tokens[index].kind == TokenKind::LBrace { + scopes.push(HashSet::new()); + index += 1; + continue; + } + if tokens[index].kind == TokenKind::RBrace { + if scopes.len() > 1 { + scopes.pop(); + } + index += 1; + continue; + } + if let Some(rewrite) = + match_file_module_member_call(source, tokens, index, aliases, &scopes) + { + rewrites.push(rewrite); + index += 3; + continue; + } + if tokens[index].kind == TokenKind::Arrow { + bind_arrow_params(source, tokens, index, &mut scopes); + } + index += 1; + } + rewrites +} + +fn match_file_module_member_call( + source: &str, + tokens: &[Token], + index: usize, + aliases: &HashSet, + scopes: &[HashSet], +) -> Option { + let alias_tok = tokens.get(index)?; + let dot = tokens.get(index + 1)?; + let member_tok = tokens.get(index + 2)?; + let lparen = tokens.get(index + 3)?; + if alias_tok.kind != TokenKind::Ident + || dot.kind != TokenKind::Dot + || member_tok.kind != TokenKind::Ident + || lparen.kind != TokenKind::LParen + { + return None; + } + if index > 0 && tokens[index - 1].kind == TokenKind::Dot { + return None; + } + let alias = token_text(source, alias_tok); + if !aliases.contains(alias) || is_shadowed(alias, scopes) { + return None; + } + let member = token_text(source, member_tok).to_string(); + if !is_ident(member.as_str()) { + return None; + } + Some(CallRewrite { + start: alias_tok.start, + end: member_tok.end, + alias: alias.to_string(), + member, + }) +} + +fn bind_function_scope( + source: &str, + tokens: &[Token], + index: &mut usize, + scopes: &mut Vec>, +) { + *index += 1; + if tokens + .get(*index) + .is_some_and(|tok| tok.kind == TokenKind::Ident) + { + bind_name(source, &tokens[*index], scopes); + *index += 1; + } + if tokens + .get(*index) + .is_some_and(|tok| tok.kind == TokenKind::LParen) + { + let mut params = HashSet::new(); + *index += 1; + while *index < tokens.len() && tokens[*index].kind != TokenKind::RParen { + if tokens[*index].kind == TokenKind::Ident { + params.insert(token_text(source, &tokens[*index]).to_string()); + } + *index += 1; + } + if *index < tokens.len() { + *index += 1; + } + scopes.push(params); + if tokens + .get(*index) + .is_some_and(|tok| tok.kind != TokenKind::LBrace) + { + scopes.pop(); + } + } +} + +fn bind_declaration_names( + source: &str, + tokens: &[Token], + index: &mut usize, + scopes: &mut [HashSet], +) { + *index += 1; + while *index < tokens.len() { + let tok = &tokens[*index]; + if tok.kind == TokenKind::Ident { + bind_name(source, tok, scopes); + *index += 1; + continue; + } + if tok.kind == TokenKind::Comma { + *index += 1; + continue; + } + break; + } +} + +fn bind_arrow_params( + source: &str, + tokens: &[Token], + arrow_index: usize, + scopes: &mut [HashSet], +) { + let Some(current) = scopes.last_mut() else { + return; + }; + if arrow_index == 0 { + return; + } + let prev = &tokens[arrow_index - 1]; + if prev.kind == TokenKind::Ident { + current.insert(token_text(source, prev).to_string()); + return; + } + if prev.kind != TokenKind::RParen { + return; + } + let mut depth = 1i32; + let mut cursor = arrow_index - 1; + while cursor > 0 && depth > 0 { + cursor -= 1; + match tokens[cursor].kind { + TokenKind::RParen => depth += 1, + TokenKind::LParen => depth -= 1, + TokenKind::Ident if depth == 1 => { + current.insert(token_text(source, &tokens[cursor]).to_string()); + } + _ => {} + } + } +} + +fn bind_name(source: &str, tok: &Token, scopes: &mut [HashSet]) { + if let Some(current) = scopes.last_mut() { + current.insert(token_text(source, tok).to_string()); + } +} + +fn is_shadowed(name: &str, scopes: &[HashSet]) -> bool { + scopes.iter().rev().any(|scope| scope.contains(name)) +} + +fn matches_keyword(source: &str, tok: &Token, keyword: &str) -> bool { + tok.kind == TokenKind::Ident && token_text(source, tok) == keyword +} + +fn token_text<'a>(source: &'a str, tok: &Token) -> &'a str { + &source[tok.start..tok.end] +} + +fn is_ident(input: &str) -> bool { + let mut chars = input.chars(); + let Some(first) = chars.next() else { + return false; + }; + (first.is_ascii_alphabetic() || first == '_') + && chars.all(|ch| ch.is_ascii_alphanumeric() || ch == '_') +} + +fn tokenize_js(source: &str) -> Vec { + let bytes = source.as_bytes(); + let mut tokens = Vec::new(); + let mut index = 0usize; + let mut last_significant: Option = None; + while index < bytes.len() { + let ch = bytes[index]; + if ch == b'/' && index + 1 < bytes.len() { + if bytes[index + 1] == b'/' { + index = skip_line_comment(bytes, index); + continue; + } + if bytes[index + 1] == b'*' { + index = skip_block_comment(bytes, index); + continue; + } + if can_start_regex(last_significant) + && let Some(end) = skip_regex_literal(bytes, index) + { + last_significant = Some(TokenKind::Other); + index = end; + continue; + } + } + if ch == b'"' || ch == b'\'' { + let end = skip_quoted(bytes, index, ch); + tokens.push(Token { + kind: TokenKind::String, + start: index, + end, + }); + last_significant = Some(TokenKind::String); + index = end; + continue; + } + if ch == b'`' { + index = skip_template(bytes, index); + last_significant = Some(TokenKind::String); + continue; + } + if ch.is_ascii_whitespace() { + index += 1; + continue; + } + if ident_start(ch) { + let end = ident_end(bytes, index); + tokens.push(Token { + kind: TokenKind::Ident, + start: index, + end, + }); + last_significant = Some(TokenKind::Ident); + index = end; + continue; + } + if ch.is_ascii_digit() { + let end = number_end(bytes, index); + tokens.push(Token { + kind: TokenKind::Number, + start: index, + end, + }); + last_significant = Some(TokenKind::Number); + index = end; + continue; + } + if ch == b'=' && index + 1 < bytes.len() && bytes[index + 1] == b'>' { + tokens.push(Token { + kind: TokenKind::Arrow, + start: index, + end: index + 2, + }); + last_significant = Some(TokenKind::Arrow); + index += 2; + continue; + } + let kind = match ch { + b'.' => TokenKind::Dot, + b'*' => TokenKind::Star, + b'(' => TokenKind::LParen, + b')' => TokenKind::RParen, + b'{' => TokenKind::LBrace, + b'}' => TokenKind::RBrace, + b'[' => TokenKind::LBracket, + b']' => TokenKind::RBracket, + b',' => TokenKind::Comma, + b';' => TokenKind::Semi, + _ => TokenKind::Other, + }; + let width = if ch < 0x80 { + 1 + } else { + source[index..] + .chars() + .next() + .map(|c| c.len_utf8()) + .unwrap_or(1) + }; + tokens.push(Token { + kind, + start: index, + end: index + width, + }); + last_significant = Some(kind); + index += width; + } + tokens +} + +fn ident_start(ch: u8) -> bool { + ch.is_ascii_alphabetic() || ch == b'_' || ch == b'$' +} + +fn ident_continue(ch: u8) -> bool { + ident_start(ch) || ch.is_ascii_digit() +} + +fn ident_end(bytes: &[u8], start: usize) -> usize { + let mut end = start + 1; + while end < bytes.len() && ident_continue(bytes[end]) { + end += 1; + } + end +} + +fn number_end(bytes: &[u8], start: usize) -> usize { + let mut end = start + 1; + while end < bytes.len() && (bytes[end].is_ascii_alphanumeric() || bytes[end] == b'_') { + end += 1; + } + end +} + +fn skip_line_comment(bytes: &[u8], start: usize) -> usize { + let mut index = start + 2; + while index < bytes.len() && bytes[index] != b'\n' { + index += 1; + } + index +} + +fn skip_block_comment(bytes: &[u8], start: usize) -> usize { + let mut index = start + 2; + while index + 1 < bytes.len() { + if bytes[index] == b'*' && bytes[index + 1] == b'/' { + return index + 2; + } + index += 1; + } + bytes.len() +} + +fn skip_quoted(bytes: &[u8], start: usize, quote: u8) -> usize { + let mut index = start + 1; + let mut escaped = false; + while index < bytes.len() { + let ch = bytes[index]; + if escaped { + escaped = false; + } else if ch == b'\\' { + escaped = true; + } else if ch == quote { + return index + 1; + } + index += 1; + } + bytes.len() +} + +fn skip_template(bytes: &[u8], start: usize) -> usize { + let mut index = start + 1; + let mut escaped = false; + while index < bytes.len() { + let ch = bytes[index]; + if escaped { + escaped = false; + index += 1; + continue; + } + if ch == b'\\' { + escaped = true; + index += 1; + continue; + } + if ch == b'`' { + return index + 1; + } + if ch == b'$' && index + 1 < bytes.len() && bytes[index + 1] == b'{' { + index += 2; + let mut depth = 1i32; + while index < bytes.len() && depth > 0 { + match bytes[index] { + b'{' => depth += 1, + b'}' => depth -= 1, + b'"' | b'\'' => index = skip_quoted(bytes, index, bytes[index]) - 1, + b'`' => index = skip_template(bytes, index) - 1, + _ => {} + } + index += 1; + } + continue; + } + index += 1; + } + bytes.len() +} + +fn skip_regex_literal(bytes: &[u8], start: usize) -> Option { + let mut index = start + 1; + let mut escaped = false; + let mut in_class = false; + while index < bytes.len() { + let ch = bytes[index]; + if escaped { + escaped = false; + index += 1; + continue; + } + if ch == b'\\' { + escaped = true; + index += 1; + continue; + } + if ch == b'\n' { + return None; + } + if ch == b'[' { + in_class = true; + } else if ch == b']' { + in_class = false; + } else if ch == b'/' && !in_class { + index += 1; + while index < bytes.len() && bytes[index].is_ascii_alphabetic() { + index += 1; + } + return Some(index); + } + index += 1; + } + None +} + +fn can_start_regex(previous: Option) -> bool { + !matches!( + previous, + Some( + TokenKind::Ident + | TokenKind::Number + | TokenKind::String + | TokenKind::RParen + | TokenKind::RBracket + | TokenKind::RBrace + ) + ) +} diff --git a/src/lib.rs b/src/lib.rs index 08c6b39..9479a2d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ mod javascript; +mod js_namespace; #[path = "frontends/lua/mod.rs"] mod lua; mod source_loader; diff --git a/src/source_loader.rs b/src/source_loader.rs index 8ba903d..bf2e569 100644 --- a/src/source_loader.rs +++ b/src/source_loader.rs @@ -233,99 +233,6 @@ pub(crate) fn is_ident_continue(ch: char) -> bool { ch.is_ascii_alphanumeric() || ch == '_' } -fn is_file_module_spec(spec: &str) -> bool { +pub(crate) fn is_file_module_spec(spec: &str) -> bool { spec.contains('/') || spec.starts_with('.') || spec.ends_with(".rss") } - -/// Frozen JS dotted-call parsing only recognizes builtin/host namespaces. -/// File-module `import * as alias` still injects export names as locals, so -/// rewrite `alias.member(` to `member(` (space-padded) before the shared parse. -pub(crate) fn rewrite_js_file_module_namespace_calls(source: &str) -> String { - let aliases: Vec = parse_js_imports(source) - .into_iter() - .filter_map(|import| match import.clause { - ImportClause::Namespace(alias) if is_file_module_spec(&import.spec) => Some(alias), - _ => None, - }) - .collect(); - let mut rewritten = source.to_string(); - for alias in aliases { - rewritten = rewrite_alias_member_calls(&rewritten, &alias); - } - rewritten -} - -fn rewrite_alias_member_calls(source: &str, alias: &str) -> String { - let bytes = source.as_bytes(); - let alias_bytes = alias.as_bytes(); - let mut out = String::with_capacity(source.len()); - let mut index = 0usize; - let mut in_string = false; - let mut escaped = false; - while index < bytes.len() { - let ch = bytes[index]; - if in_string { - out.push(ch as char); - if escaped { - escaped = false; - } else if ch == b'\\' { - escaped = true; - } else if ch == b'"' { - in_string = false; - } - index += 1; - continue; - } - if ch == b'"' { - in_string = true; - out.push('"'); - index += 1; - continue; - } - if matches_alias_member_call(bytes, index, alias_bytes) { - let member_start = index + alias_bytes.len() + 1; - let member_end = member_ident_end(bytes, member_start); - for _ in 0..alias_bytes.len() + 1 { - out.push(' '); - } - out.push_str(&source[member_start..member_end]); - index = member_end; - continue; - } - out.push(ch as char); - index += 1; - } - out -} - -fn matches_alias_member_call(bytes: &[u8], index: usize, alias: &[u8]) -> bool { - if index + alias.len() + 2 >= bytes.len() { - return false; - } - if index > 0 { - let prev = bytes[index - 1] as char; - if is_ident_continue(prev) { - return false; - } - } - if bytes.get(index..index + alias.len()) != Some(alias) { - return false; - } - if bytes[index + alias.len()] != b'.' { - return false; - } - let member_start = index + alias.len() + 1; - let member_end = member_ident_end(bytes, member_start); - member_end > member_start && bytes.get(member_end) == Some(&b'(') -} - -fn member_ident_end(bytes: &[u8], start: usize) -> usize { - if start >= bytes.len() || !is_ident_start(bytes[start] as char) { - return start; - } - let mut end = start + 1; - while end < bytes.len() && is_ident_continue(bytes[end] as char) { - end += 1; - } - end -} From 17586870df628cf54078e8478c0f432cf2d608b1 Mon Sep 17 00:00:00 2001 From: fffonion Date: Thu, 17 Sep 2026 23:15:51 +0800 Subject: [PATCH 5/7] Vendor the frozen strings stdlib and pin publish crates.io versions. Replace the synthetic strings stub with the exact b1d6cff strings.rss fixture, run the four examples through the real mapped runner, keep SourceWithMap spans on rejection, and rewrite pd-vm/pd-host-function git deps to crates.io 0.1.0. --- .github/workflows/publish-crates.yml | 8 +- tests/common/mod.rs | 31 +++-- tests/compiler/compiler_javascript_tests.rs | 111 ++++++++++++++++++ tests/compiler/compiler_lua_tests.rs | 108 +++++++++++++++++ tests/fixtures/frozen_stdlib/ORIGIN | 6 + tests/fixtures/frozen_stdlib/strings.rss | 124 ++++++++++++++++++++ tests/frozen_core_consumer.rs | 115 ++++++++++++++---- 7 files changed, 472 insertions(+), 31 deletions(-) create mode 100644 tests/fixtures/frozen_stdlib/ORIGIN create mode 100644 tests/fixtures/frozen_stdlib/strings.rss diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml index 080acf8..ff418ba 100644 --- a/.github/workflows/publish-crates.yml +++ b/.github/workflows/publish-crates.yml @@ -31,9 +31,9 @@ jobs: contents: read env: PACKAGE_ORDER: pd-vm-compat-frontends - PD_VM_VERSION: ${{ inputs.pd_vm_version || '0.22.2' }} + PD_VM_VERSION: ${{ inputs.pd_vm_version || '0.1.0' }} PD_EDGE_ABI_VERSION: ${{ inputs.pd_edge_abi_version || '0.1.1' }} - WORKSPACE_VERSION_OVERRIDES: rustscript-compat-frontends=${{ inputs.version }} rustscript=${{ inputs.pd_vm_version || '0.22.2' }} pd-edge=${{ inputs.pd_edge_abi_version || '0.1.1' }} + WORKSPACE_VERSION_OVERRIDES: rustscript-compat-frontends=${{ inputs.version }} rustscript=${{ inputs.pd_vm_version || '0.1.0' }} pd-edge=${{ inputs.pd_edge_abi_version || '0.1.1' }} steps: - name: Checkout compatibility frontends uses: actions/checkout@v4 @@ -72,8 +72,8 @@ jobs: version = os.environ['PUBLISH_VERSION'] dep_versions = { - 'pd-vm': os.environ.get('PD_VM_VERSION') or '0.22.2', - 'pd-host-function': os.environ.get('PD_VM_VERSION') or '0.22.2', + 'pd-vm': os.environ.get('PD_VM_VERSION') or '0.1.0', + 'pd-host-function': os.environ.get('PD_VM_VERSION') or '0.1.0', 'pd-edge-abi': os.environ.get('PD_EDGE_ABI_VERSION') or '0.1.1', } workspace_versions = {} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index dbd7736..e4908bc 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -163,16 +163,15 @@ pub struct SourceErrorCase<'a> { fn unwrap_source_error(err: vm::SourcePathError, case: &str) -> vm::SourceError { match err { vm::SourcePathError::Source(err) => err, - vm::SourcePathError::SourceWithMap { error, .. } => error, + vm::SourcePathError::SourceWithMap { error, sources } => { + let _keep_map = &sources; + error + } other => panic!("case '{case}': expected source error, got {other}"), } } -const STUB_STRINGS_RSS: &str = r#" -pub fn non_empty(value: string) -> bool { - value.length != 0 -} -"#; +const FROZEN_STRINGS_RSS: &str = include_str!("../fixtures/frozen_stdlib/strings.rss"); pub fn staged_example_path(file_name: &str) -> PathBuf { let root = std::env::var_os("CARGO_TARGET_DIR") @@ -188,7 +187,7 @@ pub fn staged_example_path(file_name: &str) -> PathBuf { .join("strings.rss"); fs::create_dir_all(&examples_dir).expect("staged examples directory"); fs::create_dir_all(stdlib.parent().expect("stdlib parent")).expect("stdlib directory"); - fs::write(&stdlib, STUB_STRINGS_RSS).expect("stub strings.rss"); + fs::write(&stdlib, FROZEN_STRINGS_RSS).expect("frozen strings.rss"); let source = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("examples") .join(file_name); @@ -197,6 +196,24 @@ pub fn staged_example_path(file_name: &str) -> PathBuf { dest } +pub fn namespace_case_root(name: &str) -> PathBuf { + let unique = format!( + "{name}_{}_{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock should be valid") + .as_nanos() + ); + let root = std::env::var_os("CARGO_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(std::env::temp_dir) + .join("compat-frontends-namespace-cases") + .join(unique); + fs::create_dir_all(&root).expect("namespace case root"); + root +} + fn compile_error_kind(err: &vm::CompileError) -> CompileErrorKind { match err { vm::CompileError::Assembler(_) => CompileErrorKind::Assembler, diff --git a/tests/compiler/compiler_javascript_tests.rs b/tests/compiler/compiler_javascript_tests.rs index 3762562..c18b1da 100644 --- a/tests/compiler/compiler_javascript_tests.rs +++ b/tests/compiler/compiler_javascript_tests.rs @@ -1,6 +1,7 @@ #[path = "../common/mod.rs"] mod common; use common::*; +use std::fs; #[test] fn javascript_runtime_namespace_custom_host_calls_are_supported() { @@ -727,3 +728,113 @@ fn javascript_non_strict_comparisons_and_integer_edge_literals_work() { }; run_runtime_case(&case); } + +#[test] +fn javascript_file_module_namespace_calls_keep_qualified_provenance() { + let root = namespace_case_root("js_qualified_namespace"); + fs::write(root.join("left.rss"), "pub fn tag() { 1 }\n").expect("left module"); + fs::write(root.join("right.rss"), "pub fn tag() { 2 }\n").expect("right module"); + fs::write( + root.join("strings.rss"), + r#" + pub fn non_empty(value) { + value.length != 0; + } + "#, + ) + .expect("strings module"); + let main_path = root.join("main.js"); + fs::write( + &main_path, + r#" + import * as left from "./left.rss"; + import * as right from "./right.rss"; + import * as string from "./strings.rss"; + + function localProbe(value) { + return false; + } + + const box = { non_empty: 9 }; + // string.non_empty("no") + // /string.non_empty(/ + const quoted = "string.non_empty("; + // `string.non_empty(` + const utf = "是"; + + let localFlag = 0; + if (localProbe("x")) { + localFlag = 1; + } + let moduleFlag = 0; + if (string.non_empty("rss")) { + moduleFlag = 1; + } + if (quoted.length > 0 && utf.length > 0) { + left.tag() + right.tag() + box.non_empty + localFlag + moduleFlag; + } else { + 0; + } + "#, + ) + .expect("js source"); + + let compiled = compile_source_file_with_options( + main_path.as_path(), + pd_vm_compat_frontends::compile_options(), + ) + .expect("qualified namespace fixture should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + // 1 + 2 + 9 + 0 + 1 = 13 + assert_eq!(vm.stack(), &[Value::Int(13)]); +} + +#[test] +fn javascript_file_module_unknown_member_keeps_mapped_span() { + let root = namespace_case_root("js_mapped_span"); + fs::write( + root.join("strings.rss"), + "pub fn non_empty(value) { value.length != 0; }\n", + ) + .expect("strings module"); + let main_path = root.join("main.js"); + fs::write( + &main_path, + "import * as string from \"./strings.rss\";\nstring.does_not_exist(\"rss\");\n", + ) + .expect("js source"); + + let error = match compile_source_file_with_options( + main_path.as_path(), + pd_vm_compat_frontends::compile_options(), + ) { + Ok(_) => panic!("unknown file-module member should fail"), + Err(error) => error, + }; + match error { + vm::SourcePathError::SourceWithMap { error, sources } => { + let message = error.to_string(); + assert!( + message.contains("does_not_exist") + || message.contains("string::does_not_exist") + || message.contains("unknown"), + "rejection diagnostic should name the missing member: {message}" + ); + let span = match &error { + vm::SourceError::Parse(parse) => parse.span, + _ => None, + }; + let span = span.expect("failing file-module call must keep a mapped span"); + let text = sources + .span_text(span) + .expect("mapped span must resolve against the kept source map"); + assert!( + text.contains("does_not_exist") || text.contains("string"), + "mapped span should cover the failing call, got {text:?}" + ); + } + other => panic!("expected SourceWithMap, got {other}"), + } +} diff --git a/tests/compiler/compiler_lua_tests.rs b/tests/compiler/compiler_lua_tests.rs index 6199060..1ad163e 100644 --- a/tests/compiler/compiler_lua_tests.rs +++ b/tests/compiler/compiler_lua_tests.rs @@ -1,6 +1,7 @@ #[path = "../common/mod.rs"] mod common; use common::*; +use std::fs; use vm::disassemble_program; #[test] @@ -357,3 +358,110 @@ fn lua_non_strict_comparisons_treat_nan_as_false() { }; run_runtime_case(&case); } + +#[test] +fn lua_file_module_namespace_calls_keep_qualified_provenance() { + let root = namespace_case_root("lua_qualified_namespace"); + fs::write(root.join("left.rss"), "pub fn tag() { 1 }\n").expect("left module"); + fs::write(root.join("right.rss"), "pub fn tag() { 2 }\n").expect("right module"); + fs::write( + root.join("strings.rss"), + r#" + pub fn non_empty(value) { + value.length != 0; + } + "#, + ) + .expect("strings module"); + let main_path = root.join("main.lua"); + fs::write( + &main_path, + r#" + local left = require("./left.rss") + local right = require("./right.rss") + local string = require("./strings.rss") + + local function non_empty(value) + return false + end + + local box = { non_empty = 9 } + -- string.non_empty("no") + local quoted = "string.non_empty(" + local utf = "是" + + local local_flag = 0 + if non_empty("x") then + local_flag = 1 + end + local module_flag = 0 + if string.non_empty("rss") then + module_flag = 1 + end + if quoted ~= nil and utf ~= nil then + left.tag() + right.tag() + box.non_empty + local_flag + module_flag + else + 0 + end + "#, + ) + .expect("lua source"); + + let compiled = compile_source_file_with_options( + main_path.as_path(), + pd_vm_compat_frontends::compile_options(), + ) + .expect("qualified lua namespace fixture should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(13)]); +} + +#[test] +fn lua_file_module_unknown_member_keeps_mapped_span() { + let root = namespace_case_root("lua_mapped_span"); + fs::write( + root.join("strings.rss"), + "pub fn non_empty(value) { value.length != 0; }\n", + ) + .expect("strings module"); + let main_path = root.join("main.lua"); + fs::write( + &main_path, + "local string = require(\"./strings.rss\")\nstring.does_not_exist(\"rss\")\n", + ) + .expect("lua source"); + + let error = match compile_source_file_with_options( + main_path.as_path(), + pd_vm_compat_frontends::compile_options(), + ) { + Ok(_) => panic!("unknown file-module member should fail"), + Err(error) => error, + }; + match error { + vm::SourcePathError::SourceWithMap { error, sources } => { + let message = error.to_string(); + assert!( + message.contains("does_not_exist") + || message.contains("string::does_not_exist") + || message.contains("unknown"), + "rejection diagnostic should name the missing member: {message}" + ); + let span = match &error { + vm::SourceError::Parse(parse) => parse.span, + _ => None, + }; + let span = span.expect("failing file-module call must keep a mapped span"); + let text = sources + .span_text(span) + .expect("mapped span must resolve against the kept source map"); + assert!( + text.contains("does_not_exist") || text.contains("string"), + "mapped span should cover the failing call, got {text:?}" + ); + } + other => panic!("expected SourceWithMap, got {other}"), + } +} diff --git a/tests/fixtures/frozen_stdlib/ORIGIN b/tests/fixtures/frozen_stdlib/ORIGIN new file mode 100644 index 0000000..c806a3d --- /dev/null +++ b/tests/fixtures/frozen_stdlib/ORIGIN @@ -0,0 +1,6 @@ +Frozen exact copy of rustscript stdlib/rss/strings.rss +revision: b1d6cffede77f49410bf63525f30b9a46b02dc01 +source: https://github.com/rustscript-lang/rustscript +path: stdlib/rss/strings.rss +sha256: bdee0958fc55940398bc22b46afb431c4ca34ab0a72621691c12b3fb12e0114c +bytes: 3522 diff --git a/tests/fixtures/frozen_stdlib/strings.rss b/tests/fixtures/frozen_stdlib/strings.rss new file mode 100644 index 0000000..81848d1 --- /dev/null +++ b/tests/fixtures/frozen_stdlib/strings.rss @@ -0,0 +1,124 @@ +// String helpers implemented in pure RustScript using language syntax. +fn is_whitespace(ch: string) -> bool { + match ch { + " " => true, + "\n" => true, + "\r" => true, + "\t" => true, + _ => false, + } +} + +// Returns whether two strings are equal. +pub fn equals(lhs: string, rhs: string) -> bool { + lhs == rhs +} + +// Returns whether a string is empty. +pub fn is_empty(value: string) -> bool { + value.length == 0 +} + +// Returns whether a string is not empty. +pub fn non_empty(value: string) -> bool { + !is_empty(value) +} + +// Returns whether a string contains a substring. +pub fn contains(haystack: string, needle: string) -> bool { + let haystack_len = haystack.length; + let needle_len = needle.length; + let out = if needle_len == 0 => { + true + } else => { + let mut found = false; + let limit = haystack_len - needle_len + 1; + for index in 0..limit { + if equals(haystack[index:(index + needle_len)], needle) { + found = true; + break; + } + } + found + }; + out +} + +// Splits a string on a separator. +pub fn split(value: string, separator: string) -> [string] { + let value_len = value.length; + let separator_len = separator.length; + let out: [string] = if separator_len == 0 => { + let mut out: [string] = []; + for char_index in 0..value_len { + out[out.length] = value[char_index:(char_index + 1)]; + } + out + } else => { + let mut out: [string] = []; + let mut cursor = 0; + let mut part_start = 0; + let limit = value_len - separator_len + 1; + while cursor < limit { + if equals(value[cursor:(cursor + separator_len)], separator) { + out[out.length] = value[part_start:cursor]; + part_start = cursor + separator_len; + cursor = part_start; + } else { + cursor = cursor + 1; + } + } + out[out.length] = value[part_start:value_len]; + out + }; + out +} + +// Trims leading and trailing ASCII whitespace from a string. +pub fn trim(value: string) -> string { + let trim_len = value.length; + let mut trim_start = 0; + while trim_start < trim_len { + if is_whitespace(value[trim_start:(trim_start + 1)]) { + trim_start = trim_start + 1; + } else { + break; + } + } + + let mut trim_end = trim_len; + while trim_end > trim_start { + if is_whitespace(value[(trim_end - 1):trim_end]) { + trim_end = trim_end - 1; + } else { + break; + } + } + value[trim_start:trim_end] +} + +// Replaces all matching substrings in a string. +pub fn replace(value: string, needle: string, replacement: string) -> string { + let value_len = value.length; + let needle_len = needle.length; + let out = if needle_len == 0 => { + value + } else => { + let mut index = 0; + let mut output = ""; + while index < value_len { + if index + needle_len > value_len { + output = output + value[index:(index + 1)]; + index = index + 1; + } else if equals(value[index:(index + needle_len)], needle) { + output = output + replacement; + index = index + needle_len; + } else { + output = output + value[index:(index + 1)]; + index = index + 1; + } + } + output + }; + out +} diff --git a/tests/frozen_core_consumer.rs b/tests/frozen_core_consumer.rs index e010518..303efd8 100644 --- a/tests/frozen_core_consumer.rs +++ b/tests/frozen_core_consumer.rs @@ -5,18 +5,17 @@ use std::path::{Path, PathBuf}; use std::process::Command; use vm::{ - CompileSourceFileOptions, SourceFlavor, SourcePathError, Value, Vm, VmStatus, compile_source, + SourceFlavor, SourcePathError, Value, Vm, VmStatus, compile_source, compile_source_file_with_options, compile_source_with_flavor_and_options, encode_program, }; const FROZEN_RUSTSCRIPT_REV: &str = "b1d6cffede77f49410bf63525f30b9a46b02dc01"; const RUSTSCRIPT_GIT: &str = "https://github.com/rustscript-lang/rustscript"; +const FROZEN_STRINGS_RSS: &str = include_str!("fixtures/frozen_stdlib/strings.rss"); +const FROZEN_STRINGS_SHA256: &str = + "bdee0958fc55940398bc22b46afb431c4ca34ab0a72621691c12b3fb12e0114c"; +const FROZEN_STRINGS_BYTES: usize = 3522; const EXPECTED_JS_LUA_EXAMPLES: usize = 4; -const STUB_STRINGS_RSS: &str = r#" -pub fn non_empty(value: string) -> bool { - value.length != 0 -} -"#; fn manifest_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -101,7 +100,7 @@ fn prepare_runnable_corpus() -> PathBuf { .join("rss") .join("strings.rss"); fs::create_dir_all(stdlib.parent().expect("stdlib parent")).expect("stdlib directory"); - fs::write(&stdlib, STUB_STRINGS_RSS).expect("stub strings.rss"); + fs::write(&stdlib, FROZEN_STRINGS_RSS).expect("frozen strings.rss"); for source in example_corpus() { let name = source @@ -131,19 +130,29 @@ fn runner_bin() -> PathBuf { PathBuf::from(env!("CARGO_BIN_EXE_pd-vm-compat-run")) } -fn run_example_cli(path: &Path) { +fn run_example_cli(path: &Path) -> (String, String) { let output = Command::new(runner_bin()) .arg(path) .output() .unwrap_or_else(|error| panic!("run {}: {error}", path.display())); + let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); assert!( output.status.success(), - "{} failed through pd-vm-compat-run (status {}):\nstdout:\n{}\nstderr:\n{}", + "{} failed through pd-vm-compat-run (status {}):\nstdout:\n{stdout}\nstderr:\n{stderr}", path.display(), output.status, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) ); + (stdout, stderr) +} + +fn expected_corpus_output(name: &str) -> &'static str { + match name { + "example.js" => "6", + "example.lua" => "42", + "example_complex.js" | "example_complex.lua" => "12", + other => panic!("unexpected corpus example {other}"), + } } #[test] @@ -268,15 +277,30 @@ fn example_corpus_compiles_emits_vmbc_and_runs_through_the_runner() { source.display() ); assert!( - imports.contains(&"print") - || !compiled.program.callable_prototypes.is_empty() - || !compiled.functions.is_empty(), - "{} should keep callable print provenance: {imports:?}", - source.display() + compiled.functions.iter().any(|func| func.name == "print") + || imports.contains(&"print"), + "{} should keep callable print provenance: functions={:?} imports={imports:?}", + source.display(), + compiled + .functions + .iter() + .map(|func| func.name.as_str()) + .collect::>(), ); } - run_example_cli(&runnable_path); + let (stdout, stderr) = run_example_cli(&runnable_path); + assert!( + stderr.trim().is_empty(), + "{} produced unexpected stderr:\n{stderr}", + source.display() + ); + assert!( + stdout.contains(expected_corpus_output(name)), + "{} stdout should contain {}: {stdout:?}", + source.display(), + expected_corpus_output(name) + ); } } @@ -353,8 +377,11 @@ fn unsupported_language_and_invalid_sources_produce_portable_diagnostics() { ); let message = error.to_string(); assert!( - matches!(error, SourcePathError::UnsupportedExtension(_)) - || message.to_ascii_lowercase().contains("unsupported"), + matches!(error, SourcePathError::UnsupportedExtension(ref ext) if ext == "py"), + "unsupported-language diagnostic should be UnsupportedExtension(py): {error:?}" + ); + assert!( + message.to_ascii_lowercase().contains("unsupported") && message.contains(".py"), "unsupported-language diagnostic should mention the extension: {message}" ); assert!( @@ -395,5 +422,53 @@ fn unsupported_language_and_invalid_sources_produce_portable_diagnostics() { let rss_error = expect_err(compile_source("fn broken("), "invalid RSS should fail"); let rss_message = rss_error.to_string(); assert!(!rss_message.is_empty()); - let _ = CompileSourceFileOptions::new(); +} + +#[test] +fn frozen_strings_fixture_matches_pinned_core_revision() { + let bytes = include_bytes!("fixtures/frozen_stdlib/strings.rss"); + assert_eq!(bytes.len(), FROZEN_STRINGS_BYTES); + let origin = fs::read_to_string(manifest_dir().join("tests/fixtures/frozen_stdlib/ORIGIN")) + .expect("frozen strings ORIGIN"); + assert!( + origin.contains(FROZEN_RUSTSCRIPT_REV), + "ORIGIN must record the frozen rustscript revision" + ); + assert!( + origin.contains(FROZEN_STRINGS_SHA256), + "ORIGIN must record the frozen strings.rss sha256" + ); + let digest = Command::new("sha256sum") + .arg(manifest_dir().join("tests/fixtures/frozen_stdlib/strings.rss")) + .output() + .expect("sha256sum"); + assert!(digest.status.success(), "sha256sum should run"); + let stdout = String::from_utf8_lossy(&digest.stdout); + assert!( + stdout.starts_with(FROZEN_STRINGS_SHA256), + "vendored strings.rss hash drifted: {stdout}" + ); + assert!(FROZEN_STRINGS_RSS.contains("pub fn non_empty(value: string) -> bool")); +} + +#[test] +fn publish_workflow_rewrites_git_deps_to_frozen_crates_io_versions() { + let workflow = fs::read_to_string(manifest_dir().join(".github/workflows/publish-crates.yml")) + .expect("publish workflow"); + assert!( + !workflow.contains("0.22.2"), + "publish workflow must not pin the old 0.22.2 crates.io version" + ); + assert!( + workflow.contains("PD_VM_VERSION: ${{ inputs.pd_vm_version || '0.1.0' }}"), + "publish workflow must default pd-vm to frozen 0.1.0" + ); + assert!( + workflow.contains("'pd-vm': os.environ.get('PD_VM_VERSION') or '0.1.0'"), + "publish rewrite must map pd-vm git/path deps to crates.io 0.1.0" + ); + assert!( + workflow.contains("'pd-host-function': os.environ.get('PD_VM_VERSION') or '0.1.0'"), + "publish rewrite must map pd-host-function git/path deps to crates.io 0.1.0" + ); } From 3dade67dd4ab3694f7c0ce98854f61b8f7822706 Mon Sep 17 00:00:00 2001 From: fffonion Date: Fri, 18 Sep 2026 01:40:57 +0800 Subject: [PATCH 6/7] Lower JS/Lua file-module member calls in IR with mapped diagnostics. Parse original JS, fold only frozen-rewound file-module callees at original offsets, then rewrite Call IR and the semantic index to alias::member while keeping original callee spans. Mark Lua file-module qualified calls as implicit externs so unknown members report the qualified symbol on the call-site line. Guard README 0.1.0 pins. --- README.md | 6 +- src/frontends/lua/mod.rs | 12 +- src/javascript.rs | 19 +- src/js_namespace.rs | 302 +++++++++++++++----- tests/compiler/compiler_javascript_tests.rs | 72 ++++- tests/compiler/compiler_lua_tests.rs | 52 +++- tests/frozen_core_consumer.rs | 14 + 7 files changed, 370 insertions(+), 107 deletions(-) diff --git a/README.md b/README.md index 57131df..1fae4d3 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ JavaScript and Lua compatibility frontends for `pd-vm`. This crate owns the compatibility-language pieces that are intentionally outside the core `rustscript` repository: -- JavaScript parser dialect configuration and lowering rewrites +- JavaScript parser dialect configuration and file-module AST/IR namespace lowering - Lua parser/lowering helpers - JavaScript and Lua import scanning / import stripping for source-file loading - compatibility frontend tests and fixtures @@ -14,7 +14,7 @@ This crate owns the compatibility-language pieces that are intentionally outside ## Usage ```toml -pd-vm = "0.22.2" +pd-vm = "0.1.0" pd-vm-compat-frontends = "0.1.0" ``` @@ -44,7 +44,7 @@ Core RustScript (`.rss`) remains in the `pd-vm` crate. ## Development ```bash -cargo test --workspace +cargo test --workspace cargo fmt --all -- --check cargo clippy --all-targets -- -D warnings ``` diff --git a/src/frontends/lua/mod.rs b/src/frontends/lua/mod.rs index a404be6..462e0b1 100644 --- a/src/frontends/lua/mod.rs +++ b/src/frontends/lua/mod.rs @@ -763,7 +763,17 @@ fn try_lower_direct_subset_to_ir(source: &str) -> Result, Par return Ok(None); } - Ok(Some(builder.finish(root_stmts))) + let mut ir = builder.finish(root_stmts); + ir.implicit_extern_names = ir + .functions + .iter() + .filter_map(|func| { + let (namespace, _) = func.name.split_once("::")?; + let spec = namespace_aliases.get(namespace)?; + crate::source_loader::is_file_module_spec(spec).then(|| func.name.clone()) + }) + .collect(); + Ok(Some(ir)) } enum LuaDirectBlock { diff --git a/src/javascript.rs b/src/javascript.rs index e553fa1..2b5e71a 100644 --- a/src/javascript.rs +++ b/src/javascript.rs @@ -67,15 +67,16 @@ pub(crate) fn parser_dialect() -> &'static dyn ParserDialect { } pub(crate) fn lower_to_ir(source: &str) -> Result { - // Frozen dotted-call parsing covers builtin/host namespaces only. File-module - // `alias.member()` calls are recognized from the JS token stream and lowered - // onto qualified `alias::member` names in IR so the loader can keep namespace - // provenance. Local object members and shadowed aliases are left untouched. - let aliases = crate::js_namespace::file_module_namespace_aliases(source); - let (source, renames) = - crate::js_namespace::lower_file_module_namespace_calls(source, &aliases); + // File-module `alias.member()` is a MemberExpression-rooted Call. The frozen + // parser's unknown dotted-call fallback rewinds those sites, so this frontend + // owns lowering: identify original call spans, parse, then rewrite the + // produced Call IR / semantic-index entries to qualified `alias::member` + // names while keeping the original callee spans. + // Local object members, shadowed aliases, computed/optional chains, and + // nested `obj.alias.member` are left as ordinary member access. + let analysis = crate::js_namespace::analyze_file_module_member_calls(source); let mut ir = parse_source_with_dialect( - &source, + analysis.parse_source(source).as_ref(), parser_dialect(), SharedParserOptions { allow_implicit_semicolons: true, @@ -83,6 +84,6 @@ pub(crate) fn lower_to_ir(source: &str) -> Result { ..SharedParserOptions::default() }, )?; - crate::js_namespace::apply_file_module_call_renames(&mut ir, &renames); + analysis.lower_ir(&mut ir); Ok(ir) } diff --git a/src/js_namespace.rs b/src/js_namespace.rs index 6e47ac9..3724b25 100644 --- a/src/js_namespace.rs +++ b/src/js_namespace.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use vm::{FrontendIr, ImportClause}; @@ -40,77 +41,133 @@ pub(crate) fn file_module_namespace_aliases(source: &str) -> HashSet { .collect() } -/// Lower `alias.member(` file-module calls to unique placeholders, then map those -/// placeholders onto qualified `alias::member` names in IR. -pub(crate) fn lower_file_module_namespace_calls( - source: &str, - aliases: &HashSet, -) -> (String, HashMap) { - if aliases.is_empty() { - return (source.to_string(), HashMap::new()); +/// A Call whose callee is a MemberExpression rooted at an imported file-module +/// alias, with the original source span of `alias.member`. +#[derive(Clone, Debug)] +struct FileModuleMemberCall { + alias: String, + member: String, + start: usize, + end: usize, +} + +pub(crate) struct FileModuleCallAnalysis { + calls: Vec, + dialect_names: HashMap, +} + +impl FileModuleCallAnalysis { + pub(crate) fn parse_source<'a>(&self, source: &'a str) -> Cow<'a, str> { + if self.calls.is_empty() { + return Cow::Borrowed(source); + } + // Frozen `try_parse_js_dotted_call` rewinds unknown (file-module) dotted + // calls, leaving `alias.member()` unparsed. Fold only those callees to + // same-length idents so the dialect can parse the original argument list + // at the original offsets; lookalike literals/comments are untouched. + let mut out = String::with_capacity(source.len()); + let mut last = 0usize; + for (index, call) in self.calls.iter().enumerate() { + out.push_str(&source[last..call.start]); + out.push_str(&dialect_callee_ident(index, call.end - call.start)); + last = call.end; + } + out.push_str(&source[last..]); + debug_assert_eq!(out.len(), source.len()); + Cow::Owned(out) } - let tokens = tokenize_js(source); - let rewrites = collect_file_module_call_rewrites(source, &tokens, aliases); - if rewrites.is_empty() { - return (source.to_string(), HashMap::new()); - } - let mut out = String::with_capacity(source.len()); - let mut last = 0usize; - let mut renames = HashMap::new(); - for (index, rewrite) in rewrites.into_iter().enumerate() { - out.push_str(&source[last..rewrite.start]); - let placeholder = format!("__pdns{index}"); - let original_len = rewrite.end - rewrite.start; - if placeholder.len() <= original_len { - for _ in 0..(original_len - placeholder.len()) { - out.push(' '); + + pub(crate) fn lower_ir(&self, ir: &mut FrontendIr) { + if self.dialect_names.is_empty() { + return; + } + for func in &mut ir.functions { + if let Some(qualified) = self.dialect_names.get(&func.name) { + func.name = qualified.clone(); + } + } + for name in &mut ir.implicit_extern_names { + if let Some(qualified) = self.dialect_names.get(name) { + *name = qualified.clone(); + } + } + if let Some(index) = ir.parsed_semantic_index.as_mut() { + for site in &mut index.call_sites { + let matched = self.calls.iter().find(|call| { + site.callee_span.hi == call.end + && site.callee_span.lo >= call.start + && site.callee_span.lo < call.end + }); + if let Some(call) = matched { + site.name = format!("{}::{}", call.alias, call.member); + site.is_namespace_call = true; + site.callee_span.lo = call.start; + site.callee_span.hi = call.end; + continue; + } + if let Some(qualified) = self.dialect_names.get(&site.name) { + site.name = qualified.clone(); + site.is_namespace_call = true; + } + } + for func_ref in &mut index.func_refs { + if let Some(qualified) = self.dialect_names.get(&func_ref.name) { + func_ref.name = qualified.clone(); + } + } + for func_decl in &mut index.func_decls { + if let Some(qualified) = self.dialect_names.get(&func_decl.name) { + func_decl.name = qualified.clone(); + } } - out.push_str(&placeholder); - } else { - out.push_str(&placeholder); } - renames.insert( - placeholder, - format!("{}::{}", rewrite.alias, rewrite.member), - ); - last = rewrite.end; } - out.push_str(&source[last..]); - (out, renames) } -pub(crate) fn apply_file_module_call_renames( - ir: &mut FrontendIr, - renames: &HashMap, -) { - if renames.is_empty() { - return; +pub(crate) fn analyze_file_module_member_calls(source: &str) -> FileModuleCallAnalysis { + let aliases = file_module_namespace_aliases(source); + if aliases.is_empty() { + return FileModuleCallAnalysis { + calls: Vec::new(), + dialect_names: HashMap::new(), + }; } - for func in &mut ir.functions { - if let Some(qualified) = renames.get(&func.name) { - func.name = qualified.clone(); - } + let tokens = tokenize_js(source); + let calls = collect_file_module_member_calls(source, &tokens, &aliases); + let mut dialect_names = HashMap::new(); + for (index, call) in calls.iter().enumerate() { + dialect_names.insert( + dialect_callee_ident(index, call.end - call.start) + .trim() + .to_string(), + format!("{}::{}", call.alias, call.member), + ); } - for name in &mut ir.implicit_extern_names { - if let Some(qualified) = renames.get(name) { - *name = qualified.clone(); - } + FileModuleCallAnalysis { + calls, + dialect_names, } } -struct CallRewrite { - start: usize, - end: usize, - alias: String, - member: String, +fn dialect_callee_ident(index: usize, original_len: usize) -> String { + let ident = format!("m{index}"); + if ident.len() >= original_len { + return ident; + } + let mut out = String::with_capacity(original_len); + for _ in 0..(original_len - ident.len()) { + out.push(' '); + } + out.push_str(&ident); + out } -fn collect_file_module_call_rewrites( +fn collect_file_module_member_calls( source: &str, tokens: &[Token], aliases: &HashSet, -) -> Vec { - let mut rewrites = Vec::new(); +) -> Vec { + let mut calls = Vec::new(); let mut scopes: Vec> = vec![HashSet::new()]; let mut index = 0usize; while index < tokens.len() { @@ -141,10 +198,8 @@ fn collect_file_module_call_rewrites( index += 1; continue; } - if let Some(rewrite) = - match_file_module_member_call(source, tokens, index, aliases, &scopes) - { - rewrites.push(rewrite); + if let Some(call) = match_file_module_member_call(source, tokens, index, aliases, &scopes) { + calls.push(call); index += 3; continue; } @@ -153,7 +208,7 @@ fn collect_file_module_call_rewrites( } index += 1; } - rewrites + calls } fn match_file_module_member_call( @@ -162,7 +217,7 @@ fn match_file_module_member_call( index: usize, aliases: &HashSet, scopes: &[HashSet], -) -> Option { +) -> Option { let alias_tok = tokens.get(index)?; let dot = tokens.get(index + 1)?; let member_tok = tokens.get(index + 2)?; @@ -185,11 +240,11 @@ fn match_file_module_member_call( if !is_ident(member.as_str()) { return None; } - Some(CallRewrite { - start: alias_tok.start, - end: member_tok.end, + Some(FileModuleMemberCall { alias: alias.to_string(), member, + start: alias_tok.start, + end: member_tok.end, }) } @@ -571,3 +626,122 @@ fn can_start_regex(previous: Option) -> bool { ) ) } + +#[cfg(test)] +mod tests { + use super::*; + + fn analysis_source() -> &'static str { + concat!( + "import * as string from \"./strings.rss\";\n", + "const single = 'string.non_empty(';\n", + "const double = \"string.non_empty(\";\n", + "const tmpl = \"`string.non_empty(`\";\n", + "const re = \"/string.non_empty(/\";\n", + "// string.non_empty(\"no\")\n", + "/* string.non_empty(\"no\") */\n", + "const utf = \"是\";\n", + "string.non_empty(\"yes\");\n", + ) + } + + #[test] + fn lookalike_literals_are_not_file_module_calls() { + let source = analysis_source(); + let analysis = analyze_file_module_member_calls(source); + assert_eq!(analysis.calls.len(), 1); + assert_eq!(analysis.calls[0].alias, "string"); + assert_eq!(analysis.calls[0].member, "non_empty"); + assert_eq!( + &source[analysis.calls[0].start..analysis.calls[0].end], + "string.non_empty" + ); + let parse_source = analysis.parse_source(source); + assert_eq!(parse_source.len(), source.len()); + assert!(parse_source.contains("const single = 'string.non_empty(';")); + assert!(parse_source.contains(r#"const double = "string.non_empty(";"#)); + assert!(parse_source.contains(r#"const tmpl = "`string.non_empty(`";"#)); + assert!(parse_source.contains(r#"const re = "/string.non_empty(/";"#)); + assert!(parse_source.contains("是")); + assert!(!parse_source.contains("__pdns")); + assert!( + !source[analysis.calls[0].start..analysis.calls[0].end].contains("string.non_empty(") + ); + } + + #[test] + fn same_name_locals_object_members_nested_computed_optional_are_not_namespace_calls() { + let source = concat!( + "import * as string from \"./strings.rss\";\n", + "string.non_empty(\"yes\");\n", + "{\n", + " const string = {};\n", + " string.non_empty(\"no\");\n", + "}\n", + "const box = { non_empty: 1 };\n", + "box.non_empty;\n", + "obj.string.non_empty(\"no\");\n", + "string[\"non_empty\"](\"no\");\n", + "string?.non_empty(\"no\");\n", + ); + let analysis = analyze_file_module_member_calls(source); + assert_eq!(analysis.calls.len(), 1); + assert_eq!(analysis.calls[0].alias, "string"); + assert_eq!(analysis.calls[0].member, "non_empty"); + assert_eq!( + &source[analysis.calls[0].start..analysis.calls[0].end], + "string.non_empty" + ); + assert!(source[..analysis.calls[0].start].contains("import * as string")); + assert!(source[analysis.calls[0].end..].starts_with("(\"yes\")")); + } + + #[test] + fn lowered_ir_keeps_qualified_names_and_original_callee_spans() { + let source = analysis_source(); + let ir = crate::javascript::lower_to_ir(source).expect("original lookalikes must parse"); + assert!( + ir.functions + .iter() + .any(|func| func.name == "string::non_empty"), + "function table must carry the qualified file-module call, got {:?}", + ir.functions + .iter() + .map(|func| &func.name) + .collect::>() + ); + assert!( + ir.implicit_extern_names + .iter() + .any(|name| name == "string::non_empty"), + "implicit externs must carry the qualified file-module call, got {:?}", + ir.implicit_extern_names + ); + assert!( + ir.functions + .iter() + .all(|func| !func.name.contains("m0") && !func.name.contains("__pdns")), + "function table must not leak parse placeholders" + ); + let index = ir + .parsed_semantic_index + .as_ref() + .expect("parser-produced semantic index"); + let site = index + .call_sites + .iter() + .find(|site| site.is_namespace_call && site.name == "string::non_empty") + .expect("qualified namespace call site"); + assert_eq!( + source.get(site.callee_span.lo..site.callee_span.hi), + Some("string.non_empty") + ); + assert!( + index + .call_sites + .iter() + .all(|site| !site.name.contains("m0") && !site.name.contains("__pdns")), + "semantic call sites must not leak parse placeholders" + ); + } +} diff --git a/tests/compiler/compiler_javascript_tests.rs b/tests/compiler/compiler_javascript_tests.rs index c18b1da..1c30feb 100644 --- a/tests/compiler/compiler_javascript_tests.rs +++ b/tests/compiler/compiler_javascript_tests.rs @@ -800,11 +800,19 @@ fn javascript_file_module_unknown_member_keeps_mapped_span() { ) .expect("strings module"); let main_path = root.join("main.js"); - fs::write( - &main_path, - "import * as string from \"./strings.rss\";\nstring.does_not_exist(\"rss\");\n", - ) - .expect("js source"); + let source = concat!( + "import * as string from \"./strings.rss\";\n", + "const single = 'string.does_not_exist(';\n", + "const double = \"string.does_not_exist(\";\n", + "const tmpl = \"`string.does_not_exist(`\";\n", + "const re = \"/string.does_not_exist(/\";\n", + "// template lookalike: `string.does_not_exist(`\n", + "/* string.does_not_exist(\"no\") */\n", + "const utf_before = \"是\";\n", + " string.does_not_exist(\"rss\");\n", + "const utf_after = \"後\";\n", + ); + fs::write(&main_path, source).expect("js source"); let error = match compile_source_file_with_options( main_path.as_path(), @@ -817,22 +825,56 @@ fn javascript_file_module_unknown_member_keeps_mapped_span() { vm::SourcePathError::SourceWithMap { error, sources } => { let message = error.to_string(); assert!( - message.contains("does_not_exist") - || message.contains("string::does_not_exist") - || message.contains("unknown"), - "rejection diagnostic should name the missing member: {message}" + message.contains("unknown namespace call 'string::does_not_exist'"), + "diagnostic must identify the qualified namespace member, got {message}" ); - let span = match &error { - vm::SourceError::Parse(parse) => parse.span, - _ => None, + assert!( + !message.contains("__pdns") && !message.contains(" m0"), + "diagnostic must not leak dialect placeholders: {message}" + ); + let parse = match &error { + vm::SourceError::Parse(parse) => parse, + other => panic!("expected parse diagnostic, got {other:?}"), }; - let span = span.expect("failing file-module call must keep a mapped span"); + assert_eq!( + parse.line, 9, + "mapped diagnostic must use the call-site line, got {} ({message})", + parse.line + ); + let span = parse + .span + .expect("failing file-module call must keep a mapped span"); + let call_line = source.lines().nth(8).expect("call-site line"); + let lo = source.find(call_line).expect("call-site offset"); + assert_eq!( + (span.lo, span.hi), + (lo, lo + call_line.len()), + "core source-loader maps unknown namespace calls to the full call-site line" + ); let text = sources .span_text(span) .expect("mapped span must resolve against the kept source map"); + assert_eq!(text, call_line); + assert!( + text.contains("string.does_not_exist"), + "mapped span should cover the call expression, got {text:?}" + ); + assert!( + !text.contains("import") && !text.contains("是") && !text.contains("後"), + "mapped span must not be the import line or surrounding literals, got {text:?}" + ); + let original = fs::read_to_string(&main_path).expect("original source"); + assert!( + original.contains("const single = 'string.does_not_exist(';"), + "single-quoted lookalike must stay in original source" + ); + assert!( + original.contains("const tmpl = \"`string.does_not_exist(`\";"), + "template-literal lookalike must stay in original source" + ); assert!( - text.contains("does_not_exist") || text.contains("string"), - "mapped span should cover the failing call, got {text:?}" + original.contains("const re = \"/string.does_not_exist(/\";"), + "regex-literal lookalike must stay in original source" ); } other => panic!("expected SourceWithMap, got {other}"), diff --git a/tests/compiler/compiler_lua_tests.rs b/tests/compiler/compiler_lua_tests.rs index 1ad163e..17997eb 100644 --- a/tests/compiler/compiler_lua_tests.rs +++ b/tests/compiler/compiler_lua_tests.rs @@ -427,11 +427,16 @@ fn lua_file_module_unknown_member_keeps_mapped_span() { ) .expect("strings module"); let main_path = root.join("main.lua"); - fs::write( - &main_path, - "local string = require(\"./strings.rss\")\nstring.does_not_exist(\"rss\")\n", - ) - .expect("lua source"); + let source = concat!( + "local string = require(\"./strings.rss\")\n", + "local single = 'string.does_not_exist('\n", + "local double = \"string.does_not_exist(\"\n", + "-- string.does_not_exist(\"no\")\n", + "local utf_before = \"是\"\n", + " string.does_not_exist(\"rss\")\n", + "local utf_after = \"後\"\n", + ); + fs::write(&main_path, source).expect("lua source"); let error = match compile_source_file_with_options( main_path.as_path(), @@ -444,22 +449,39 @@ fn lua_file_module_unknown_member_keeps_mapped_span() { vm::SourcePathError::SourceWithMap { error, sources } => { let message = error.to_string(); assert!( - message.contains("does_not_exist") - || message.contains("string::does_not_exist") - || message.contains("unknown"), - "rejection diagnostic should name the missing member: {message}" + message.contains("unknown namespace call 'string::does_not_exist'"), + "diagnostic must identify the qualified namespace member, got {message}" ); - let span = match &error { - vm::SourceError::Parse(parse) => parse.span, - _ => None, + let parse = match &error { + vm::SourceError::Parse(parse) => parse, + other => panic!("expected parse diagnostic, got {other:?}"), }; - let span = span.expect("failing file-module call must keep a mapped span"); + assert_eq!( + parse.line, 6, + "mapped diagnostic must use the call-site line, got {} ({message})", + parse.line + ); + let span = parse + .span + .expect("failing file-module call must keep a mapped span"); + let call_line = source.lines().nth(5).expect("call-site line"); + let lo = source.find(call_line).expect("call-site offset"); + assert_eq!( + (span.lo, span.hi), + (lo, lo + call_line.len()), + "core source-loader maps unknown namespace calls to the full call-site line" + ); let text = sources .span_text(span) .expect("mapped span must resolve against the kept source map"); + assert_eq!(text, call_line); + assert!( + text.contains("string.does_not_exist"), + "mapped span should cover the call expression, got {text:?}" + ); assert!( - text.contains("does_not_exist") || text.contains("string"), - "mapped span should cover the failing call, got {text:?}" + !text.contains("require") && !text.contains("是") && !text.contains("後"), + "mapped span must not be the import line or surrounding literals, got {text:?}" ); } other => panic!("expected SourceWithMap, got {other}"), diff --git a/tests/frozen_core_consumer.rs b/tests/frozen_core_consumer.rs index 303efd8..7a76fda 100644 --- a/tests/frozen_core_consumer.rs +++ b/tests/frozen_core_consumer.rs @@ -471,4 +471,18 @@ fn publish_workflow_rewrites_git_deps_to_frozen_crates_io_versions() { workflow.contains("'pd-host-function': os.environ.get('PD_VM_VERSION') or '0.1.0'"), "publish rewrite must map pd-host-function git/path deps to crates.io 0.1.0" ); + + let readme = fs::read_to_string(manifest_dir().join("README.md")).expect("README"); + assert!( + !readme.contains("0.22.2"), + "README must not advertise the old 0.22.2 crates.io version" + ); + assert!( + readme.contains("pd-vm = \"0.1.0\""), + "README dependency example must use frozen pd-vm 0.1.0" + ); + assert!( + readme.contains("pd-vm-compat-frontends = \"0.1.0\""), + "README dependency example must use frozen pd-vm-compat-frontends 0.1.0" + ); } From 32d5e0a4c4595c6a4f0ada011ce6c6c5fd4dc386 Mon Sep 17 00:00:00 2001 From: fffonion Date: Fri, 18 Sep 2026 02:10:14 +0800 Subject: [PATCH 7/7] Fail closed on JS file-module fold length and collisions. Assign collision-free same-length placeholders before parse, reject multiline callees that would move line mapping, and rewrite only the span-matched implicit externs. --- README.md | 2 +- src/javascript.rs | 19 +- src/js_namespace.rs | 655 +++++++++++++++++--- tests/compiler/compiler_javascript_tests.rs | 109 ++++ 4 files changed, 703 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index 1fae4d3..6c07c75 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ JavaScript and Lua compatibility frontends for `pd-vm`. This crate owns the compatibility-language pieces that are intentionally outside the core `rustscript` repository: -- JavaScript parser dialect configuration and file-module AST/IR namespace lowering +- JavaScript parser dialect, span-preserving parser compatibility fold for file-module `alias.member()` calls, and IR lowering onto qualified `alias::member` names - Lua parser/lowering helpers - JavaScript and Lua import scanning / import stripping for source-file loading - compatibility frontend tests and fixtures diff --git a/src/javascript.rs b/src/javascript.rs index 2b5e71a..91d0704 100644 --- a/src/javascript.rs +++ b/src/javascript.rs @@ -67,16 +67,19 @@ pub(crate) fn parser_dialect() -> &'static dyn ParserDialect { } pub(crate) fn lower_to_ir(source: &str) -> Result { - // File-module `alias.member()` is a MemberExpression-rooted Call. The frozen - // parser's unknown dotted-call fallback rewinds those sites, so this frontend - // owns lowering: identify original call spans, parse, then rewrite the - // produced Call IR / semantic-index entries to qualified `alias::member` - // names while keeping the original callee spans. + // Frozen `try_parse_js_dotted_call` rewinds unknown file-module dotted + // calls. Fold those callee spans to collision-free same-length identifiers + // (byte length and `\n`/`\r` positions unchanged), parse the folded source, + // then rewrite only the implicit-extern Call IR / semantic-index entries + // that match the original spans to qualified `alias::member` names. // Local object members, shadowed aliases, computed/optional chains, and - // nested `obj.alias.member` are left as ordinary member access. - let analysis = crate::js_namespace::analyze_file_module_member_calls(source); + // nested `obj.alias.member` stay ordinary member access. Multiline callees + // cannot be an identifier without moving line boundaries, so they fail + // closed before parse. + let analysis = crate::js_namespace::analyze_file_module_member_calls(source)?; + let folded = analysis.parse_source(source)?; let mut ir = parse_source_with_dialect( - analysis.parse_source(source).as_ref(), + folded.as_ref(), parser_dialect(), SharedParserOptions { allow_implicit_semicolons: true, diff --git a/src/js_namespace.rs b/src/js_namespace.rs index 3724b25..2437add 100644 --- a/src/js_namespace.rs +++ b/src/js_namespace.rs @@ -1,7 +1,7 @@ use std::borrow::Cow; -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; -use vm::{FrontendIr, ImportClause}; +use vm::{FrontendIr, ImportClause, ParseError, Span}; use crate::source_loader::{is_file_module_spec, parse_js_imports}; @@ -49,17 +49,18 @@ struct FileModuleMemberCall { member: String, start: usize, end: usize, + placeholder: String, } +#[derive(Debug)] pub(crate) struct FileModuleCallAnalysis { calls: Vec, - dialect_names: HashMap, } impl FileModuleCallAnalysis { - pub(crate) fn parse_source<'a>(&self, source: &'a str) -> Cow<'a, str> { + pub(crate) fn parse_source<'a>(&self, source: &'a str) -> Result, ParseError> { if self.calls.is_empty() { - return Cow::Borrowed(source); + return Ok(Cow::Borrowed(source)); } // Frozen `try_parse_js_dotted_call` rewinds unknown (file-module) dotted // calls, leaving `alias.member()` unparsed. Fold only those callees to @@ -67,99 +68,222 @@ impl FileModuleCallAnalysis { // at the original offsets; lookalike literals/comments are untouched. let mut out = String::with_capacity(source.len()); let mut last = 0usize; - for (index, call) in self.calls.iter().enumerate() { + for call in &self.calls { + if call.placeholder.len() != call.end.saturating_sub(call.start) { + return Err(fold_error( + source, + call.start, + call.end, + format!( + "file-module member call '{}'.'{}' cannot be folded without changing source length", + call.alias, call.member + ), + )); + } out.push_str(&source[last..call.start]); - out.push_str(&dialect_callee_ident(index, call.end - call.start)); + out.push_str(&call.placeholder); last = call.end; } out.push_str(&source[last..]); - debug_assert_eq!(out.len(), source.len()); - Cow::Owned(out) + if out.len() != source.len() { + return Err(ParseError::new( + "file-module member call fold changed source length", + )); + } + if newline_offsets(&out) != newline_offsets(source) { + return Err(ParseError::new( + "file-module member call fold moved line boundaries", + )); + } + Ok(Cow::Owned(out)) } pub(crate) fn lower_ir(&self, ir: &mut FrontendIr) { - if self.dialect_names.is_empty() { + if self.calls.is_empty() { return; } + let extern_names = ir + .implicit_extern_names + .iter() + .cloned() + .collect::>(); for func in &mut ir.functions { - if let Some(qualified) = self.dialect_names.get(&func.name) { - func.name = qualified.clone(); + if !extern_names.contains(&func.name) { + continue; + } + if let Some(call) = self.call_for_placeholder(&func.name) { + func.name = qualified_name(call); } } for name in &mut ir.implicit_extern_names { - if let Some(qualified) = self.dialect_names.get(name) { - *name = qualified.clone(); + if let Some(call) = self.call_for_placeholder(name) { + *name = qualified_name(call); } } if let Some(index) = ir.parsed_semantic_index.as_mut() { for site in &mut index.call_sites { - let matched = self.calls.iter().find(|call| { - site.callee_span.hi == call.end - && site.callee_span.lo >= call.start - && site.callee_span.lo < call.end - }); - if let Some(call) = matched { - site.name = format!("{}::{}", call.alias, call.member); + if let Some(call) = self.call_for_span(site.callee_span.lo, site.callee_span.hi) { + site.name = qualified_name(call); site.is_namespace_call = true; site.callee_span.lo = call.start; site.callee_span.hi = call.end; - continue; - } - if let Some(qualified) = self.dialect_names.get(&site.name) { - site.name = qualified.clone(); - site.is_namespace_call = true; } } for func_ref in &mut index.func_refs { - if let Some(qualified) = self.dialect_names.get(&func_ref.name) { - func_ref.name = qualified.clone(); + if let Some(call) = + self.call_for_span(func_ref.ident_span.lo, func_ref.ident_span.hi) + { + func_ref.name = qualified_name(call); } } - for func_decl in &mut index.func_decls { - if let Some(qualified) = self.dialect_names.get(&func_decl.name) { - func_decl.name = qualified.clone(); - } + } + for token in &mut ir.lexer_tokens { + if token.kind != "Ident" { + continue; + } + if let Some(call) = self.call_for_span(token.span.lo, token.span.hi) { + token.ident = qualified_name(call); } } } + + fn call_for_placeholder(&self, name: &str) -> Option<&FileModuleMemberCall> { + self.calls.iter().find(|call| call.placeholder == name) + } + + fn call_for_span(&self, lo: usize, hi: usize) -> Option<&FileModuleMemberCall> { + self.calls + .iter() + .find(|call| call.start == lo && call.end == hi) + } } -pub(crate) fn analyze_file_module_member_calls(source: &str) -> FileModuleCallAnalysis { +pub(crate) fn analyze_file_module_member_calls( + source: &str, +) -> Result { let aliases = file_module_namespace_aliases(source); if aliases.is_empty() { - return FileModuleCallAnalysis { - calls: Vec::new(), - dialect_names: HashMap::new(), - }; + return Ok(FileModuleCallAnalysis { calls: Vec::new() }); } let tokens = tokenize_js(source); - let calls = collect_file_module_member_calls(source, &tokens, &aliases); - let mut dialect_names = HashMap::new(); - for (index, call) in calls.iter().enumerate() { - dialect_names.insert( - dialect_callee_ident(index, call.end - call.start) - .trim() - .to_string(), - format!("{}::{}", call.alias, call.member), - ); + let mut calls = collect_file_module_member_calls(source, &tokens, &aliases); + assign_placeholders(source, &tokens, &mut calls)?; + Ok(FileModuleCallAnalysis { calls }) +} + +fn qualified_name(call: &FileModuleMemberCall) -> String { + format!("{}::{}", call.alias, call.member) +} + +fn newline_offsets(source: &str) -> Vec { + source + .bytes() + .enumerate() + .filter(|(_, byte)| *byte == b'\n' || *byte == b'\r') + .map(|(index, _)| index) + .collect() +} + +fn line_number(source: &str, offset: usize) -> usize { + source.as_bytes()[..offset.min(source.len())] + .iter() + .filter(|byte| **byte == b'\n') + .count() + + 1 +} + +fn fold_error(source: &str, start: usize, end: usize, message: String) -> ParseError { + ParseError { + line: line_number(source, start), + message, + span: Some(Span::new(0, start, end)), + code: None, + } +} + +/// Lexer keywords in the frozen parser, plus JS dialect aliases. Placeholders +/// that match these are tokenized as keywords, not identifiers. +const FROZEN_LEXER_KEYWORDS: &[&str] = &[ + "pub", "use", "import", "from", "as", "fn", "function", "struct", "let", "const", "var", "for", + "if", "else", "match", "while", "break", "continue", "true", "false", "null", "return", + "typeof", "require", +]; + +const PLACEHOLDER_FIRST: &[u8] = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_"; +const PLACEHOLDER_REST: &[u8] = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"; + +fn assign_placeholders( + source: &str, + tokens: &[Token], + calls: &mut [FileModuleMemberCall], +) -> Result<(), ParseError> { + let mut used = tokens + .iter() + .filter(|tok| tok.kind == TokenKind::Ident) + .map(|tok| token_text(source, tok).to_string()) + .collect::>(); + used.extend(FROZEN_LEXER_KEYWORDS.iter().map(|name| (*name).to_string())); + for call in calls.iter_mut() { + let span = &source.as_bytes()[call.start..call.end]; + if span.iter().any(|byte| *byte == b'\n' || *byte == b'\r') { + return Err(fold_error( + source, + call.start, + call.end, + format!( + "file-module member call '{}'.'{}' spans a line break; the compatibility frontend cannot fold it without shifting parser line mapping", + call.alias, call.member + ), + )); + } + let len = call.end - call.start; + let Some(placeholder) = unique_placeholder(len, &mut used) else { + return Err(fold_error( + source, + call.start, + call.end, + format!( + "file-module member call '{}'.'{}' cannot be folded to a collision-free identifier of {len} bytes", + call.alias, call.member + ), + )); + }; + call.placeholder = placeholder; } - FileModuleCallAnalysis { - calls, - dialect_names, + Ok(()) +} + +fn unique_placeholder(len: usize, used: &mut HashSet) -> Option { + let mut index = 0u128; + while let Some(ident) = nth_ident(len, index) { + if used.insert(ident.clone()) { + return Some(ident); + } + index += 1; + if index > 1_000_000 { + break; + } } + None } -fn dialect_callee_ident(index: usize, original_len: usize) -> String { - let ident = format!("m{index}"); - if ident.len() >= original_len { - return ident; +fn nth_ident(len: usize, mut index: u128) -> Option { + if len == 0 { + return None; } - let mut out = String::with_capacity(original_len); - for _ in 0..(original_len - ident.len()) { - out.push(' '); + let first_len = PLACEHOLDER_FIRST.len() as u128; + let rest_len = PLACEHOLDER_REST.len() as u128; + let mut bytes = vec![0u8; len]; + for slot in (1..len).rev() { + bytes[slot] = PLACEHOLDER_REST[(index % rest_len) as usize]; + index /= rest_len; } - out.push_str(&ident); - out + bytes[0] = PLACEHOLDER_FIRST[(index % first_len) as usize]; + index /= first_len; + if index != 0 { + return None; + } + String::from_utf8(bytes).ok() } fn collect_file_module_member_calls( @@ -245,6 +369,7 @@ fn match_file_module_member_call( member, start: alias_tok.start, end: member_tok.end, + placeholder: String::new(), }) } @@ -645,10 +770,93 @@ mod tests { ) } + fn assert_fold_preserves_layout(source: &str, analysis: &FileModuleCallAnalysis) -> String { + let folded = analysis + .parse_source(source) + .expect("safe fold must succeed") + .into_owned(); + assert_eq!( + folded.len(), + source.len(), + "fold must not change byte length" + ); + assert_eq!( + newline_offsets(&folded), + newline_offsets(source), + "fold must keep every \\n and \\r at the original offset" + ); + for call in &analysis.calls { + let original = &source[call.start..call.end]; + let folded_callee = &folded[call.start..call.end]; + assert_eq!(folded_callee, call.placeholder); + assert_eq!(folded_callee.len(), original.len()); + assert!( + !folded_callee.contains('.'), + "folded callee must be a single identifier, got {folded_callee:?}" + ); + assert!( + !folded_callee.as_bytes().contains(&b'\n') + && !folded_callee.as_bytes().contains(&b'\r'), + "folded callee must not swallow line breaks, got {folded_callee:?}" + ); + assert!( + is_ident(folded_callee), + "folded callee must be a valid identifier, got {folded_callee:?}" + ); + } + folded + } + + fn assert_no_placeholder_leak(ir: &FrontendIr, analysis: &FileModuleCallAnalysis) { + for call in &analysis.calls { + let placeholder = call.placeholder.as_str(); + assert!( + ir.functions.iter().all(|func| func.name != placeholder), + "function table leaked placeholder {placeholder}, got {:?}", + ir.functions + .iter() + .map(|func| &func.name) + .collect::>() + ); + assert!( + ir.implicit_extern_names + .iter() + .all(|name| name != placeholder), + "implicit externs leaked placeholder {placeholder}, got {:?}", + ir.implicit_extern_names + ); + let index = ir + .parsed_semantic_index + .as_ref() + .expect("parser-produced semantic index"); + assert!( + index.call_sites.iter().all(|site| site.name != placeholder), + "call sites leaked placeholder {placeholder}" + ); + assert!( + index.func_decls.iter().all(|decl| decl.name != placeholder), + "func decls leaked placeholder {placeholder}" + ); + assert!( + index + .func_refs + .iter() + .all(|func_ref| func_ref.name != placeholder), + "func refs leaked placeholder {placeholder}" + ); + assert!( + ir.lexer_tokens + .iter() + .all(|token| token.ident != placeholder), + "lexer tokens leaked placeholder {placeholder}" + ); + } + } + #[test] fn lookalike_literals_are_not_file_module_calls() { let source = analysis_source(); - let analysis = analyze_file_module_member_calls(source); + let analysis = analyze_file_module_member_calls(source).expect("analyze"); assert_eq!(analysis.calls.len(), 1); assert_eq!(analysis.calls[0].alias, "string"); assert_eq!(analysis.calls[0].member, "non_empty"); @@ -656,16 +864,16 @@ mod tests { &source[analysis.calls[0].start..analysis.calls[0].end], "string.non_empty" ); - let parse_source = analysis.parse_source(source); - assert_eq!(parse_source.len(), source.len()); + let parse_source = assert_fold_preserves_layout(source, &analysis); assert!(parse_source.contains("const single = 'string.non_empty(';")); assert!(parse_source.contains(r#"const double = "string.non_empty(";"#)); assert!(parse_source.contains(r#"const tmpl = "`string.non_empty(`";"#)); assert!(parse_source.contains(r#"const re = "/string.non_empty(/";"#)); assert!(parse_source.contains("是")); assert!(!parse_source.contains("__pdns")); - assert!( - !source[analysis.calls[0].start..analysis.calls[0].end].contains("string.non_empty(") + assert_ne!( + &parse_source[analysis.calls[0].start..analysis.calls[0].end], + "string.non_empty" ); } @@ -684,7 +892,7 @@ mod tests { "string[\"non_empty\"](\"no\");\n", "string?.non_empty(\"no\");\n", ); - let analysis = analyze_file_module_member_calls(source); + let analysis = analyze_file_module_member_calls(source).expect("analyze"); assert_eq!(analysis.calls.len(), 1); assert_eq!(analysis.calls[0].alias, "string"); assert_eq!(analysis.calls[0].member, "non_empty"); @@ -699,6 +907,7 @@ mod tests { #[test] fn lowered_ir_keeps_qualified_names_and_original_callee_spans() { let source = analysis_source(); + let analysis = analyze_file_module_member_calls(source).expect("analyze"); let ir = crate::javascript::lower_to_ir(source).expect("original lookalikes must parse"); assert!( ir.functions @@ -717,12 +926,7 @@ mod tests { "implicit externs must carry the qualified file-module call, got {:?}", ir.implicit_extern_names ); - assert!( - ir.functions - .iter() - .all(|func| !func.name.contains("m0") && !func.name.contains("__pdns")), - "function table must not leak parse placeholders" - ); + assert_no_placeholder_leak(&ir, &analysis); let index = ir .parsed_semantic_index .as_ref() @@ -736,12 +940,317 @@ mod tests { source.get(site.callee_span.lo..site.callee_span.hi), Some("string.non_empty") ); + } + + #[test] + fn user_local_m0_is_not_rewritten_as_namespace_target() { + let source = concat!( + "import * as a from \"./m.rss\";\n", + "const m0 = 2;\n", + "a.b();\n", + "m0;\n", + ); + let analysis = analyze_file_module_member_calls(source).expect("analyze"); + assert_eq!(analysis.calls.len(), 1); + assert_ne!(analysis.calls[0].placeholder, "m0"); + assert_fold_preserves_layout(source, &analysis); + let ir = crate::javascript::lower_to_ir(source).expect("local m0 must parse"); + assert_no_placeholder_leak(&ir, &analysis); + let index = ir + .parsed_semantic_index + .as_ref() + .expect("parser-produced semantic index"); + assert!( + index.local_decls.iter().any(|decl| decl.name == "m0"), + "user local m0 must remain a local, got {:?}", + index + .local_decls + .iter() + .map(|decl| &decl.name) + .collect::>() + ); + assert!( + ir.functions.iter().all(|func| func.name != "m0"), + "local m0 must not become a function, got {:?}", + ir.functions + .iter() + .map(|func| &func.name) + .collect::>() + ); + assert_eq!(ir.implicit_extern_names, vec!["a::b".to_string()]); + let site = index + .call_sites + .iter() + .find(|site| site.is_namespace_call) + .expect("namespace call"); + assert_eq!(site.name, "a::b"); + assert_eq!( + source.get(site.callee_span.lo..site.callee_span.hi), + Some("a.b") + ); + } + + #[test] + fn user_function_m0_is_not_renamed_with_file_module_call() { + let source = concat!( + "import * as a from \"./m.rss\";\n", + "function m0() { return 1; }\n", + "a.b();\n", + "m0();\n", + ); + let analysis = analyze_file_module_member_calls(source).expect("analyze"); + assert_eq!(analysis.calls.len(), 1); + assert_ne!(analysis.calls[0].placeholder, "m0"); + assert_fold_preserves_layout(source, &analysis); + let ir = crate::javascript::lower_to_ir(source).expect("function m0 must parse"); + assert_no_placeholder_leak(&ir, &analysis); + assert!( + ir.functions.iter().any(|func| func.name == "m0"), + "user function m0 must keep its name, got {:?}", + ir.functions + .iter() + .map(|func| &func.name) + .collect::>() + ); assert!( + ir.functions.iter().any(|func| func.name == "a::b"), + "file-module call must lower to a::b, got {:?}", + ir.functions + .iter() + .map(|func| &func.name) + .collect::>() + ); + let index = ir + .parsed_semantic_index + .as_ref() + .expect("parser-produced semantic index"); + assert!( + index.func_decls.iter().any(|decl| decl.name == "m0"), + "func_decls must keep user function m0, got {:?}", index - .call_sites + .func_decls .iter() - .all(|site| !site.name.contains("m0") && !site.name.contains("__pdns")), - "semantic call sites must not leak parse placeholders" + .map(|decl| &decl.name) + .collect::>() + ); + assert!( + index.func_decls.iter().all(|decl| decl.name != "a::b"), + "IR lowering must not rewrite every declaration sharing placeholder text" + ); + let names: Vec<&str> = index + .call_sites + .iter() + .map(|site| site.name.as_str()) + .collect(); + assert!( + names.contains(&"a::b"), + "namespace call site missing, got {names:?}" + ); + assert!( + names.contains(&"m0"), + "user m0() call site missing, got {names:?}" + ); + } + + #[test] + fn many_short_calls_keep_unique_same_length_placeholders() { + let mut source = String::from("import * as a from \"./m.rss\";\n"); + for _ in 0..110 { + source.push_str("a.b();\n"); + } + let analysis = analyze_file_module_member_calls(&source).expect("analyze"); + assert_eq!(analysis.calls.len(), 110); + let mut placeholders = HashSet::new(); + for call in &analysis.calls { + assert_eq!(call.end - call.start, 3); + assert_eq!(call.placeholder.len(), 3); + assert_eq!(&source[call.start..call.end], "a.b"); + assert!( + placeholders.insert(call.placeholder.clone()), + "placeholder {} reused", + call.placeholder + ); + } + let folded = assert_fold_preserves_layout(&source, &analysis); + let ir = crate::javascript::lower_to_ir(&source).expect("110 short calls must parse"); + assert_no_placeholder_leak(&ir, &analysis); + assert!( + ir.functions.iter().all(|func| func.name == "a::b"), + "every implicit extern must lower to a::b, got {:?}", + ir.functions + .iter() + .map(|func| &func.name) + .collect::>() + ); + assert!( + ir.implicit_extern_names.iter().all(|name| name == "a::b"), + "implicit externs must all be a::b, got {:?}", + ir.implicit_extern_names + ); + let index = ir + .parsed_semantic_index + .as_ref() + .expect("parser-produced semantic index"); + let sites: Vec<_> = index + .call_sites + .iter() + .filter(|site| site.is_namespace_call) + .collect(); + assert_eq!(sites.len(), 110); + for site in sites { + assert_eq!(site.name, "a::b"); + assert_eq!( + source.get(site.callee_span.lo..site.callee_span.hi), + Some("a.b") + ); + assert_eq!(&folded[site.callee_span.lo..site.callee_span.hi].len(), &3); + } + } + + #[test] + fn distinct_same_length_qualified_calls_keep_separate_targets() { + let source = concat!( + "import * as ab from \"./left.rss\";\n", + "import * as cd from \"./right.rss\";\n", + "ab.xy();\n", + "cd.uv();\n", + ); + let analysis = analyze_file_module_member_calls(source).expect("analyze"); + assert_eq!(analysis.calls.len(), 2); + assert_eq!(analysis.calls[0].end - analysis.calls[0].start, 5); + assert_eq!(analysis.calls[1].end - analysis.calls[1].start, 5); + assert_ne!(analysis.calls[0].placeholder, analysis.calls[1].placeholder); + assert_fold_preserves_layout(source, &analysis); + let ir = crate::javascript::lower_to_ir(source).expect("distinct calls must parse"); + assert_no_placeholder_leak(&ir, &analysis); + let mut names = ir + .functions + .iter() + .map(|func| func.name.as_str()) + .collect::>(); + names.sort_unstable(); + names.dedup(); + assert!( + names.contains(&"ab::xy") && names.contains(&"cd::uv"), + "distinct qualified targets must survive, got {names:?}" + ); + let index = ir + .parsed_semantic_index + .as_ref() + .expect("parser-produced semantic index"); + let sites: Vec<_> = index + .call_sites + .iter() + .filter(|site| site.is_namespace_call) + .map(|site| { + ( + site.name.as_str(), + source.get(site.callee_span.lo..site.callee_span.hi), + ) + }) + .collect(); + assert!(sites.contains(&("ab::xy", Some("ab.xy")))); + assert!(sites.contains(&("cd::uv", Some("cd.uv")))); + } + + #[test] + fn alias_member_lengths_near_ident_boundaries() { + let source = concat!( + "import * as a from \"./a.rss\";\n", + "import * as aa from \"./aa.rss\";\n", + "a.b();\n", + "aa.b();\n", + "a.bb();\n", + ); + let analysis = analyze_file_module_member_calls(source).expect("analyze"); + assert_eq!(analysis.calls.len(), 3); + let lens: Vec = analysis + .calls + .iter() + .map(|call| call.placeholder.len()) + .collect(); + assert_eq!(lens, vec![3, 4, 4]); + assert_eq!( + &source[analysis.calls[0].start..analysis.calls[0].end], + "a.b" + ); + assert_eq!( + &source[analysis.calls[1].start..analysis.calls[1].end], + "aa.b" + ); + assert_eq!( + &source[analysis.calls[2].start..analysis.calls[2].end], + "a.bb" + ); + assert_fold_preserves_layout(source, &analysis); + let ir = crate::javascript::lower_to_ir(source).expect("boundary lengths must parse"); + assert_no_placeholder_leak(&ir, &analysis); + let mut names = ir.implicit_extern_names.to_vec(); + names.sort(); + assert_eq!( + names, + vec!["a::b".to_string(), "a::bb".to_string(), "aa::b".to_string()] + ); + } + + #[test] + fn multiline_alias_member_call_fails_closed() { + let source = concat!( + "import * as string from \"./strings.rss\";\n", + "string\n", + ".non_empty(\"x\");\n", + ); + let error = + analyze_file_module_member_calls(source).expect_err("multiline must fail closed"); + assert!( + error.message.contains("spans a line break"), + "diagnostic must name the unsupported construct, got {}", + error.message + ); + assert!( + error.message.contains("string") && error.message.contains("non_empty"), + "diagnostic must name the original target, got {}", + error.message + ); + assert_eq!(error.line, 2); + let span = error.span.expect("multiline fold error must carry a span"); + assert!(source[span.lo..span.hi].contains('\n')); + assert!(source[span.lo..span.hi].contains("string")); + assert!(source[span.lo..span.hi].contains("non_empty")); + } + + #[test] + fn multiline_dot_before_member_fails_closed() { + let source = concat!( + "import * as string from \"./strings.rss\";\n", + "string.\n", + "non_empty(\"x\");\n", + ); + let error = + analyze_file_module_member_calls(source).expect_err("multiline must fail closed"); + assert!(error.message.contains("spans a line break")); + assert_eq!(error.line, 2); + } + + #[test] + fn carriage_return_in_callee_fails_closed() { + let source = "import * as string from \"./strings.rss\";\nstring\r.non_empty(\"x\");\n"; + let error = + analyze_file_module_member_calls(source).expect_err("CR callee must fail closed"); + assert!(error.message.contains("spans a line break")); + assert!(source[error.span.expect("span").lo..error.span.expect("span").hi].contains('\r')); + } + + #[test] + fn carriage_return_outside_callee_is_preserved() { + let source = concat!( + "import * as a from \"./m.rss\";\n", + "const cr = \"x\ry\";\n", + "a.b();\n", ); + let analysis = analyze_file_module_member_calls(source).expect("analyze"); + let folded = assert_fold_preserves_layout(source, &analysis); + assert!(folded.contains('\r')); + assert_eq!(folded.find('\r'), source.find('\r')); } } diff --git a/tests/compiler/compiler_javascript_tests.rs b/tests/compiler/compiler_javascript_tests.rs index 1c30feb..b3e0cf3 100644 --- a/tests/compiler/compiler_javascript_tests.rs +++ b/tests/compiler/compiler_javascript_tests.rs @@ -880,3 +880,112 @@ fn javascript_file_module_unknown_member_keeps_mapped_span() { other => panic!("expected SourceWithMap, got {other}"), } } + +#[test] +fn javascript_file_module_call_with_user_function_m0_keeps_both() { + let root = namespace_case_root("js_user_fn_m0"); + fs::write(root.join("m.rss"), "pub fn b() { 1 }\n").expect("module"); + let main_path = root.join("main.js"); + fs::write( + &main_path, + concat!( + "import * as a from \"./m.rss\";\n", + "function m0() { return 2; }\n", + "a.b() + m0();\n", + ), + ) + .expect("js source"); + let compiled = compile_source_file_with_options( + main_path.as_path(), + pd_vm_compat_frontends::compile_options(), + ) + .expect("user function m0 plus file-module call should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(3)]); +} + +#[test] +fn javascript_file_module_call_with_user_local_m0_keeps_both() { + let root = namespace_case_root("js_user_local_m0"); + fs::write(root.join("m.rss"), "pub fn b() { 1 }\n").expect("module"); + let main_path = root.join("main.js"); + fs::write( + &main_path, + concat!( + "import * as a from \"./m.rss\";\n", + "const m0 = 2;\n", + "a.b() + m0;\n", + ), + ) + .expect("js source"); + let compiled = compile_source_file_with_options( + main_path.as_path(), + pd_vm_compat_frontends::compile_options(), + ) + .expect("user local m0 plus file-module call should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(3)]); +} + +#[test] +fn javascript_multiline_file_module_call_fails_closed_before_parse() { + let root = namespace_case_root("js_multiline_fold"); + fs::write( + root.join("strings.rss"), + "pub fn non_empty(value) { value.length != 0; }\n", + ) + .expect("module"); + let main_path = root.join("main.js"); + let source = concat!( + "import * as string from \"./strings.rss\";\n", + "string\n", + ".non_empty(\"rss\");\n", + ); + fs::write(&main_path, source).expect("js source"); + let error = match compile_source_file_with_options( + main_path.as_path(), + pd_vm_compat_frontends::compile_options(), + ) { + Ok(_) => panic!("multiline file-module call should fail closed"), + Err(error) => error, + }; + let message = error.to_string(); + assert!( + message.contains("spans a line break"), + "diagnostic must fail closed on multiline fold, got {message}" + ); + assert!( + message.contains("string") && message.contains("non_empty"), + "diagnostic must name the original qualified target, got {message}" + ); + assert!( + !message.contains("aaa") && !message.contains("__pdns") && !message.contains(" m0"), + "diagnostic must not leak placeholders: {message}" + ); +} + +#[test] +fn javascript_many_short_file_module_calls_compile() { + let root = namespace_case_root("js_many_short_calls"); + fs::write(root.join("m.rss"), "pub fn b() { 1 }\n").expect("module"); + let main_path = root.join("main.js"); + let mut source = String::from("import * as a from \"./m.rss\";\nlet total = 0;\n"); + for _ in 0..110 { + source.push_str("total = total + a.b();\n"); + } + source.push_str("total;\n"); + fs::write(&main_path, source).expect("js source"); + let compiled = compile_source_file_with_options( + main_path.as_path(), + pd_vm_compat_frontends::compile_options(), + ) + .expect("110 short file-module calls should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(110)]); +}