diff --git a/packages/cashc/src/ast/AST.ts b/packages/cashc/src/ast/AST.ts index 864db595b..b8858e23f 100644 --- a/packages/cashc/src/ast/AST.ts +++ b/packages/cashc/src/ast/AST.ts @@ -47,7 +47,6 @@ export class SourceFileNode extends Node { export class ConstantDefinitionNode extends Node implements Named, Typed { // Source provenance for debugging. Set on imported constants, left undefined for constants in the contract's own file. - sourceCode?: string; sourceFile?: string; modifiers = [Modifier.CONSTANT]; @@ -101,7 +100,6 @@ export class FunctionDefinitionNode extends Node implements Named { constant?: ConstantDefinitionNode; // Source provenance for debugging. Set on imported functions, left undefined for functions in the contract's own file. - sourceCode?: string; sourceFile?: string; constructor( diff --git a/packages/cashc/src/compiler.ts b/packages/cashc/src/compiler.ts index dcfcf5fac..dcc21d97b 100644 --- a/packages/cashc/src/compiler.ts +++ b/packages/cashc/src/compiler.ts @@ -67,7 +67,7 @@ export const compileString: (code: string, compilerOptions?: CompileStringOption /** * Read a `.cash` source file from disk and compile it to an `Artifact`. * - * Import directives are resolved from the filesystem: file paths (starting with './', '../' or '/') + * Import directives are resolved from the filesystem: file paths (starting with './' or '../') * relative to the importing file's directory, and package imports (e.g. 'pkg/math.cash') * from node_modules directories, walking up from the importing file's directory. * @@ -115,7 +115,7 @@ function compileCode( let ast = parseCode(code, errorListener); checkVersionConstraints(ast.pragmas); - ast = resolveDependencies(ast, resolver, errorListener) as Ast; + const sources = resolveDependencies(ast, resolver, errorListener); if (!ast.contract) throw new MissingContractError(); const constructorParamLength = ast.contract.parameters.length; @@ -162,6 +162,7 @@ function compileCode( requires: optimisationResult.requires, sourceTags: generateSourceTags(optimisationResult.sourceTags) || undefined, functions: traversal.frames.length > 0 ? traversal.frames : undefined, + sources: Object.keys(sources).length > 0 ? sources : undefined, inlineRanges: generateInlineRanges(optimisationResult.inlineRanges) || undefined, }; diff --git a/packages/cashc/src/dependency-resolution.ts b/packages/cashc/src/dependency-resolution.ts index aaa621fc4..9be514ae2 100644 --- a/packages/cashc/src/dependency-resolution.ts +++ b/packages/cashc/src/dependency-resolution.ts @@ -11,63 +11,81 @@ import type { CashScriptErrorListener } from './ast/error-listeners.js'; import { ImportResolutionError } from './Errors.js'; import { parseCode } from './parser.js'; -// A minimal virtual filesystem used to resolve import directives. Canonical paths are opaque keys: -// absolute filesystem paths for the disk resolver, normalised POSIX paths relative to the main -// source for the in-memory resolver. `resolve` returns undefined when a package import -// cannot be located. +// A minimal virtual filesystem used to locate and read imported files. Imported files are identified by their +// logical path (see below), while their location is where a resolver reads them from: their real (symlink-resolved) +// path on disk, or their logical path in memory. `locate` returns undefined when a package import cannot be found. export interface ImportResolver { - rootDir: string; - resolve(fromDir: string, importPath: string): string | undefined; - read(canonicalPath: string): string | undefined; - dirname(canonicalPath: string): string; - sourceName(canonicalPath: string): string; + // Locates an import from the file at `importerLocation` (undefined for the main source) + locate(importerLocation: string | undefined, importPath: string): string | undefined; + read(location: string): string | undefined; } export function createDiskResolver(rootDir: string): ImportResolver { + const realRootDir = toRealPath(rootDir); + return { - rootDir, - resolve: (fromDir, importPath) => (isPackageImport(importPath) - ? resolveFromNodeModules(fromDir, importPath) - : path.resolve(fromDir, importPath)), - read: (canonicalPath) => { + locate: (importerLocation, importPath) => { + const fromDir = importerLocation === undefined ? realRootDir : path.dirname(importerLocation); + const filePath = isPackageImport(importPath) + ? resolveFromNodeModules(fromDir, importPath) + : path.resolve(fromDir, importPath); + + return filePath === undefined ? undefined : toRealPath(filePath); + }, + read: (location) => { try { - return fs.readFileSync(canonicalPath, { encoding: 'utf-8' }); + return fs.readFileSync(location, { encoding: 'utf-8' }); } catch { return undefined; } }, - dirname: (canonicalPath) => path.dirname(canonicalPath), - sourceName: (canonicalPath) => { - const relativePath = getPackageImportPath(canonicalPath) ?? path.relative(rootDir, canonicalPath); - return relativePath.split(path.sep).join(path.posix.sep); - }, }; } +function toRealPath(filePath: string): string { + try { + return fs.realpathSync(filePath); + } catch { + return filePath; + } +} + export function createMemoryResolver(files: Record): ImportResolver { // Normalise keys so that './utils.cash' and 'utils.cash' address the same file const normalisedFiles = Object.fromEntries( - Object.entries(files).map(([filePath, source]) => [path.posix.normalize(filePath), source]), + Object.entries(files).map(([filePath, source]) => [normaliseLogicalPath(filePath), source]), ); return { - rootDir: '.', - // Package imports are looked up verbatim ('pkg/math.cash'), regardless of the importing file - resolve: (fromDir, importPath) => (isPackageImport(importPath) - ? path.posix.normalize(importPath) - : path.posix.normalize(path.posix.join(fromDir, importPath))), - read: (canonicalPath) => normalisedFiles[canonicalPath], - dirname: (canonicalPath) => path.posix.dirname(canonicalPath), - sourceName: (canonicalPath) => canonicalPath, + locate: resolveLogicalPath, + read: (logicalPath) => normalisedFiles[logicalPath], }; } +const PACKAGE_PREFIX = 'package:'; + +function resolveLogicalPath(importerLogicalPath: string | undefined, importPath: string): string { + if (isPackageImport(importPath)) return normaliseLogicalPath(`${PACKAGE_PREFIX}${importPath}`); + + const importerDir = importerLogicalPath === undefined ? '.' : mapLogicalPath(importerLogicalPath, path.posix.dirname); + return mapLogicalPath(importerDir, (dir) => path.posix.join(dir, importPath)); +} + +function normaliseLogicalPath(logicalPath: string): string { + return mapLogicalPath(logicalPath, path.posix.normalize); +} + +function mapLogicalPath(logicalPath: string, operation: (filePath: string) => string): string { + if (!logicalPath.startsWith(PACKAGE_PREFIX)) return operation(logicalPath); + return `${PACKAGE_PREFIX}${operation(logicalPath.slice(PACKAGE_PREFIX.length))}`; +} + export function resolveDependencies( ast: SourceFileNode, resolver: ImportResolver | undefined, errorListener?: CashScriptErrorListener, -): SourceFileNode { - if (ast.imports.length === 0) return ast; +): Record { + if (ast.imports.length === 0) return {}; if (resolver === undefined) { throw new ImportResolutionError( @@ -76,12 +94,12 @@ export function resolveDependencies( ); } - const importedDefinitions = collectImports(ast.imports, resolver, errorListener); - ast.functions = [...importedDefinitions.functions, ...ast.functions]; - ast.constants = [...importedDefinitions.constants, ...ast.constants]; + const { functions, constants, sources } = collectImports(ast.imports, resolver, errorListener); + ast.functions = [...functions, ...ast.functions]; + ast.constants = [...constants, ...ast.constants]; ast.imports = []; - return ast; + return sources; } interface ImportedDefinitions { @@ -89,56 +107,87 @@ interface ImportedDefinitions { constants: ConstantDefinitionNode[]; } -// Depth-first walk of the import graph, returning every global definition it reaches. `visitedPaths` -// de-duplicates files by canonical path so a diamond's shared leaf is read once, while `activePaths` -// tracks the files currently being resolved so cyclic imports are rejected. +interface CollectedImports extends ImportedDefinitions { + sources: Record; +} + +interface ImportedFile { + logicalPath: string; + location: string; +} + +// Depth-first walk of the import graph, returning every global definition it reaches and every imported file's source +// code. Files are de-duplicated by logical path so a diamond's shared leaf is read once, while `activePaths` tracks +// the files currently being resolved so cyclic imports are rejected. function collectImports( imports: ImportNode[], resolver: ImportResolver, errorListener?: CashScriptErrorListener, -): ImportedDefinitions { - const visitedPaths = new Set(); +): CollectedImports { + const sources: Record = {}; + const locations = new Map(); const activePaths = new Set(); - const collect = (currentImports: ImportNode[], currentDir: string): ImportedDefinitions[] => + // A logical path must always refer to the same file, so that the imported sources can be used to recompile the + // contract. This is not the case when different copies of a package are imported (e.g. nested in node_modules). + const locateImport = (importNode: ImportNode, importer?: ImportedFile): ImportedFile => { + // Absolute paths only work on a single machine, and would end up in the artifact's `debug.sources` + if (importNode.path.startsWith('/')) { + throw new ImportResolutionError( + importNode, + `Absolute import paths are not supported, use a relative path or a package import instead of '${importNode.path}'`, + ); + } + + const logicalPath = resolveLogicalPath(importer?.logicalPath, importNode.path); + const location = resolver.locate(importer?.location, importNode.path); + if (location === undefined) { + throw new ImportResolutionError( + importNode, + `Could not find imported file '${importNode.path}' in any node_modules directory`, + ); + } + + const knownLocation = locations.get(logicalPath); + if (knownLocation !== undefined && knownLocation !== location) { + throw new ImportResolutionError( + importNode, + `Imported file '${importNode.path}' resolves to '${location}', but '${logicalPath}' was already imported from '${knownLocation}'. Importing multiple copies of a package is not supported`, + ); + } + + locations.set(logicalPath, location); + return { logicalPath, location }; + }; + + const collect = (currentImports: ImportNode[], importer?: ImportedFile): ImportedDefinitions[] => currentImports.flatMap((importNode) => { - const canonicalPath = resolver.resolve(currentDir, importNode.path); - if (canonicalPath === undefined) { - throw new ImportResolutionError( - importNode, - `Could not find imported file '${importNode.path}' in any node_modules directory`, - ); - } - if (activePaths.has(canonicalPath)) { + const file = locateImport(importNode, importer); + if (activePaths.has(file.logicalPath)) { throw new ImportResolutionError(importNode, `Cyclic import of '${importNode.path}'`); } - if (visitedPaths.has(canonicalPath)) return []; - visitedPaths.add(canonicalPath); + if (sources[file.logicalPath] !== undefined) return []; - const importedSource = resolver.read(canonicalPath); + const importedSource = resolver.read(file.location); if (importedSource === undefined) { throw new ImportResolutionError( importNode, - `Could not read imported file '${importNode.path}' (resolved to '${canonicalPath}')`, + `Could not read imported file '${importNode.path}' (resolved to '${file.location}')`, ); } + sources[file.logicalPath] = importedSource; const importedAst = parseCode(importedSource, errorListener); - checkVersionConstraints(importedAst.pragmas, resolver.sourceName(canonicalPath)); + checkVersionConstraints(importedAst.pragmas, file.logicalPath); // Record source provenance so debug frames can attribute to the imported file - importedAst.functions.forEach((func) => { - func.sourceCode = importedSource; - func.sourceFile = resolver.sourceName(canonicalPath); - }); - importedAst.constants.forEach((constant) => { - constant.sourceCode = importedSource; - constant.sourceFile = resolver.sourceName(canonicalPath); + [...importedAst.functions, ...importedAst.constants].forEach((definition) => { + definition.sourceFile = file.logicalPath; }); - activePaths.add(canonicalPath); - const transitiveDefinitions = collect(importedAst.imports, resolver.dirname(canonicalPath)); - activePaths.delete(canonicalPath); + activePaths.add(file.logicalPath); + const transitiveDefinitions = collect(importedAst.imports, file); + activePaths.delete(file.logicalPath); return [ ...transitiveDefinitions, @@ -146,14 +195,14 @@ function collectImports( ]; }); - const collected = collect(imports, resolver.rootDir); + const collected = collect(imports); return { functions: collected.flatMap((definitions) => definitions.functions), constants: collected.flatMap((definitions) => definitions.constants), + sources, }; } - function isPackageImport(importPath: string): boolean { return !importPath.startsWith('./') && !importPath.startsWith('../') && !importPath.startsWith('/'); } @@ -179,11 +228,3 @@ function isValidCandidate(nodeModulesDir: string, candidate: string): boolean { const stats = fs.statSync(candidate, { throwIfNoEntry: false }); return stats?.isFile() ?? false; } - -// A file inside node_modules is named by its package import path ('pkg/math.cash') -function getPackageImportPath(canonicalPath: string): string | undefined { - const nodeModulesSegment = `${path.sep}node_modules${path.sep}`; - const nodeModulesIndex = canonicalPath.lastIndexOf(nodeModulesSegment); - if (nodeModulesIndex === -1) return undefined; - return canonicalPath.slice(nodeModulesIndex + nodeModulesSegment.length); -} diff --git a/packages/cashc/src/generation/GenerateTargetTraversal.ts b/packages/cashc/src/generation/GenerateTargetTraversal.ts index 96912a644..86c726207 100644 --- a/packages/cashc/src/generation/GenerateTargetTraversal.ts +++ b/packages/cashc/src/generation/GenerateTargetTraversal.ts @@ -287,7 +287,6 @@ export default class GenerateTargetTraversal extends AstTraversal { bytecode: binToHex(scriptToBytecode(optimised.script)), sourceMap: generateSourceMap(optimised.locationData), sourceTags: generateSourceTags(optimised.sourceTags) || undefined, - source: node.sourceCode, sourceFile: node.sourceFile, logs: optimised.logs, requires: optimised.requires, diff --git a/packages/cashc/src/semantic/LowerGlobalConstantsTraversal.ts b/packages/cashc/src/semantic/LowerGlobalConstantsTraversal.ts index 7a38b0577..481f2ac80 100644 --- a/packages/cashc/src/semantic/LowerGlobalConstantsTraversal.ts +++ b/packages/cashc/src/semantic/LowerGlobalConstantsTraversal.ts @@ -91,7 +91,6 @@ function createConstantFunction(constant: ConstantDefinitionNode): FunctionDefin const definition = new FunctionDefinitionNode(FunctionKind.GLOBAL, constant.name, [], body, [constant.type]); definition.constant = constant; definition.location = constant.location; - definition.sourceCode = constant.sourceCode; definition.sourceFile = constant.sourceFile; return definition; } diff --git a/packages/cashc/test/generation/fixtures/import-fixtures/diamond.ts b/packages/cashc/test/generation/fixtures/import-fixtures/diamond.ts index 12d7dc67d..afaf06761 100644 --- a/packages/cashc/test/generation/fixtures/import-fixtures/diamond.ts +++ b/packages/cashc/test/generation/fixtures/import-fixtures/diamond.ts @@ -1,5 +1,3 @@ -import fs from 'fs'; -import { URL } from 'url'; import { Fixture } from '../../fixture-utils.js'; export const fixtures: Fixture[] = [ @@ -23,7 +21,6 @@ export const fixtures: Fixture[] = [ inputs: [{ name: 'a', type: 'int' }], bytecode: '8b', sourceMap: '2:11:2:16:1', - source: fs.readFileSync(new URL('../../../import-fixtures/leaf.cash', import.meta.url), { encoding: 'utf-8' }), sourceFile: 'leaf.cash', logs: [], requires: [], @@ -33,7 +30,6 @@ export const fixtures: Fixture[] = [ inputs: [{ name: 'a', type: 'int' }], bytecode: '8b5295', sourceMap: '3:11:3:18:1;:21::22:0;:11:::1', - source: fs.readFileSync(new URL('../../../import-fixtures/mid1.cash', import.meta.url), { encoding: 'utf-8' }), sourceFile: 'mid1.cash', logs: [], requires: [], @@ -44,13 +40,17 @@ export const fixtures: Fixture[] = [ inputs: [{ name: 'a', type: 'int' }], bytecode: '8b5393', sourceMap: '3:11:3:18:1;:21::22:0;:11:::1', - source: fs.readFileSync(new URL('../../../import-fixtures/mid2.cash', import.meta.url), { encoding: 'utf-8' }), sourceFile: 'mid2.cash', logs: [], requires: [], inlineRanges: '0:0:leaf', }, ], + sources: { + 'mid1.cash': 'import "./leaf.cash";\nfunction m1(int a) returns (int) {\n return leaf(a) * 2;\n}\n', + 'leaf.cash': 'function leaf(int a) returns (int) {\n return a + 1;\n}\n', + 'mid2.cash': 'import "./leaf.cash";\nfunction m2(int a) returns (int) {\n return leaf(a) + 3;\n}\n', + }, inlineRanges: '1:3:m1;5:7:m2', }, }, @@ -90,7 +90,6 @@ export const fixtures: Fixture[] = [ sourceMap: '2:11:2:16:1', logs: [], requires: [], - source: fs.readFileSync(new URL('../../../import-fixtures/leaf.cash', import.meta.url), { encoding: 'utf-8' }), sourceFile: 'leaf.cash', }, { @@ -101,7 +100,6 @@ export const fixtures: Fixture[] = [ sourceMap: '3:11:3:18:1;;:21::22:0;:11:::1', logs: [], requires: [], - source: fs.readFileSync(new URL('../../../import-fixtures/mid1.cash', import.meta.url), { encoding: 'utf-8' }), sourceFile: 'mid1.cash', }, { @@ -112,10 +110,14 @@ export const fixtures: Fixture[] = [ sourceMap: '3:11:3:18:1;;:21::22:0;:11:::1', logs: [], requires: [], - source: fs.readFileSync(new URL('../../../import-fixtures/mid2.cash', import.meta.url), { encoding: 'utf-8' }), sourceFile: 'mid2.cash', }, ], + sources: { + 'mid1.cash': 'import "./leaf.cash";\nfunction m1(int a) returns (int) {\n return leaf(a) * 2;\n}\n', + 'leaf.cash': 'function leaf(int a) returns (int) {\n return a + 1;\n}\n', + 'mid2.cash': 'import "./leaf.cash";\nfunction m2(int a) returns (int) {\n return leaf(a) + 3;\n}\n', + }, }, fingerprint: '316a3305152ec0695bf80303736c79dd1f9cc2f1dbccf57d9965094401363307', }, diff --git a/packages/cashc/test/global-definitions.test.ts b/packages/cashc/test/global-definitions.test.ts index 8560e86cb..ea09f474e 100644 --- a/packages/cashc/test/global-definitions.test.ts +++ b/packages/cashc/test/global-definitions.test.ts @@ -291,9 +291,9 @@ describe('Inlining and shared definitions', () => { expect(artifact.debug?.functions).toContainEqual(expect.objectContaining({ name: 'assertPositive', sourceFile: 'helpers.cash', - source: importedSource, requires: [expect.objectContaining({ line: 2, message: 'must be positive' })], })); + expect(artifact.debug?.sources).toEqual({ 'helpers.cash': importedSource }); expect(artifact.debug?.functions?.[0].id).toBeUndefined(); expect(artifact.debug?.inlineRanges).toMatch(/^\d+:\d+:assertPositive$/); }); @@ -368,9 +368,9 @@ describe('Global constants', () => { expect(artifact.debug?.functions?.[0]).toMatchObject({ name: 'IMPORTED_HASH', kind: 'constant', - source: importedSource, sourceFile: 'constants.cash', }); + expect(artifact.debug?.sources).toEqual({ 'constants.cash': importedSource }); }); it('folds constant definitions to literals at compile time', () => { diff --git a/packages/cashc/test/import-fixtures/absolute_import_main.cash b/packages/cashc/test/import-fixtures/absolute_import_main.cash new file mode 100644 index 000000000..dbcc9afac --- /dev/null +++ b/packages/cashc/test/import-fixtures/absolute_import_main.cash @@ -0,0 +1,7 @@ +import "/absolute/path/math.cash"; + +contract AbsoluteImport() { + function spend(int x) { + require(x == 1); + } +} diff --git a/packages/cashc/test/import-fixtures/copies/main.cash b/packages/cashc/test/import-fixtures/copies/main.cash new file mode 100644 index 000000000..501ebcee5 --- /dev/null +++ b/packages/cashc/test/import-fixtures/copies/main.cash @@ -0,0 +1,8 @@ +import "foo/x.cash"; +import "bar/y.cash"; + +contract PackageCopies() { + function spend(int x) { + require(fooTop(x) + viaBar(x) == 5); + } +} diff --git a/packages/cashc/test/import-fixtures/copies/node_modules/bar/node_modules/foo/x.cash b/packages/cashc/test/import-fixtures/copies/node_modules/bar/node_modules/foo/x.cash new file mode 100644 index 000000000..1bb2163ac --- /dev/null +++ b/packages/cashc/test/import-fixtures/copies/node_modules/bar/node_modules/foo/x.cash @@ -0,0 +1,3 @@ +function fooNested(int a) returns (int) { + return a + 2; +} diff --git a/packages/cashc/test/import-fixtures/copies/node_modules/bar/y.cash b/packages/cashc/test/import-fixtures/copies/node_modules/bar/y.cash new file mode 100644 index 000000000..1fd970e52 --- /dev/null +++ b/packages/cashc/test/import-fixtures/copies/node_modules/bar/y.cash @@ -0,0 +1,5 @@ +import "foo/x.cash"; + +function viaBar(int a) returns (int) { + return fooNested(a); +} diff --git a/packages/cashc/test/import-fixtures/copies/node_modules/foo/x.cash b/packages/cashc/test/import-fixtures/copies/node_modules/foo/x.cash new file mode 100644 index 000000000..4d2fb53a7 --- /dev/null +++ b/packages/cashc/test/import-fixtures/copies/node_modules/foo/x.cash @@ -0,0 +1,3 @@ +function fooTop(int a) returns (int) { + return a + 1; +} diff --git a/packages/cashc/test/imports.test.ts b/packages/cashc/test/imports.test.ts index 862522f8e..3cf87258d 100644 --- a/packages/cashc/test/imports.test.ts +++ b/packages/cashc/test/imports.test.ts @@ -1,5 +1,8 @@ import fs from 'fs'; +import os from 'os'; +import path from 'path'; import { fileURLToPath } from 'url'; +import { Artifact } from '@cashscript/utils'; import { compileFile, compileString } from '../src/internal.js'; import { ImportResolutionError, RedefinitionError, VersionError } from '../src/Errors.js'; @@ -9,6 +12,11 @@ const readFixture = (name: string): string => fs.readFileSync(fixture(name), { e const countOpDefines = (bytecode: string): number => [...bytecode.matchAll(/OP_DEFINE/g)].length; +// An artifact stores the source code of every imported file, so it can be recompiled from itself +const recompile = (artifact: Artifact): Artifact => ( + compileString(artifact.source, { ...artifact.compiler.options, files: artifact.debug?.sources }) +); + describe('Imports from the filesystem (compileFile)', () => { it('merges global functions from an imported file', () => { const artifact = compileFile(fixture('main.cash'), { disableInlining: true }); @@ -39,6 +47,11 @@ describe('Imports from the filesystem (compileFile)', () => { expect(() => compileFile(fixture('missing_import_main.cash'))).toThrow(ImportResolutionError); }); + it('throws on absolute import paths', () => { + expect(() => compileFile(fixture('absolute_import_main.cash'))).toThrow(ImportResolutionError); + expect(() => compileFile(fixture('absolute_import_main.cash'))).toThrow(/Absolute import paths are not supported/); + }); + it('throws when an imported function collides with a local function of the same name', () => { // duplicate_import_main defines `shared` and imports a file that also defines `shared`. expect(() => compileFile(fixture('duplicate_import_main.cash'))).toThrow(RedefinitionError); @@ -159,6 +172,31 @@ describe('Imports from in-memory files (compileString)', () => { expect(() => compileString(code, { files })).toThrow(VersionError); expect(() => compileString(code, { files })).toThrow(/lib\/b\.cash/); }); + + it('stores every imported file in debug.sources, including files without used definitions', () => { + const code = 'import "./forwarding.cash";\ncontract C() { function spend(int x) { require(add(x, 3) == 8); } }'; + const files = { + 'forwarding.cash': 'import "./lib/math.cash";\nimport "./lib/unused.cash";', + 'lib/math.cash': 'function add(int a, int b) returns (int) { return a + b; }', + 'lib/unused.cash': 'int constant UNUSED = 1;', + }; + + const artifact = compileString(code, { files, warningListener: () => {} }); + expect(artifact.debug?.sources).toEqual(files); + }); + + it('does not store debug.sources when nothing is imported', () => { + const artifact = compileString('contract C() { function spend(int x) { require(x == 1); } }'); + expect(artifact.debug?.sources).toBeUndefined(); + }); + + it('throws on absolute import paths', () => { + const code = 'import "/lib/math.cash";\ncontract C() { function spend(int x) { require(add(x, 3) == 8); } }'; + const files = { '/lib/math.cash': 'function add(int a, int b) returns (int) { return a + b; }' }; + + expect(() => compileString(code, { files })).toThrow(ImportResolutionError); + expect(() => compileString(code, { files })).toThrow(/Absolute import paths are not supported/); + }); }); describe('Imports from node_modules (package imports)', () => { @@ -198,7 +236,7 @@ describe('Imports from node_modules (package imports)', () => { expect(artifact.contractName).toEqual('NodeModulesPackageDeps'); expect(countOpDefines(artifact.bytecode)).toEqual(2); expect(artifact.debug?.functions?.map((func) => func.sourceFile).sort()) - .toEqual(['@cashlibs/utils/util.cash', 'mathlib/combined.cash']); + .toEqual(['package:@cashlibs/utils/util.cash', 'package:mathlib/combined.cash']); }); it('resolves scoped package imports', () => { @@ -207,10 +245,16 @@ describe('Imports from node_modules (package imports)', () => { expect(countOpDefines(artifact.bytecode)).toEqual(1); }); - it('records provenance as the package import path rather than a filesystem path', () => { + it('records provenance as the logical package path rather than a filesystem path', () => { const artifact = compileFile(fixture('nm_transitive_main.cash'), { disableInlining: true }); expect(artifact.debug?.functions?.map((func) => func.sourceFile).sort()) - .toEqual(['mathlib/helper.cash', 'mathlib/main.cash']); + .toEqual(['package:mathlib/helper.cash', 'package:mathlib/main.cash']); + }); + + it('throws when different copies of a package are imported', () => { + // copies/main.cash imports foo/x.cash, and bar/y.cash imports bar's own nested copy of foo/x.cash + expect(() => compileFile(fixture('copies/main.cash'))).toThrow(ImportResolutionError); + expect(() => compileFile(fixture('copies/main.cash'))).toThrow(/Importing multiple copies of a package is not supported/); }); it('throws when a package import cannot be found in any node_modules directory', () => { @@ -220,22 +264,44 @@ describe('Imports from node_modules (package imports)', () => { ); }); - it('resolves package imports verbatim from the files option when compiling from a string', () => { + it('resolves package imports from package: keys in the files option when compiling from a string', () => { const code = 'import "mathlib/math.cash";\n' + 'contract C() { function spend(int x) { require(nmAdd(x, 3) == 8); } }'; - const files = { 'mathlib/math.cash': readFixture('node_modules/mathlib/math.cash') }; + const files = { 'package:mathlib/math.cash': readFixture('node_modules/mathlib/math.cash') }; const artifact = compileString(code, { files, disableInlining: true }); expect(countOpDefines(artifact.bytecode)).toEqual(1); - expect(artifact.debug?.functions?.map((func) => func.sourceFile)).toEqual(['mathlib/math.cash']); + expect(artifact.debug?.functions?.map((func) => func.sourceFile)).toEqual(['package:mathlib/math.cash']); + }); + + it('does not resolve package imports from local files in the files option', () => { + // A package import that is missing its './' must fail like it does in compileFile + const code = 'import "lib/math.cash";\n' + + 'contract C() { function spend(int x) { require(nmAdd(x, 3) == 8); } }'; + const files = { 'lib/math.cash': readFixture('node_modules/mathlib/math.cash') }; + + expect(() => compileString(code, { files })).toThrow(ImportResolutionError); + expect(() => compileString(code, { files })).toThrow(/resolved to 'package:lib\/math\.cash'/); + }); + + it('keeps in-memory package files separate from local files with the same path', () => { + const code = 'import "pkg/math.cash";\nimport "./pkg/math.cash";\n' + + 'contract C() { function spend(int x) { require(fromPackage(x) + fromLocal(x) == 3); } }'; + const files = { + 'package:pkg/math.cash': 'function fromPackage(int n) returns (int) { return n + 1; }', + 'pkg/math.cash': 'function fromLocal(int n) returns (int) { return n + 2; }', + }; + + const artifact = compileString(code, { files, disableInlining: true }); + expect(countOpDefines(artifact.bytecode)).toEqual(2); }); it('resolves relative imports inside an in-memory package relative to the package file', () => { const code = 'import "pkg/main.cash";\n' + 'contract C() { function spend(int x) { require(pkgMain(x) == 12); } }'; const files = { - 'pkg/main.cash': 'import "./helper.cash";\nfunction pkgMain(int n) returns (int) { return pkgHelper(n) * 2; }', - 'pkg/helper.cash': 'function pkgHelper(int n) returns (int) { return n + 1; }', + 'package:pkg/main.cash': 'import "./helper.cash";\nfunction pkgMain(int n) returns (int) { return pkgHelper(n) * 2; }', + 'package:pkg/helper.cash': 'function pkgHelper(int n) returns (int) { return n + 1; }', }; const artifact = compileString(code, { files, disableInlining: true }); @@ -246,16 +312,154 @@ describe('Imports from node_modules (package imports)', () => { const fromDisk = compileFile(fixture('nm_transitive_main.cash'), { disableInlining: true }); const files = { - 'mathlib/main.cash': readFixture('node_modules/mathlib/main.cash'), - 'mathlib/helper.cash': readFixture('node_modules/mathlib/helper.cash'), + 'package:mathlib/main.cash': readFixture('node_modules/mathlib/main.cash'), + 'package:mathlib/helper.cash': readFixture('node_modules/mathlib/helper.cash'), }; const fromString = compileString(readFixture('nm_transitive_main.cash'), { files, disableInlining: true }); expect(fromString).toEqual({ ...fromDisk, updatedAt: expect.any(String) }); }); + + it('compiles transitive package dependencies to the exact same artifact from disk and from memory', () => { + const fromDisk = compileFile(fixture('nm_package_deps_main.cash'), { disableInlining: true }); + + const files = { + 'package:mathlib/combined.cash': readFixture('node_modules/mathlib/combined.cash'), + 'package:@cashlibs/utils/util.cash': readFixture('node_modules/@cashlibs/utils/util.cash'), + }; + const fromString = compileString(readFixture('nm_package_deps_main.cash'), { files, disableInlining: true }); + + expect(fromString).toEqual({ ...fromDisk, updatedAt: expect.any(String) }); + }); +}); + +describe('Imports through symlinked packages (pnpm, npm link)', () => { + const PACKAGE_SOURCES = { + 'util.cash': 'function innerDouble(int x) returns (int) { return x * 2; }', + 'lib.cash': 'import "inner-lib/util.cash";\nfunction outerFn(int x) returns (int) { return innerDouble(x) + 1; }', + 'main.cash': 'import "./sub/helper.cash";\nfunction linkedFn(int x) returns (int) { return linkedHelper(x) + 1; }', + 'helper.cash': 'function linkedHelper(int x) returns (int) { return x * 3; }', + }; + + let tempDir: string; + const write = (filePath: string, source: string): void => { + fs.mkdirSync(path.dirname(path.join(tempDir, filePath)), { recursive: true }); + fs.writeFileSync(path.join(tempDir, filePath), source); + }; + // Junctions are used on Windows, so the tests do not require symlink permissions + const link = (target: string, linkPath: string): void => { + fs.mkdirSync(path.dirname(path.join(tempDir, linkPath)), { recursive: true }); + fs.symlinkSync(path.join(tempDir, target), path.join(tempDir, linkPath), 'junction'); + }; + const compileTemp = (filePath: string): Artifact => ( + compileFile(path.join(tempDir, filePath), { disableInlining: true }) + ); + const sourceFiles = (artifact: Artifact): string[] => artifact.debug!.functions!.map((func) => func.sourceFile!).sort(); + + beforeAll(() => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cashc-symlinks-')); + + // pnpm: only direct dependencies are linked into node_modules, and a package's own dependencies are linked next + // to the package in node_modules/.pnpm/@/node_modules + write('pnpm/node_modules/.pnpm/inner-lib@1.0.0/node_modules/inner-lib/util.cash', PACKAGE_SOURCES['util.cash']); + write('pnpm/node_modules/.pnpm/outer-lib@1.0.0/node_modules/outer-lib/lib.cash', PACKAGE_SOURCES['lib.cash']); + link('pnpm/node_modules/.pnpm/inner-lib@1.0.0/node_modules/inner-lib', 'pnpm/node_modules/.pnpm/outer-lib@1.0.0/node_modules/inner-lib'); + link('pnpm/node_modules/.pnpm/outer-lib@1.0.0/node_modules/outer-lib', 'pnpm/node_modules/outer-lib'); + write('pnpm/transitive.cash', 'import "outer-lib/lib.cash";\n' + + 'contract Transitive() { function spend(int x) { require(outerFn(x) == 5); } }'); + + // A project that depends on both outer-lib and inner-lib (which outer-lib also depends on) + link('pnpm/node_modules/.pnpm/outer-lib@1.0.0/node_modules/outer-lib', 'pnpm-shared/node_modules/outer-lib'); + link('pnpm/node_modules/.pnpm/inner-lib@1.0.0/node_modules/inner-lib', 'pnpm-shared/node_modules/inner-lib'); + write('pnpm-shared/shared.cash', 'import "outer-lib/lib.cash";\nimport "inner-lib/util.cash";\n' + + 'contract Shared() { function spend(int x) { require(outerFn(x) + innerDouble(x) == 9); } }'); + + // npm link / workspaces: the package is linked from outside of node_modules + write('workspace/packages/linked/main.cash', PACKAGE_SOURCES['main.cash']); + write('workspace/packages/linked/sub/helper.cash', PACKAGE_SOURCES['helper.cash']); + link('workspace/packages/linked', 'workspace/app/node_modules/linked'); + write('workspace/app/main.cash', 'import "linked/main.cash";\n' + + 'contract Linked() { function spend(int x) { require(linkedFn(x) == 7); } }'); + + // A project that imports a local file through a symlinked directory + write('shared-code/helper.cash', PACKAGE_SOURCES['helper.cash']); + link('shared-code', 'linked-dir/shared'); + write('linked-dir/main.cash', 'import "./shared/helper.cash";\n' + + 'contract LinkedDir() { function spend(int x) { require(linkedHelper(x) == 6); } }'); + + // A project that is itself reached through a symlink + write('project/lib.cash', 'function localFn(int x) returns (int) { return x + 2; }'); + write('project/main.cash', 'import "./lib.cash";\ncontract Project() { function spend(int x) { require(localFn(x) == 4); } }'); + link('project', 'project-link'); + }); + + afterAll(() => { + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + it('resolves a package\'s own dependencies from where the package is installed', () => { + const artifact = compileTemp('pnpm/transitive.cash'); + expect(sourceFiles(artifact)).toEqual(['package:inner-lib/util.cash', 'package:outer-lib/lib.cash']); + }); + + it('imports a package once when it is reached through different symlinks', () => { + const artifact = compileTemp('pnpm-shared/shared.cash'); + expect(countOpDefines(artifact.bytecode)).toEqual(2); + }); + + it('names files of a linked package by their logical package path', () => { + const artifact = compileTemp('workspace/app/main.cash'); + expect(sourceFiles(artifact)).toEqual(['package:linked/main.cash', 'package:linked/sub/helper.cash']); + }); + + it('compiles a linked package to the exact same artifact from disk and from memory', () => { + const files = { + 'package:linked/main.cash': PACKAGE_SOURCES['main.cash'], + 'package:linked/sub/helper.cash': PACKAGE_SOURCES['helper.cash'], + }; + const code = fs.readFileSync(path.join(tempDir, 'workspace/app/main.cash'), 'utf-8'); + + const fromString = compileString(code, { files, disableInlining: true }); + expect(fromString).toEqual({ ...compileTemp('workspace/app/main.cash'), updatedAt: expect.any(String) }); + }); + + it('names local files in a symlinked directory by their import path, like compileString does', () => { + const files = { 'shared/helper.cash': PACKAGE_SOURCES['helper.cash'] }; + const code = fs.readFileSync(path.join(tempDir, 'linked-dir/main.cash'), 'utf-8'); + const fromDisk = compileTemp('linked-dir/main.cash'); + + expect(sourceFiles(fromDisk)).toEqual(['shared/helper.cash']); + expect(compileString(code, { files, disableInlining: true })).toEqual({ ...fromDisk, updatedAt: expect.any(String) }); + }); + + it('recompiles every symlinked layout from the sources in its own artifact', () => { + const mainFiles = ['pnpm/transitive.cash', 'pnpm-shared/shared.cash', 'workspace/app/main.cash', 'linked-dir/main.cash']; + mainFiles.forEach((mainFile) => { + const artifact = compileFile(path.join(tempDir, mainFile)); + expect(recompile(artifact)).toEqual({ ...artifact, updatedAt: expect.any(String) }); + }); + }); + + it('names local imports relative to a main file that is reached through a symlink', () => { + const artifact = compileTemp('project-link/main.cash'); + expect(sourceFiles(artifact)).toEqual(['lib.cash']); + }); }); describe('compileFile / compileString equivalence', () => { + const RECOMPILABLE_FIXTURES = [ + 'main.cash', 'diamond.cash', 'multi_return_main.cash', 'nested_main.cash', 'complex/main.cash', 'nm_main.cash', + 'nm_transitive_main.cash', 'nm_package_deps_main.cash', 'nm_scoped_main.cash', 'nested/nm_nested_main.cash', + 'shadow/nm_shadow_main.cash', + ]; + + RECOMPILABLE_FIXTURES.forEach((file) => { + it(`recompiles ${file} from the sources in its own artifact`, () => { + const artifact = compileFile(fixture(file)); + expect(recompile(artifact)).toEqual({ ...artifact, updatedAt: expect.any(String) }); + }); + }); + it('compiles a complex import graph to the exact same artifact from disk and from memory', () => { // complex/main.cash exercises nested directories, a diamond (a and b both import util/leaf), // and a parent-directory import reached through two different routes (main imports diff --git a/packages/cashscript/src/debug-frame.ts b/packages/cashscript/src/debug-frame.ts index 234131c93..7a82f07cf 100644 --- a/packages/cashscript/src/debug-frame.ts +++ b/packages/cashscript/src/debug-frame.ts @@ -68,7 +68,7 @@ const resolveFrameByBytecode = (artifact: Artifact, activeBytecode: string): Res const resolveDebugFrame = (artifact: Artifact, frame: DebugFrame): ResolvedFrame => ({ sourceMap: frame.sourceMap, - source: frame.source ?? artifact.source, + source: frame.sourceFile === undefined ? artifact.source : artifact.debug!.sources![frame.sourceFile], sourceName: frame.sourceFile ?? `${artifact.contractName}.cash`, ipOffset: 0, // function bodies have no constructor-arg prefix; their ips start at 0 requires: frame.requires, diff --git a/packages/utils/src/artifact.ts b/packages/utils/src/artifact.ts index ba2c34d41..1e664a849 100644 --- a/packages/utils/src/artifact.ts +++ b/packages/utils/src/artifact.ts @@ -20,6 +20,7 @@ export interface DebugInformation { requires: readonly RequireStatement[]; // messages for failing `require` statements sourceTags?: string; // semantic tags for opcodes (e.g. loop update/condition ranges) functions?: readonly DebugFrame[]; // Debug metadata for each global definition (defined frames first, then inlined ones) + sources?: { readonly [logicalPath: string]: string }; // every imported file's source code (compileString's `files`) inlineRanges?: string; // runs where inlined callables' bodies were emitted (see `generateInlineRanges`) } @@ -31,8 +32,7 @@ export interface DebugFrame { bytecode: string; // hex of the function body bytecode (exactly what OP_DEFINE stores and the VM runs) sourceMap: string; // frame-local source map (ips starting from 0) sourceTags?: string; // frame-local semantic tags for opcodes (e.g. loop update/condition ranges) - source?: string; // full text of the defining file; absent means the function lives in the contract's own source file - sourceFile?: string; // originating file name for imported functions; absent means the contract's file + sourceFile?: string; // key of the defining file in `debug.sources`; absent means the function lives in the contract's file logs: readonly LogEntry[]; // frame-local log entries requires: readonly RequireStatement[]; // frame-local require statements inlineRanges?: string; // runs where inlined callables' bodies were emitted within this body (frame-local ips) @@ -165,7 +165,11 @@ function formatObject( if (entries.length === 0) return '{}'; - const formatKey = (key: string): string => (format === 'json' ? JSON.stringify(key) : key); + // TS keys stay bare when they are identifiers, others (e.g. the file names in `debug.sources`) are quoted + const formatKey = (key: string): string => { + if (format === 'json') return JSON.stringify(key); + return /^[A-Za-z_$][\w$]*$/.test(key) ? key : formatString(key, format); + }; const formatted = entries.map( ([key, value]) => `${formatKey(key)}: ${stringify(value, format, indentationLevel + 1)}`, ); diff --git a/packages/utils/src/bitauth-script.ts b/packages/utils/src/bitauth-script.ts index 1a137b7a8..7cd2f7b48 100644 --- a/packages/utils/src/bitauth-script.ts +++ b/packages/utils/src/bitauth-script.ts @@ -13,6 +13,7 @@ export function formatBitAuthScript(debug: DebugInformation, sourceCode: string) sourceMap: debug.sourceMap, sourceTags: debug.sourceTags, functions: debug.functions, + sources: debug.sources, sourceLines, startLine: 1, endLine: sourceLines.length, @@ -27,6 +28,7 @@ interface WalkParams { sourceMap: string; sourceTags?: string; functions?: readonly DebugFrame[]; + sources?: DebugInformation['sources']; sourceLines: string[]; startLine: number; endLine: number; @@ -155,7 +157,7 @@ function renderSegments(segments: Segment[], params: WalkParams): Row[] { function deriveFunctionSectionLines(segments: Segment[], sourceLines: string[]): Set { const localFunctionSegments = segments.filter( - (segment): segment is FunctionDefinitionSegment => segment.kind === 'functionDefinition' && segment.frame.source === undefined, + (segment): segment is FunctionDefinitionSegment => segment.kind === 'functionDefinition' && segment.frame.sourceFile === undefined, ); return new Set(localFunctionSegments.flatMap(({ location }) => { @@ -202,7 +204,7 @@ function renderFunctionDefinition( ): RenderState { const { frame, location } = segment; const isImported = frame.sourceFile !== undefined; - const sourceLines = isImported ? frame.source!.split('\n') : context.sourceLines; + const sourceLines = isImported ? context.sources![frame.sourceFile!].split('\n') : context.sourceLines; const headerRows = isImported ? [{ asm: '', comment: `>>> imported from ${frame.sourceFile}` }] diff --git a/website/docs/compiler/artifacts.md b/website/docs/compiler/artifacts.md index 766dc907f..d4c8d0e68 100644 --- a/website/docs/compiler/artifacts.md +++ b/website/docs/compiler/artifacts.md @@ -28,6 +28,7 @@ interface Artifact { requires: RequireStatement[] // messages for failing `require` statements sourceTags?: string // semantic tags for opcodes (e.g. loop update/condition ranges) functions?: DebugFrame[] // debug metadata for each global definition (defined frames first, then inlined ones) + sources?: Record // source code of every imported file, in the format of compileString's `files` option inlineRanges?: string // "startIp:endIp:name;..." — runs where inlined callables' bodies were emitted } updatedAt: string // Last datetime this artifact was updated (in ISO format) @@ -71,8 +72,7 @@ interface DebugFrame { bytecode: string; // hex-encoded bytecode of the function body (exactly what OP_DEFINE stores) sourceMap: string; // source map of the function body (instruction pointers starting from 0) sourceTags?: string; // semantic tags for opcodes within the function body - source?: string; // full source code of the defining file (only present for imported functions) - sourceFile?: string; // file name the function is imported from (absent for the contract's own file) + sourceFile?: string; // key of the file the function is imported from in `debug.sources` (absent for the contract's own file) logs: LogEntry[]; // log entries within the function body requires: RequireStatement[]; // messages for failing `require` statements within the function body inlineRanges?: string; // runs where inlined callables' bodies were emitted within this body (frame-local ips) diff --git a/website/docs/compiler/compiler.md b/website/docs/compiler/compiler.md index 262083fc0..0c89d7b3e 100644 --- a/website/docs/compiler/compiler.md +++ b/website/docs/compiler/compiler.md @@ -124,9 +124,15 @@ const Doubler = compileString(source, { files: { './math.cash': mathSource } }); ``` :::note -Imports inside imported files are resolved relative to the *importing* file, but their keys in `files` remain relative to the main source. For example, if `lib/a.cash` contains `import "./b.cash";`, that file must be provided under the key `lib/b.cash`. Package imports such as `import "@example/math-lib/math.cash"` are looked up verbatim, so they must be provided under exactly that key. +Imports inside imported files are resolved relative to the *importing* file, but their keys in `files` remain relative to the main source. For example, if `lib/a.cash` contains `import "./b.cash";`, that file must be provided under the key `lib/b.cash`. Files from [npm packages](/docs/language/contracts#importing-from-npm-packages) are provided under their package import path prefixed with `package:`, so `import "@example/math-lib/math.cash";` requires a key of `package:@example/math-lib/math.cash`. Relative imports inside a package file stay inside that package, so if that file contains `import "./utils.cash";`, it must be provided as `package:@example/math-lib/utils.cash`. ::: +Artifacts store the source code of every imported file in `debug.sources`, using the same keys as the `files` option. So a contract with imports can always be recompiled from its artifact, for example to verify that an artifact matches its source code: + +```ts +const recompiled = compileString(artifact.source, { ...artifact.compiler.options, files: artifact.debug?.sources }); +``` + ### Compilation Warnings Some issues, such as unused variables that are not marked [`unused`](/docs/language/contracts#intentionally-unused-values), do not prevent compilation but produce a compiler warning instead. By default these warnings are printed with `console.warn`. When compiling from JavaScript, a custom `warningListener` can be passed as a compiler option to capture the structured warnings instead. It is called for each warning. The default listener is exported as `defaultWarningListener`, so a custom listener can compose with it to keep the standard console output. diff --git a/website/docs/language/contracts.md b/website/docs/language/contracts.md index 2c7f73f50..22e00c010 100644 --- a/website/docs/language/contracts.md +++ b/website/docs/language/contracts.md @@ -192,7 +192,7 @@ Global constants do not become constructor arguments or mutable stack variables. ## Importing functions and constants from other files Top-level functions and constants can be split across files and pulled in with an `import` directive, which makes the imported functions and constants available as if they were declared locally. All `import` directives must appear at the **top of the file** after any `pragma` directives and before any constant, function or contract definitions. Cyclic imports are not allowed and result in a compile error. -Import paths starting with `./`, `../` or `/` are resolved relative to the importing file: from the filesystem when compiling with [`compileFile`](/docs/compiler#compilefile), or from the `files` compiler option when using [`compileString`](/docs/compiler#compilestring). +Import paths starting with `./` or `../` are resolved relative to the importing file: from the filesystem when compiling with [`compileFile`](/docs/compiler#compilefile), or from the `files` compiler option when using [`compileString`](/docs/compiler#compilestring). Absolute import paths (starting with `/`) are not supported. ```solidity // math.cash @@ -220,7 +220,7 @@ Imported function and constant names share a single global namespace, so a name Imported files can declare their own [`pragma` directives](#pragma), and every pragma across the whole import graph — the main file and all (transitively) imported files — must be satisfied by the compiler version. ### Importing from npm packages -Import paths that do *not* start with `./`, `../` or `/` are treated as package imports and are resolved from `node_modules`, mirroring Node.js module resolution. This makes it possible to publish reusable CashScript functions as npm packages and import them by package name. The import path must contain the full path of the `.cash` file within the package. +Other import paths are treated as package imports and are resolved from `node_modules`, mirroring Node.js module resolution. This makes it possible to publish reusable CashScript functions as npm packages and import them by package name. The import path must contain the full path of the `.cash` file within the package. ```solidity pragma cashscript ^0.14.0; @@ -233,9 +233,9 @@ contract Main() { } ``` -Relative imports *inside* an imported package are resolved relative to the package's own files, so packages can split their functions across multiple internal files. +Relative imports *inside* an imported package are resolved relative to the package's own files, so packages can split their functions across multiple internal files. Like in Node.js, symlinks are followed to the package's actual location, so packages installed with pnpm or linked with `npm link` resolve their own dependencies from where they are installed. -When compiling with [`compileString`](/docs/compiler#compilestring), package imports are looked up verbatim in the `files` compiler option. The example above would require a `files` key of `@example/math-lib/math.cash`. +When compiling with [`compileString`](/docs/compiler#compilestring), package files are provided in the `files` compiler option under their import path prefixed with `package:`. The example above would require a `files` key of `package:@example/math-lib/math.cash`. ## Statements CashScript functions are made up of a collection of statements that determine whether money may be spent from the contract.