diff --git a/packages/core/src/helpers/fs.ts b/packages/core/src/helpers/fs.ts index e9f9c12dd..04b0728a3 100644 --- a/packages/core/src/helpers/fs.ts +++ b/packages/core/src/helpers/fs.ts @@ -47,10 +47,15 @@ export const outputJson = async (filepath: string, data: any) => { const writable = fs.createWriteStream(filepath); const readable = new JsonStreamStringify(data, undefined, 2); const prom = new Promise((resolve, reject) => { - readable.on('end', () => { + // Resolve once the data is on disk, not when it's been read. + writable.on('finish', () => { resolve(); }); + writable.on('error', (err) => { + reject(err); + }); + readable.on('error', (err) => { reject(err); }); @@ -59,6 +64,15 @@ export const outputJson = async (filepath: string, data: any) => { return prom; }; +// Return the canonical path, with symlinks resolved, or the input path if it doesn't exist. +export const realpathSync = (filepath: string) => { + try { + return fs.realpathSync(filepath); + } catch { + return filepath; + } +}; + export const outputJsonSync = (filepath: string, data: any) => { // FIXME: This will crash on strings too long. const dataString = JSON.stringify(data, null, 4); diff --git a/packages/plugins/live-debugger/CONTRIBUTING.md b/packages/plugins/live-debugger/CONTRIBUTING.md index 191c71e80..001135be1 100644 --- a/packages/plugins/live-debugger/CONTRIBUTING.md +++ b/packages/plugins/live-debugger/CONTRIBUTING.md @@ -6,6 +6,7 @@ Developer notes for the Live Debugger plugin. - [Development workflow](#development-workflow) +- [Symbol table](#symbol-table) - [Consumer build canary](#consumer-build-canary) - [Fresh checkouts and CI](#fresh-checkouts-and-ci) - [Excalidraw target](#excalidraw-target) @@ -44,6 +45,35 @@ Generated code should keep the dormant runtime path small: call `$dd_probes(func When adding or changing `liveDebugger` configuration, update [`src/types.ts`](./src/types.ts), [`src/validate.ts`](./src/validate.ts), [`src/validate.test.ts`](./src/validate.test.ts), and the consumer-facing [`README.md`](./README.md). +## Symbol table + +The plugin can list the instrumented functions that ship in a build, for upload to Datadog's Symbol Database. Each entry has the function ID, a display name, the path relative to the git repository root, and the original line range. The table lives in [`src/symbol-table`](./src/symbol-table). + +For now, the only consumer is an opt-in debug dump. Set `DD_LIVE_DEBUGGER_SYMBOL_TABLE_PATH` to a file path to generate the table and write it there: + +```bash +DD_LIVE_DEBUGGER_SYMBOL_TABLE_PATH=/tmp/symbol-table.json yarn build +``` + +Without that variable, the plugin doesn't add any hook or build time. The dump is never written inside the bundler's output directory, and generation never fails the build: problems are logged as warnings. + +How it works: + +- After a one-off build, outside watch mode and dev servers, the plugin scans the text of every emitted JavaScript chunk, without parsing it. It doesn't use the transform hook, which doesn't run for modules restored from a persistent cache and doesn't know which functions tree-shaking removes. +- Every instrumented function has exactly one `$dd_probes('')` call at the start of its body and one `$dd_throw(` call at its end. The scan pairs them like brackets, which also gives each function's parent. +- The transform anchors the `$dd_probes` call to the original body's opening brace, which gives the start line through the chunk's final source map. The end line is the last mapped position up to the `$dd_throw` call: for block bodies that's the closing brace. The code appended after an arrow function's expression body has no mapping, and minifiers leave some tokens unmapped, so the closest mapped token before it is used instead. +- The path and display name come from the function ID. Query strings are removed, Yarn PnP virtual paths and symlinks are resolved, and the path is made relative to the git root. Without a git repository, the table is skipped. +- Each chunk's debug ID is read from the runtime bootstrap, when debug IDs are enabled, to record which chunks contain each file. + +When changing instrumentation output, keep exactly one `$dd_probes` call at the start and one `$dd_throw` call at the end of every instrumented function, anchored to the body's braces. [`src/symbolTable.integration.test.ts`](./src/symbolTable.integration.test.ts) checks the table on every bundler. + +Known limitations: + +- esbuild doesn't tell plugins whether it's watching, so its rebuilds always regenerate the table. +- In minified builds, the end line of an arrow function's expression body is the last line the minifier keeps mapped, which can be before the real end. +- Helpers that compilers inject, such as `_define_property`, have no original source, so they have no line range. +- On webpack and Rspack, files that reach the plugin without a source map from an earlier loader get shifted line numbers. + ## Consumer build canary Canary targets build real consumer projects with local Datadog build plugins. diff --git a/packages/plugins/live-debugger/package.json b/packages/plugins/live-debugger/package.json index b4dfb40da..542edd4b4 100644 --- a/packages/plugins/live-debugger/package.json +++ b/packages/plugins/live-debugger/package.json @@ -25,6 +25,7 @@ "dependencies": { "@dd/core": "workspace:*", "@jridgewell/remapping": "2.3.5", + "@jridgewell/trace-mapping": "0.3.31", "chalk": "2.3.1", "picomatch": "4.0.7" }, @@ -32,7 +33,6 @@ "@babel/parser": "7.24.5", "@babel/traverse": "7.24.5", "@babel/types": "7.24.5", - "@jridgewell/trace-mapping": "0.3.31", "@types/picomatch": "4.0.2", "magic-string": "0.30.21", "typescript": "5.4.3" diff --git a/packages/plugins/live-debugger/src/constants.ts b/packages/plugins/live-debugger/src/constants.ts index 0c12cbbb3..2544e2adc 100644 --- a/packages/plugins/live-debugger/src/constants.ts +++ b/packages/plugins/live-debugger/src/constants.ts @@ -10,6 +10,19 @@ export const PLUGIN_NAME: PluginName = 'datadog-live-debugger-plugin' as const; // Skip instrumentation comment export const SKIP_INSTRUMENTATION_COMMENT = '@dd-no-instrumentation'; +// Runtime global that instrumented functions call when they throw. +// The symbol table relies on it marking the end of every instrumented function's body. +export const LIVE_DEBUGGER_THROW_FUNCTION = '$dd_throw'; + +// Function IDs are `;`. +export const FUNCTION_ID_SEPARATOR = ';'; +// Function name used in the IDs of anonymous functions, followed by their location. +export const ANONYMOUS_FUNCTION_NAME = ''; + +// Path where an opt-in debug dump of the symbol table is written. +// Until the symbol table upload exists, setting it is also what enables the symbol table. +export const SYMBOL_TABLE_PATH_ENV = 'DD_LIVE_DEBUGGER_SYMBOL_TABLE_PATH'; + export const DEFAULT_FILE_EXTENSIONS = [ '.js', '.jsx', diff --git a/packages/plugins/live-debugger/src/index.ts b/packages/plugins/live-debugger/src/index.ts index 673b01773..dd07454db 100644 --- a/packages/plugins/live-debugger/src/index.ts +++ b/packages/plugins/live-debugger/src/index.ts @@ -12,6 +12,7 @@ import type { SourceMapCompact, UnpluginBuildContext } from 'unplugin'; import { CONFIG_KEY, PLUGIN_NAME } from './constants'; import { createFileExtensionPattern, createFileFilter } from './filter'; import { getRuntimeBootstrap } from './runtime-bootstrap'; +import { getSymbolTableHooks } from './symbol-table'; import { transformCode } from './transform'; import type { LiveDebuggerOptions, LiveDebuggerOptionsWithDefaults } from './types'; import { validateOptions } from './validate'; @@ -116,6 +117,7 @@ export const getLiveDebuggerPlugin = ( } }, }, + ...getSymbolTableHooks(pluginOptions, context, log), buildEnd: () => { if (totalFunctions > 0) { log.info( diff --git a/packages/plugins/live-debugger/src/symbol-table/build-mode.ts b/packages/plugins/live-debugger/src/symbol-table/build-mode.ts new file mode 100644 index 000000000..97dc53a1d --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/build-mode.ts @@ -0,0 +1,43 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import type { PluginOptions } from '@dd/core/types'; + +export type BuildModeTracker = { + hooks: Pick; + // False in watch mode and on dev servers. esbuild doesn't tell plugins whether it's watching, + // so its builds always count as one-off builds. + isOneOffBuild: () => boolean; +}; + +// Track whether the build is a one-off build, as opposed to watch mode or a dev server. +export const createBuildModeTracker = (): BuildModeTracker => { + let isContinuousBuild = false; + let isCompilerWatching = () => false; + + return { + hooks: { + webpack(compiler) { + isCompilerWatching = () => compiler.watchMode; + }, + rspack(compiler) { + isCompilerWatching = () => compiler.watchMode; + }, + rollup: { + buildStart() { + isContinuousBuild ||= this.meta.watchMode; + }, + }, + vite: { + configResolved(config) { + isContinuousBuild ||= config.command === 'serve'; + }, + buildStart() { + isContinuousBuild ||= this.meta.watchMode; + }, + }, + }, + isOneOffBuild: () => !isContinuousBuild && !isCompilerWatching(), + }; +}; diff --git a/packages/plugins/live-debugger/src/symbol-table/index.test.ts b/packages/plugins/live-debugger/src/symbol-table/index.test.ts new file mode 100644 index 000000000..abb914c00 --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/index.test.ts @@ -0,0 +1,61 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { existsSync, rm } from '@dd/core/helpers/fs'; +import { getTempWorkingDir } from '@dd/tests/_jest/helpers/env'; +import { getMockLogger, mockLogFn } from '@dd/tests/_jest/helpers/mocks'; +import path from 'path'; + +import { generateSymbolTable, writeSymbolTable } from './index'; + +const PAYLOAD = { language: 'browser' as const, scopes: [] }; + +describe('Live Debugger symbol table', () => { + let workingDir: string; + + beforeEach(() => { + const seed = `${Math.abs(jest.getSeed())}.symbol-table-unit.${Date.now()}`; + workingDir = getTempWorkingDir(seed); + }); + + afterEach(async () => { + await rm(workingDir); + }); + + test('Should not write the dump inside the output directory.', async () => { + const outDir = path.resolve(workingDir, 'dist'); + const dumpPath = path.resolve(outDir, 'symbol-table.json'); + + await writeSymbolTable(PAYLOAD, dumpPath, outDir, getMockLogger()); + + expect(existsSync(dumpPath)).toBe(false); + expect(mockLogFn).toHaveBeenCalledWith( + expect.stringContaining('inside the output directory'), + 'warn', + ); + }); + + test('Should write the dump outside the output directory.', async () => { + const outDir = path.resolve(workingDir, 'dist'); + const dumpPath = path.resolve(workingDir, 'symbol-table.json'); + + await writeSymbolTable(PAYLOAD, dumpPath, outDir, getMockLogger()); + + expect(existsSync(dumpPath)).toBe(true); + }); + + test('Should skip the table outside a git repository.', async () => { + const table = await generateSymbolTable({ + outputs: [], + buildRoot: workingDir, + log: getMockLogger(), + }); + + expect(table).toBeUndefined(); + expect(mockLogFn).toHaveBeenCalledWith( + expect.stringContaining("isn't in a git repository"), + 'warn', + ); + }); +}); diff --git a/packages/plugins/live-debugger/src/symbol-table/index.ts b/packages/plugins/live-debugger/src/symbol-table/index.ts new file mode 100644 index 000000000..38dfff69d --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/index.ts @@ -0,0 +1,190 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { outputJson, readFile } from '@dd/core/helpers/fs'; +import type { BuildReport, GlobalContext, Logger, Output, PluginOptions } from '@dd/core/types'; +import path from 'path'; + +import { SYMBOL_TABLE_PATH_ENV } from '../constants'; +import type { LiveDebuggerOptionsWithDefaults } from '../types'; + +import { createBuildModeTracker } from './build-mode'; +import { loadSourceMap, locateFunctions } from './locate'; +import { createSourceFileResolver, getRepositoryRoot, isInsideDirectory } from './paths'; +import { readChunkDebugId, scanChunk } from './scan'; +import type { SymbolTablePayload } from './serialize'; +import { serializeSymbolTable } from './serialize'; +import { buildSymbolTable } from './table'; +import type { ChunkFunction, ChunkFunctions, SymbolTable } from './types'; + +const LOG_PREFIX = 'Live Debugger symbols'; +const JS_OUTPUT_TYPES = new Set(['js', 'mjs', 'cjs']); + +type GenerateSymbolTableArgs = { + outputs: Output[]; + buildRoot: string; + log: Logger; +}; + +const getFirst = (values: Set) => values.values().next().value; + +const byFilepath = (a: Output, b: Output) => { + if (a.filepath === b.filepath) { + return 0; + } + return a.filepath < b.filepath ? -1 : 1; +}; + +/** + * List the instrumented functions that ship in the build's output, with their path relative to + * the repository root and their original line range. + */ +export const generateSymbolTable = async ({ + outputs, + buildRoot, + log, +}: GenerateSymbolTableArgs): Promise => { + const repositoryRoot = getRepositoryRoot(buildRoot); + if (!repositoryRoot) { + log.warn(`Skipping ${LOG_PREFIX}: the build root isn't in a git repository.`); + return undefined; + } + + const chunkOutputs = outputs.filter((output) => JS_OUTPUT_TYPES.has(output.type)); + chunkOutputs.sort(byFilepath); + + const chunks: ChunkFunctions[] = []; + let chunksWithoutSourceMap = 0; + let unpairedChunks = 0; + let chunksWithoutDebugId = 0; + + // One chunk at a time, so at most one source map is in memory. + for (const output of chunkOutputs) { + // eslint-disable-next-line no-await-in-loop + const code = await readFile(output.filepath); + const scan = scanChunk(code); + const debugId = readChunkDebugId(code); + let functions: ChunkFunction[] = []; + + if (scan.functions.length > 0) { + // eslint-disable-next-line no-await-in-loop + const traceMap = await loadSourceMap(output.filepath, code); + functions = locateFunctions(code, scan, traceMap); + chunksWithoutSourceMap += Number(!traceMap); + unpairedChunks += Number(!scan.paired); + chunksWithoutDebugId += Number(!debugId); + } + + chunks.push({ filepath: output.filepath, debugId, functions }); + } + + const resolveSourceFile = createSourceFileResolver(buildRoot, repositoryRoot); + const { table, outsideModulePaths, invalidFunctionIds } = buildSymbolTable( + chunks, + resolveSourceFile, + ); + + if (chunksWithoutSourceMap > 0) { + log.warn( + `${LOG_PREFIX}: ${chunksWithoutSourceMap} chunk(s) have no source map, so their functions have no line numbers.`, + ); + } + if (unpairedChunks > 0) { + log.warn( + `${LOG_PREFIX}: the instrumentation markers of ${unpairedChunks} chunk(s) don't pair up, so their functions have no line numbers.`, + ); + } + if (chunksWithoutDebugId > 0) { + log.warn( + `${LOG_PREFIX}: ${chunksWithoutDebugId} chunk(s) have no debug ID, so chunk debug IDs are left out. Enable "rum.sourceCodeContext.debugId" to include them.`, + ); + } + if (outsideModulePaths.size > 0) { + const exampleModulePath = getFirst(outsideModulePaths); + log.warn( + `${LOG_PREFIX}: left out the functions of ${outsideModulePaths.size} file(s) outside the git repository, such as "${exampleModulePath}".`, + ); + } + if (invalidFunctionIds.size > 0) { + const exampleFunctionId = getFirst(invalidFunctionIds); + log.warn( + `${LOG_PREFIX}: left out ${invalidFunctionIds.size} function(s) with an unexpected ID, such as "${exampleFunctionId}".`, + ); + } + + return table; +}; + +// Write the opt-in debug dump. It must never end up in the deployed output. +export const writeSymbolTable = async ( + payload: SymbolTablePayload, + dumpPath: string, + outDir: string, + log: Logger, +) => { + const absoluteDumpPath = path.resolve(dumpPath); + if (isInsideDirectory(absoluteDumpPath, outDir)) { + log.warn( + `${LOG_PREFIX}: not writing them to "${absoluteDumpPath}", because that's inside the output directory.`, + ); + return; + } + + await outputJson(absoluteDumpPath, payload); + log.info(`${LOG_PREFIX}: wrote ${payload.scopes.length} file(s) to "${absoluteDumpPath}".`); +}; + +/** + * Hooks that generate the symbol table after one-off builds. They're only added when something + * consumes the table, so other builds don't do any extra work. The table never fails the build. + */ +export const getSymbolTableHooks = ( + pluginOptions: LiveDebuggerOptionsWithDefaults, + context: GlobalContext, + log: Logger, +): Partial => { + const dumpPath = process.env[SYMBOL_TABLE_PATH_ENV]; + if (!dumpPath) { + return {}; + } + + const buildMode = createBuildModeTracker(); + + const handleSymbolTable = async (report: BuildReport) => { + // Wait for the injection plugin to finish rewriting esbuild's output. + await context.artifactsReady; + const timer = log.time('generate symbol table'); + try { + const table = await generateSymbolTable({ + outputs: report.outputs ?? [], + buildRoot: context.buildRoot, + log, + }); + if (table) { + const payload = serializeSymbolTable(table, pluginOptions.version); + await writeSymbolTable(payload, dumpPath, context.bundler.outDir, log); + } + } catch (error) { + log.warn(`Could not collect ${LOG_PREFIX}: ${error}`); + } finally { + timer.end(); + } + }; + + return { + ...buildMode.hooks, + async buildReport(report) { + if (!buildMode.isOneOffBuild()) { + return; + } + + const handling = handleSymbolTable(report); + if (context.artifactsPending) { + context.queue(handling); + return; + } + await handling; + }, + }; +}; diff --git a/packages/plugins/live-debugger/src/symbol-table/locate.test.ts b/packages/plugins/live-debugger/src/symbol-table/locate.test.ts new file mode 100644 index 000000000..5dd621146 --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/locate.test.ts @@ -0,0 +1,95 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import type { SourceMapSegment } from '@jridgewell/trace-mapping'; +import { TraceMap } from '@jridgewell/trace-mapping'; + +import { locateFunctions } from './locate'; +import { scanChunk } from './scan'; + +// A minified function: `body` stands for the original code of its body. +const CODE = `let p=$dd_probes('a.js;fn');try{body}catch(e){throw p&&$dd_throw(p,e),e}`; +const PROBE = CODE.indexOf('$dd_probes'); +const BODY = CODE.indexOf('body'); +const THROW = CODE.indexOf('$dd_throw'); +const CATCH = CODE.indexOf('catch'); + +// Segments map a generated column to `[source, original line (0-based), original column]`, +// or to nothing. +type Mapping = [column: number, source?: number, line?: number]; + +const getTraceMap = (mappings: Mapping[]) => { + const segments = mappings.map(([column, source, line]): SourceMapSegment => { + if (source === undefined || line === undefined) { + return [column]; + } + return [column, source, line, 0]; + }); + + return new TraceMap({ + version: 3, + sources: ['a.js', 'b.js'], + names: [], + mappings: [segments], + }); +}; + +describe('locateFunctions', () => { + const cases = [ + { + description: 'use the original positions of both markers', + mappings: [ + [PROBE, 0, 0], + [BODY, 0, 1], + [THROW, 0, 4], + ], + expected: { start: 1, end: 5 }, + }, + { + description: 'end at the last mapped token when the closing marker is not mapped', + mappings: [[PROBE, 0, 0], [BODY, 0, 2], [THROW - 8], [THROW]], + expected: { start: 1, end: 3 }, + }, + { + description: 'start at the first mapped token when the opening marker is not mapped', + mappings: [[0], [PROBE], [BODY, 0, 6], [THROW, 0, 8]], + expected: { start: 7, end: 9 }, + }, + { + description: 'ignore tokens of another source when looking for the end', + mappings: [[PROBE, 0, 0], [BODY, 0, 2], [CATCH, 1, 40], [THROW]], + expected: { start: 1, end: 3 }, + }, + { + description: 'leave out positions when nothing in the function is mapped', + mappings: [[0], [PROBE], [THROW]], + expected: undefined, + }, + ] satisfies { description: string; mappings: Mapping[]; expected: unknown }[]; + + test.each(cases)('Should $description.', ({ mappings, expected }) => { + const traceMap = getTraceMap(mappings); + const scan = scanChunk(CODE); + const [located] = locateFunctions(CODE, scan, traceMap); + const lines = located.start && + located.end && { + start: located.start.line, + end: located.end.line, + }; + + expect(lines).toEqual(expected); + }); + + test('Should leave out positions without a source map.', () => { + const scan = scanChunk(CODE); + const [located] = locateFunctions(CODE, scan, undefined); + + expect(located).toEqual({ + functionId: 'a.js;fn', + parentIndex: undefined, + start: undefined, + end: undefined, + }); + }); +}); diff --git a/packages/plugins/live-debugger/src/symbol-table/locate.ts b/packages/plugins/live-debugger/src/symbol-table/locate.ts new file mode 100644 index 000000000..1efe4c99a --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/locate.ts @@ -0,0 +1,215 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { existsSync, readFile } from '@dd/core/helpers/fs'; +import { decodedMappings, TraceMap, traceSegment } from '@jridgewell/trace-mapping'; +import type { SourceMapSegment } from '@jridgewell/trace-mapping'; +import path from 'path'; + +import type { ChunkScan } from './scan'; +import type { ChunkFunction, SourcePosition } from './types'; + +const SOURCE_MAPPING_URL_RX = /[#@]\s*sourceMappingURL=(\S+)\s*$/; +const DATA_URL_RX = /^data:[^,]*?(;base64)?,(.*)$/; +// The `sourceMappingURL` comment is at the end of the chunk. +const SOURCE_MAPPING_URL_LOOKBEHIND = 4096; + +// Load the chunk's final source map: `.map`, or the one its `sourceMappingURL` points to. +export const loadSourceMap = async ( + filepath: string, + code: string, +): Promise => { + try { + const siblingMapPath = `${filepath}.map`; + if (existsSync(siblingMapPath)) { + const siblingMap = await readFile(siblingMapPath); + return new TraceMap(siblingMap); + } + + const tail = code.slice(-SOURCE_MAPPING_URL_LOOKBEHIND); + const url = SOURCE_MAPPING_URL_RX.exec(tail)?.[1]; + if (!url) { + return undefined; + } + + const dataUrl = DATA_URL_RX.exec(url); + if (dataUrl) { + const [, base64, payload] = dataUrl; + const inlineMap = base64 + ? Buffer.from(payload, 'base64').toString('utf-8') + : decodeURIComponent(payload); + return new TraceMap(inlineMap); + } + + const chunkDir = path.dirname(filepath); + const linkedMapPath = path.resolve(chunkDir, url); + if (!existsSync(linkedMapPath)) { + return undefined; + } + const linkedMap = await readFile(linkedMapPath); + return new TraceMap(linkedMap); + } catch { + return undefined; + } +}; + +// Return a function that converts an offset in `code` to a 1-based line and 0-based column. +const createOffsetConverter = (code: string) => { + const lineStarts = [0]; + let newline = code.indexOf('\n'); + while (newline !== -1) { + lineStarts.push(newline + 1); + newline = code.indexOf('\n', newline + 1); + } + + return (offset: number): SourcePosition => { + let low = 0; + let high = lineStarts.length - 1; + while (low < high) { + const middle = Math.ceil((low + high) / 2); + if (lineStarts[middle] <= offset) { + low = middle; + } else { + high = middle - 1; + } + } + + return { line: low + 1, column: offset - lineStarts[low] }; + }; +}; + +type MappedPosition = SourcePosition & { + sourceIndex: number; +}; + +// The original position of the closest segment at or before a generated position, if it's mapped. +const getMappedPosition = ( + traceMap: TraceMap, + generated: SourcePosition, +): MappedPosition | undefined => { + const segment = traceSegment(traceMap, generated.line - 1, generated.column); + if (!segment || segment.length === 1) { + return undefined; + } + + return { line: segment[2] + 1, column: segment[3], sourceIndex: segment[1] }; +}; + +// Index of the first segment at or after `column`. +const findSegmentIndex = (segments: readonly SourceMapSegment[], column: number): number => { + let low = 0; + let high = segments.length; + while (low < high) { + const middle = Math.floor((low + high) / 2); + if (segments[middle][0] < column) { + low = middle + 1; + } else { + high = middle; + } + } + return low; +}; + +// The original position of the first mapped segment between two generated positions, both included. +const getFirstMappedPosition = ( + traceMap: TraceMap, + from: SourcePosition, + to: SourcePosition, +): MappedPosition | undefined => { + const lines = decodedMappings(traceMap); + + for (let line = from.line; line <= to.line; line++) { + const segments = lines[line - 1] ?? []; + const firstIndex = line === from.line ? findSegmentIndex(segments, from.column) : 0; + const lastColumn = line === to.line ? to.column : Infinity; + + for ( + let index = firstIndex; + index < segments.length && segments[index][0] <= lastColumn; + index++ + ) { + const segment = segments[index]; + if (segment.length !== 1) { + return { line: segment[2] + 1, column: segment[3], sourceIndex: segment[1] }; + } + } + } + + return undefined; +}; + +/** + * The original position of the last mapped segment of `sourceIndex` between two generated + * positions, both included. Some tokens have no mapping: the code the transform appends after + * an arrow function's expression body, and tokens that minifiers rewrite. The closest mapped token + * before the `$dd_throw` call is then the best estimate of where the function ends. + */ +const getLastMappedPosition = ( + traceMap: TraceMap, + from: SourcePosition, + to: SourcePosition, + sourceIndex: number, +): SourcePosition | undefined => { + const lines = decodedMappings(traceMap); + + for (let line = to.line; line >= from.line; line--) { + const segments = lines[line - 1] ?? []; + const lastIndex = + line === to.line ? findSegmentIndex(segments, to.column + 1) - 1 : segments.length - 1; + const firstColumn = line === from.line ? from.column : 0; + + for (let index = lastIndex; index >= 0 && segments[index][0] >= firstColumn; index--) { + const segment = segments[index]; + if (segment.length !== 1 && segment[1] === sourceIndex) { + return { line: segment[2] + 1, column: segment[3] }; + } + } + } + + return undefined; +}; + +/** + * Map the scanned functions to their original positions. The transform anchors the `$dd_probes` + * call to the body's opening brace, and the `$dd_throw` call to its closing brace when the body + * is a block. When a marker isn't mapped, the closest mapped token inside the function is used. + */ +export const locateFunctions = ( + code: string, + scan: ChunkScan, + traceMap: TraceMap | undefined, +): ChunkFunction[] => { + const toPosition = createOffsetConverter(code); + + return scan.functions.map((scannedFunction) => { + const { functionId, probeOffset, throwOffset } = scannedFunction; + const parentIndex = scan.paired ? scannedFunction.parentIndex : undefined; + + if (!traceMap || !scan.paired || throwOffset === undefined) { + return { functionId, parentIndex, start: undefined, end: undefined }; + } + + const generatedStart = toPosition(probeOffset); + const generatedEnd = toPosition(throwOffset); + const mappedStart = + getMappedPosition(traceMap, generatedStart) ?? + getFirstMappedPosition(traceMap, generatedStart, generatedEnd); + if (!mappedStart) { + return { functionId, parentIndex, start: undefined, end: undefined }; + } + + const start = { line: mappedStart.line, column: mappedStart.column }; + const end = getLastMappedPosition( + traceMap, + generatedStart, + generatedEnd, + mappedStart.sourceIndex, + ); + if (!end || end.line < start.line) { + return { functionId, parentIndex, start: undefined, end: undefined }; + } + + return { functionId, parentIndex, start, end }; + }); +}; diff --git a/packages/plugins/live-debugger/src/symbol-table/paths.test.ts b/packages/plugins/live-debugger/src/symbol-table/paths.test.ts new file mode 100644 index 000000000..3bfdc0e23 --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/paths.test.ts @@ -0,0 +1,124 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { + createSourceFileResolver, + isInsideDirectory, + resolveYarnVirtualPath, + stripQuery, +} from './paths'; + +// Paths that don't exist, so resolving symlinks leaves them unchanged. +const REPOSITORY_ROOT = '/does-not-exist/repo'; +const BUILD_ROOT = `${REPOSITORY_ROOT}/app`; + +describe('stripQuery', () => { + const cases = [ + { description: 'keep a path without a query', input: 'src/a.js', expected: 'src/a.js' }, + { + description: 'remove a loader query', + input: 'src/a.js?inline', + expected: 'src/a.js', + }, + { + description: 'remove a Vue single-file component query', + input: 'src/Cart.vue?vue&type=script&lang.ts', + expected: 'src/Cart.vue', + }, + ]; + + test.each(cases)('Should $description.', ({ input, expected }) => { + expect(stripQuery(input)).toBe(expected); + }); +}); + +describe('resolveYarnVirtualPath', () => { + const cases = [ + { + description: 'keep a regular path', + input: '/repo/packages/a/index.js', + expected: '/repo/packages/a/index.js', + }, + { + description: 'resolve a web-ui workspace package', + input: '/web-ui/.yarn/__virtual__/hash-05a436a78a514e39/1/packages/apps/databases/toolkit/group.ts', + expected: '/web-ui/packages/apps/databases/toolkit/group.ts', + }, + { + description: 'go up as many levels as the depth', + input: '/repo/sub/.yarn/__virtual__/pkg-virtual-abc/2/packages/a.js', + expected: '/repo/packages/a.js', + }, + { + description: 'stay in the virtual folder parent with a depth of 0', + input: '/repo/.yarn/__virtual__/pkg-virtual-abc/0/cache/a.js', + expected: '/repo/.yarn/cache/a.js', + }, + { + description: 'resolve the name used by older Yarn versions', + input: '/repo/.yarn/$$virtual/pkg-virtual-abc/1/packages/a.js', + expected: '/repo/packages/a.js', + }, + { + description: 'resolve nested virtual paths', + input: '/repo/.yarn/__virtual__/a-virtual-1/1/.yarn/__virtual__/b-virtual-2/1/packages/b.js', + expected: '/repo/packages/b.js', + }, + ]; + + test.each(cases)('Should $description.', ({ input, expected }) => { + expect(resolveYarnVirtualPath(input)).toBe(expected); + }); +}); + +describe('createSourceFileResolver', () => { + const cases = [ + { + description: 'make a path relative to the repository root', + input: 'src/a.js', + expected: 'app/src/a.js', + }, + { + description: 'strip the query string', + input: 'src/Cart.vue?vue&type=script', + expected: 'app/src/Cart.vue', + }, + { + description: 'resolve a workspace package outside the build root', + input: '../packages/lib/index.js', + expected: 'packages/lib/index.js', + }, + { + description: 'resolve a Yarn virtual path', + input: '../.yarn/__virtual__/lib-virtual-abc/1/packages/lib/index.js', + expected: 'packages/lib/index.js', + }, + { + description: 'leave out a file outside the repository', + input: '../../shared/index.js', + expected: undefined, + }, + ]; + + test.each(cases)('Should $description.', ({ input, expected }) => { + const resolveSourceFile = createSourceFileResolver(BUILD_ROOT, REPOSITORY_ROOT); + expect(resolveSourceFile(input)).toBe(expected); + }); +}); + +describe('isInsideDirectory', () => { + const cases = [ + { description: 'accept a nested file', input: '/out/dir/a.json', expected: true }, + { description: 'reject a sibling file', input: '/out/a.json', expected: false }, + { + description: 'reject a directory that only shares a prefix', + input: '/out/dir-2/a.json', + expected: false, + }, + ]; + + test.each(cases)('Should $description.', ({ input, expected }) => { + expect(isInsideDirectory(input, '/out/dir')).toBe(expected); + }); +}); diff --git a/packages/plugins/live-debugger/src/symbol-table/paths.ts b/packages/plugins/live-debugger/src/symbol-table/paths.ts new file mode 100644 index 000000000..6dfd7ef6b --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/paths.ts @@ -0,0 +1,85 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { realpathSync } from '@dd/core/helpers/fs'; +import { getClosest } from '@dd/core/helpers/paths'; +import path from 'path'; + +// `/__virtual__///`, as created by Yarn PnP for packages with peer +// dependencies. `$$virtual` is the name used by older Yarn versions. +const YARN_VIRTUAL_PATH_RX = /^(.*\/(?:__virtual__|\$\$virtual))\/[^/]+\/(\d+)(\/.*)?$/; + +const toPosixPath = (filepath: string) => filepath.replace(/\\/g, '/'); + +// Remove the query string of a module ID, e.g. `Cart.vue?vue&type=script`. +export const stripQuery = (moduleId: string): string => { + const queryStart = moduleId.indexOf('?'); + return queryStart === -1 ? moduleId : moduleId.slice(0, queryStart); +}; + +/** + * Resolve a Yarn PnP virtual path to the path it stands for, the way Yarn does: for + * `/__virtual__///`, go up `depth` levels from `` and append ``. + */ +export const resolveYarnVirtualPath = (filepath: string): string => { + const posixPath = toPosixPath(filepath); + const match = YARN_VIRTUAL_PATH_RX.exec(posixPath); + if (!match) { + return filepath; + } + + const [, virtualDir, depth, rest = ''] = match; + const target = path.posix.dirname(virtualDir); + const backtrack = '../'.repeat(Number(depth)); + const resolved = path.posix.join(target, backtrack, rest); + + // A resolved path can itself be virtual. + return resolveYarnVirtualPath(resolved); +}; + +// Return the canonical root of the git repository that contains the build root. +export const getRepositoryRoot = (buildRoot: string): string | undefined => { + const gitPath = getClosest(buildRoot, '.git'); + if (!gitPath) { + return undefined; + } + + const repositoryRoot = path.dirname(gitPath); + return realpathSync(repositoryRoot); +}; + +const isOutside = (relativePath: string) => + relativePath === '' || + relativePath === '..' || + relativePath.startsWith(`..${path.sep}`) || + path.isAbsolute(relativePath); + +export const isInsideDirectory = (filepath: string, directory: string): boolean => { + const relativePath = path.relative(directory, filepath); + return relativePath === '' || !isOutside(relativePath); +}; + +/** + * Return a function that converts the module path of a function ID to a path relative to the + * repository root, or to undefined when the module is outside the repository. + */ +export const createSourceFileResolver = (buildRoot: string, repositoryRoot: string) => { + const cache = new Map(); + + return (modulePath: string): string | undefined => { + if (cache.has(modulePath)) { + return cache.get(modulePath); + } + + const moduleFile = stripQuery(modulePath); + const absolutePath = path.resolve(buildRoot, moduleFile); + const unvirtualizedPath = resolveYarnVirtualPath(absolutePath); + const realPath = realpathSync(unvirtualizedPath); + const relativePath = path.relative(repositoryRoot, realPath); + const sourceFile = isOutside(relativePath) ? undefined : toPosixPath(relativePath); + + cache.set(modulePath, sourceFile); + return sourceFile; + }; +}; diff --git a/packages/plugins/live-debugger/src/symbol-table/scan.test.ts b/packages/plugins/live-debugger/src/symbol-table/scan.test.ts new file mode 100644 index 000000000..c6f4f7726 --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/scan.test.ts @@ -0,0 +1,117 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { readChunkDebugId, scanChunk } from './scan'; + +const DEBUG_ID = '0f5e4a5c-3b1d-4c8e-9a7f-1b2c3d4e5f60'; + +describe('scanChunk', () => { + const cases = [ + { + description: 'read single, double and backtick quoted function IDs', + code: `$dd_probes('a.js;one');$dd_throw(p);$dd_probes("a.js;two");$dd_throw(p);$dd_probes(\`a.js;three\`);$dd_throw(p);`, + expected: { + functionIds: ['a.js;one', 'a.js;two', 'a.js;three'], + parentIndexes: [undefined, undefined, undefined], + paired: true, + }, + }, + { + description: 'decode escape sequences', + code: String.raw`$dd_probes('a.js;it\'s');$dd_throw(p);$dd_probes("a.js;\x41\u0042\u{43}\\");$dd_throw(p);`, + expected: { + functionIds: ["a.js;it's", 'a.js;ABC\\'], + parentIndexes: [undefined, undefined], + paired: true, + }, + }, + { + description: 'nest functions like brackets', + code: `function a(){let p=$dd_probes('a.js;outer');try{const b=()=>{let q=$dd_probes('a.js;inner');try{}catch(e){$dd_throw(q,e)}};}catch(e){$dd_throw(p,e)}}function c(){let r=$dd_probes('a.js;sibling');try{}catch(e){$dd_throw(r,e)}}`, + expected: { + functionIds: ['a.js;outer', 'a.js;inner', 'a.js;sibling'], + parentIndexes: [undefined, 0, undefined], + paired: true, + }, + }, + { + description: 'accept whitespace around the argument', + code: `$dd_probes ( 'a.js;spaced' ) ;$dd_throw (p);`, + expected: { + functionIds: ['a.js;spaced'], + parentIndexes: [undefined], + paired: true, + }, + }, + { + description: 'ignore member calls, declarations and non-constant arguments', + code: `obj.$dd_probes('a.js;member');function $dd_throw(a){}x$dd_probes('a.js;prefixed');$dd_probes(id);$dd_probes(\`a.js;\${name}\`);$dd_probes('a.js;' + name);`, + expected: { + functionIds: [], + parentIndexes: [], + paired: true, + }, + }, + { + description: 'report a closing marker without an opening one', + code: `$dd_throw(p);$dd_probes('a.js;one');$dd_throw(p);`, + expected: { + functionIds: ['a.js;one'], + parentIndexes: [undefined], + paired: false, + }, + }, + { + description: 'report an opening marker without a closing one', + code: `$dd_probes('a.js;one');`, + expected: { + functionIds: ['a.js;one'], + parentIndexes: [undefined], + paired: false, + }, + }, + ]; + + test.each(cases)('Should $description.', ({ code, expected }) => { + const scan = scanChunk(code); + const functionIds = scan.functions.map((scanned) => scanned.functionId); + const parentIndexes = scan.functions.map((scanned) => scanned.parentIndex); + + expect(functionIds).toEqual(expected.functionIds); + expect(parentIndexes).toEqual(expected.parentIndexes); + expect(scan.paired).toBe(expected.paired); + }); + + test('Should record the offsets of both markers.', () => { + const code = `let p=$dd_probes('a.js;one');try{}catch(e){$dd_throw(p,e)}`; + const [scanned] = scanChunk(code).functions; + + expect(scanned.probeOffset).toBe(code.indexOf('$dd_probes')); + expect(scanned.throwOffset).toBe(code.indexOf('$dd_throw')); + }); +}); + +describe('readChunkDebugId', () => { + const cases = [ + { + description: 'read the debug ID of the runtime bootstrap', + code: `globalThis.__DD_LIVE_DEBUGGER_BUILD__={"version":"1.0.0","debugId":"${DEBUG_ID}"}`, + expected: DEBUG_ID, + }, + { + description: 'read the debug ID of a minified runtime bootstrap', + code: `globalThis.__DD_LIVE_DEBUGGER_BUILD__={version:"1.0.0",debugId:'${DEBUG_ID}'}`, + expected: DEBUG_ID, + }, + { + description: 'return undefined without a debug ID', + code: `globalThis.__DD_LIVE_DEBUGGER_BUILD__={"version":"1.0.0"}`, + expected: undefined, + }, + ]; + + test.each(cases)('Should $description.', ({ code, expected }) => { + expect(readChunkDebugId(code)).toBe(expected); + }); +}); diff --git a/packages/plugins/live-debugger/src/symbol-table/scan.ts b/packages/plugins/live-debugger/src/symbol-table/scan.ts new file mode 100644 index 000000000..447616750 --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/scan.ts @@ -0,0 +1,244 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { LIVE_DEBUGGER_PROBES_FUNCTION } from '@dd/core/constants'; + +import { LIVE_DEBUGGER_THROW_FUNCTION } from '../constants'; + +// An instrumented function found in a chunk's code. +export type ScannedFunction = { + functionId: string; + // Offset of the `$dd_probes('')` call at the start of the function's body. + probeOffset: number; + // Offset of the `$dd_throw(` call at the end of the function's body. + throwOffset: number | undefined; + // Index of the enclosing instrumented function in the same list. + parentIndex: number | undefined; +}; + +export type ChunkScan = { + functions: ScannedFunction[]; + // False when the markers don't nest, so `throwOffset` and `parentIndex` can't be trusted. + paired: boolean; +}; + +const escapeRegExp = (value: string) => value.replace(/[$()*+.?[\\\]^{|}]/g, '\\$&'); + +const MARKER_RX = new RegExp( + `(${escapeRegExp(LIVE_DEBUGGER_PROBES_FUNCTION)}|${escapeRegExp(LIVE_DEBUGGER_THROW_FUNCTION)})\\s*\\(`, + 'g', +); +// A marker preceded by one of these is another identifier or a member access, not a call. +const NON_CALL_PREFIX_RX = /[\w$.]/; +// A marker preceded by this is a function declaration, not a call. +const FUNCTION_DECLARATION_SUFFIX_RX = /\bfunction\s*\*?\s*$/; +const FUNCTION_DECLARATION_LOOKBEHIND = 16; +// The runtime bootstrap injected into every chunk records the chunk's debug ID. +// Minifiers can drop the quotes around the key and change the quotes around the value. +const BUILD_METADATA_DEBUG_ID_RX = + /__DD_LIVE_DEBUGGER_BUILD__\s*=\s*\{[^}]*?["']?debugId["']?\s*:\s*["']([0-9a-fA-F-]{36})["']/; + +const QUOTES = new Set(["'", '"', '`']); +const WHITESPACE_RX = /\s/; +const HEX_RX = /^[0-9a-fA-F]+$/; +const SINGLE_CHARACTER_ESCAPES: Record = { + b: '\b', + f: '\f', + n: '\n', + r: '\r', + t: '\t', + v: '\v', + 0: '\0', +}; +const LINE_CONTINUATIONS = new Set(['\n', '\u2028', '\u2029']); + +type Decoded = { + value: string; + // Offset right after the decoded source text. + end: number; +}; + +const skipWhitespace = (code: string, offset: number): number => { + let index = offset; + while (index < code.length && WHITESPACE_RX.test(code[index])) { + index++; + } + return index; +}; + +const isCallSite = (code: string, offset: number): boolean => { + if (offset > 0 && NON_CALL_PREFIX_RX.test(code[offset - 1])) { + return false; + } + + const lookbehindStart = Math.max(0, offset - FUNCTION_DECLARATION_LOOKBEHIND); + const lookbehind = code.slice(lookbehindStart, offset); + return !FUNCTION_DECLARATION_SUFFIX_RX.test(lookbehind); +}; + +const decodeCodePoint = (hex: string, end: number): Decoded | undefined => { + if (!HEX_RX.test(hex)) { + return undefined; + } + + const codePoint = parseInt(hex, 16); + if (codePoint > 0x10ffff) { + return undefined; + } + + return { value: String.fromCodePoint(codePoint), end }; +}; + +// Decode the escape sequence whose backslash is right before `offset`. +const decodeEscape = (code: string, offset: number): Decoded | undefined => { + const char = code[offset]; + if (char === undefined) { + return undefined; + } + + if (char === 'x') { + const hex = code.slice(offset + 1, offset + 3); + return hex.length === 2 ? decodeCodePoint(hex, offset + 3) : undefined; + } + + if (char === 'u') { + if (code[offset + 1] === '{') { + const closingBrace = code.indexOf('}', offset + 2); + if (closingBrace === -1) { + return undefined; + } + const hex = code.slice(offset + 2, closingBrace); + return decodeCodePoint(hex, closingBrace + 1); + } + + const hex = code.slice(offset + 1, offset + 5); + return hex.length === 4 ? decodeCodePoint(hex, offset + 5) : undefined; + } + + if (char === '\r') { + const end = code[offset + 1] === '\n' ? offset + 2 : offset + 1; + return { value: '', end }; + } + + if (LINE_CONTINUATIONS.has(char)) { + return { value: '', end: offset + 1 }; + } + + return { value: SINGLE_CHARACTER_ESCAPES[char] ?? char, end: offset + 1 }; +}; + +// Decode the string literal starting at `offset`, which must be its opening quote. +const decodeStringLiteral = (code: string, offset: number): Decoded | undefined => { + const quote = code[offset]; + if (!QUOTES.has(quote)) { + return undefined; + } + + const isTemplate = quote === '`'; + let value = ''; + let index = offset + 1; + let segmentStart = index; + + while (index < code.length) { + const char = code[index]; + + if (char === quote) { + value += code.slice(segmentStart, index); + return { value, end: index + 1 }; + } + + if (char === '\\') { + value += code.slice(segmentStart, index); + const escape = decodeEscape(code, index + 1); + if (!escape) { + return undefined; + } + value += escape.value; + index = escape.end; + segmentStart = index; + continue; + } + + // Only constant strings are function IDs. + if (isTemplate && char === '$' && code[index + 1] === '{') { + return undefined; + } + + if (!isTemplate && (char === '\n' || char === '\r')) { + return undefined; + } + + index++; + } + + return undefined; +}; + +// Read the function ID of a `$dd_probes('')` call, from right after its opening parenthesis. +const readFunctionIdArgument = (code: string, offset: number): string | undefined => { + const literalStart = skipWhitespace(code, offset); + const literal = decodeStringLiteral(code, literalStart); + if (!literal) { + return undefined; + } + + const closingParenthesis = skipWhitespace(code, literal.end); + return code[closingParenthesis] === ')' ? literal.value : undefined; +}; + +/** + * Find the instrumented functions in a chunk's code, without parsing it. + * + * Every instrumented function contains exactly one `$dd_probes('')` call at the start of its + * body and one `$dd_throw(` call at its end, in the injected `catch`. Minifiers keep both, because + * they are calls to globals. Pairing them like brackets gives each function's extent, and the + * nesting gives its enclosing function. + */ +export const scanChunk = (code: string): ChunkScan => { + const functions: ScannedFunction[] = []; + const openFunctions: number[] = []; + let paired = true; + + for (const match of code.matchAll(MARKER_RX)) { + const offset = match.index; + if (!isCallSite(code, offset)) { + continue; + } + + if (match[1] === LIVE_DEBUGGER_PROBES_FUNCTION) { + const argumentOffset = offset + match[0].length; + const functionId = readFunctionIdArgument(code, argumentOffset); + if (functionId === undefined) { + continue; + } + + functions.push({ + functionId, + probeOffset: offset, + throwOffset: undefined, + parentIndex: openFunctions.at(-1), + }); + openFunctions.push(functions.length - 1); + continue; + } + + const closedFunction = openFunctions.pop(); + if (closedFunction === undefined) { + paired = false; + continue; + } + functions[closedFunction].throwOffset = offset; + } + + if (openFunctions.length > 0) { + paired = false; + } + + return { functions, paired }; +}; + +// Read the chunk's debug ID from the runtime bootstrap. +export const readChunkDebugId = (code: string): string | undefined => { + return BUILD_METADATA_DEBUG_ID_RX.exec(code)?.[1]; +}; diff --git a/packages/plugins/live-debugger/src/symbol-table/serialize.ts b/packages/plugins/live-debugger/src/symbol-table/serialize.ts new file mode 100644 index 000000000..db6887e07 --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/serialize.ts @@ -0,0 +1,126 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import type { SourcePosition, SymbolTable, SymbolTableEntry } from './types'; + +// SymDB language of browser symbol tables. +export const SYMBOL_TABLE_LANGUAGE = 'browser'; + +// A SymDB scope. +export type SymDbScope = { + scope_type: 'MODULE' | 'FUNCTION'; + name: string; + source_file: string; + start_line?: number; + end_line?: number; + language_specifics?: { + function_id?: string; + chunk_debug_ids?: string[]; + }; + scopes?: SymDbScope[]; +}; + +// The symbol table in the format of the upload's file part. The upload adds the fields that are +// specific to each request: `service`, `upload_id`, `batch_num` and `final`. +export type SymbolTablePayload = { + language: typeof SYMBOL_TABLE_LANGUAGE; + version?: string; + chunk_debug_ids?: string[]; + scopes: SymDbScope[]; +}; + +const comparePositions = (a: SourcePosition | undefined, b: SourcePosition | undefined) => { + if (!a || !b) { + // Functions without a position go last. + return Number(!a) - Number(!b); + } + return a.line - b.line || a.column - b.column; +}; + +const compareStrings = (a: string, b: string) => (a < b ? -1 : Number(a > b)); + +// Source order, so enclosing functions come before the functions they contain. +const compareEntries = (a: SymbolTableEntry, b: SymbolTableEntry) => + comparePositions(a.start, b.start) || + compareStrings(a.functionId, b.functionId) || + comparePositions(a.end, b.end); + +const toFunctionScope = ( + entry: SymbolTableEntry, + childrenByParentKey: Map, +): SymDbScope => { + const children = childrenByParentKey.get(entry.key) ?? []; + const scope: SymDbScope = { + scope_type: 'FUNCTION', + name: entry.name, + source_file: entry.sourceFile, + }; + + if (entry.start && entry.end) { + scope.start_line = entry.start.line; + scope.end_line = entry.end.line; + } + + scope.language_specifics = { function_id: entry.functionId }; + + if (children.length > 0) { + scope.scopes = children.map((child) => toFunctionScope(child, childrenByParentKey)); + } + + return scope; +}; + +/** + * Convert the symbol table to SymDB scopes: one `MODULE` scope per file, without a line range, and + * one `FUNCTION` scope per function, nested under its enclosing function, in source order. + */ +export const serializeSymbolTable = ( + table: SymbolTable, + version: string | undefined, +): SymbolTablePayload => { + const sortedEntries = [...table.entries].sort(compareEntries); + const topLevelEntriesByFile = new Map(); + const childrenByParentKey = new Map(); + + for (const entry of sortedEntries) { + const siblings = entry.parentKey + ? childrenByParentKey.get(entry.parentKey) ?? [] + : topLevelEntriesByFile.get(entry.sourceFile) ?? []; + siblings.push(entry); + + if (entry.parentKey) { + childrenByParentKey.set(entry.parentKey, siblings); + } else { + topLevelEntriesByFile.set(entry.sourceFile, siblings); + } + } + + const sourceFiles = [...topLevelEntriesByFile.keys()].sort(compareStrings); + const scopes = sourceFiles.map((sourceFile): SymDbScope => { + const topLevelEntries = topLevelEntriesByFile.get(sourceFile) ?? []; + const chunkDebugIds = table.fileChunkDebugIds?.get(sourceFile); + const moduleScope: SymDbScope = { + scope_type: 'MODULE', + name: sourceFile, + source_file: sourceFile, + }; + + if (chunkDebugIds) { + moduleScope.language_specifics = { chunk_debug_ids: chunkDebugIds }; + } + + moduleScope.scopes = topLevelEntries.map((entry) => + toFunctionScope(entry, childrenByParentKey), + ); + + return moduleScope; + }); + + return { + language: SYMBOL_TABLE_LANGUAGE, + ...(version ? { version } : {}), + ...(table.chunkDebugIds ? { chunk_debug_ids: table.chunkDebugIds } : {}), + scopes, + }; +}; diff --git a/packages/plugins/live-debugger/src/symbol-table/table.test.ts b/packages/plugins/live-debugger/src/symbol-table/table.test.ts new file mode 100644 index 000000000..4c0bda07c --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/table.test.ts @@ -0,0 +1,234 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { serializeSymbolTable } from './serialize'; +import { buildSymbolTable } from './table'; +import type { ChunkFunction, ChunkFunctions } from './types'; + +const DEBUG_ID_A = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'; +const DEBUG_ID_B = 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'; + +// Module paths are relative to the build root, `app` in the repository. +const resolveSourceFile = (modulePath: string) => + modulePath.startsWith('../../') ? undefined : `app/${modulePath.replace(/\?.*$/, '')}`; + +const getFunction = ( + functionId: string, + lines: [number, number] | undefined, + parentIndex?: number, +): ChunkFunction => ({ + functionId, + parentIndex, + start: lines ? { line: lines[0], column: 0 } : undefined, + end: lines ? { line: lines[1], column: 0 } : undefined, +}); + +const getChunk = ( + debugId: string | undefined, + functions: ChunkFunction[], + filepath = 'out.js', +): ChunkFunctions => ({ filepath, debugId, functions }); + +describe('buildSymbolTable', () => { + const cases = [ + { + description: 'list the same function once when several chunks contain it', + chunks: [ + getChunk(DEBUG_ID_A, [getFunction('src/a.js;add', [1, 3])]), + getChunk(DEBUG_ID_B, [getFunction('src/a.js;add', [1, 3])]), + ], + expected: { functionIds: ['src/a.js;add'] }, + }, + { + description: 'keep functions that share an ID at different places', + chunks: [ + getChunk(DEBUG_ID_A, [ + getFunction('src/a.js;render', [1, 3]), + getFunction('src/a.js;render', [5, 7]), + ]), + ], + expected: { functionIds: ['src/a.js;render', 'src/a.js;render'] }, + }, + { + description: 'leave out functions outside the repository and invalid IDs', + chunks: [ + getChunk(DEBUG_ID_A, [ + getFunction('../../shared/a.js;outside', [1, 3]), + getFunction('no-separator', [1, 3]), + getFunction('src/a.js;inside', [1, 3]), + ]), + ], + expected: { + functionIds: ['src/a.js;inside'], + outsideModulePaths: ['../../shared/a.js'], + invalidFunctionIds: ['no-separator'], + }, + }, + ]; + + test.each(cases)('Should $description.', ({ chunks, expected }) => { + const result = buildSymbolTable(chunks, resolveSourceFile); + const functionIds = result.table.entries.map((entry) => entry.functionId); + const outsideModulePaths = [...result.outsideModulePaths]; + const invalidFunctionIds = [...result.invalidFunctionIds]; + + expect(functionIds).toEqual(expected.functionIds); + expect(outsideModulePaths).toEqual(expected.outsideModulePaths ?? []); + expect(invalidFunctionIds).toEqual(expected.invalidFunctionIds ?? []); + }); + + test('Should drop copies without a line range when another copy has one.', () => { + const chunks = [ + getChunk(DEBUG_ID_A, [ + getFunction('src/a.js;outer', undefined), + getFunction('src/a.js;inner', [2, 4], 0), + ]), + getChunk(DEBUG_ID_B, [ + getFunction('src/a.js;outer', [1, 9]), + getFunction('src/a.js;unmapped', undefined), + ]), + ]; + const { table } = buildSymbolTable(chunks, resolveSourceFile); + const entries = table.entries.map(({ functionId, start, parentKey }) => ({ + functionId, + located: Boolean(start), + hasParent: Boolean(parentKey), + })); + + expect(entries).toEqual([ + { functionId: 'src/a.js;inner', located: true, hasParent: false }, + { functionId: 'src/a.js;outer', located: true, hasParent: false }, + { functionId: 'src/a.js;unmapped', located: false, hasParent: false }, + ]); + }); + + test('Should record which chunks contain each file.', () => { + const chunks = [ + getChunk(DEBUG_ID_B, [getFunction('src/a.js;add', [1, 3])]), + getChunk(DEBUG_ID_A, [ + getFunction('src/a.js;add', [1, 3]), + getFunction('src/b.js;sub', [1, 3]), + ]), + getChunk(DEBUG_ID_A, []), + ]; + const { table } = buildSymbolTable(chunks, resolveSourceFile); + const fileChunkDebugIds = Object.fromEntries(table.fileChunkDebugIds ?? []); + + expect(table.chunkDebugIds).toEqual([DEBUG_ID_A, DEBUG_ID_B]); + expect(fileChunkDebugIds).toEqual({ + 'app/src/a.js': [DEBUG_ID_A, DEBUG_ID_B], + 'app/src/b.js': [DEBUG_ID_A], + }); + }); + + test('Should leave out chunk debug IDs when a chunk with functions has none.', () => { + const chunks = [ + getChunk(DEBUG_ID_A, [getFunction('src/a.js;add', [1, 3])]), + getChunk(undefined, [getFunction('src/b.js;sub', [1, 3])]), + ]; + const { table } = buildSymbolTable(chunks, resolveSourceFile); + + expect(table.chunkDebugIds).toBeUndefined(); + expect(table.fileChunkDebugIds).toBeUndefined(); + }); + + test('Should only nest functions under enclosing functions of the same file.', () => { + const chunks = [ + getChunk(DEBUG_ID_A, [ + getFunction('src/a.js;outer', [1, 9]), + // Inlined into `outer` by a minifier. + getFunction('src/b.js;inlined', [1, 3], 0), + getFunction('src/a.js;inner', [2, 4], 1), + ]), + ]; + const { table } = buildSymbolTable(chunks, resolveSourceFile); + const [outer, inlined, inner] = table.entries; + + expect(outer.parentKey).toBeUndefined(); + expect(inlined.parentKey).toBeUndefined(); + expect(inner.parentKey).toBe(outer.key); + }); +}); + +describe('serializeSymbolTable', () => { + test('Should convert the table to SymDB scopes in source order.', () => { + const chunks = [ + getChunk(DEBUG_ID_A, [ + getFunction('src/b.js;later', [10, 12]), + getFunction('src/b.js;outer', [1, 8]), + getFunction('src/b.js;@3:4:0', [3, 5], 1), + getFunction('src/a.js;unmapped', undefined), + ]), + ]; + const { table } = buildSymbolTable(chunks, resolveSourceFile); + + expect(serializeSymbolTable(table, '1.0.0')).toEqual({ + language: 'browser', + version: '1.0.0', + chunk_debug_ids: [DEBUG_ID_A], + scopes: [ + { + scope_type: 'MODULE', + name: 'app/src/a.js', + source_file: 'app/src/a.js', + language_specifics: { chunk_debug_ids: [DEBUG_ID_A] }, + scopes: [ + { + scope_type: 'FUNCTION', + name: 'unmapped', + source_file: 'app/src/a.js', + language_specifics: { function_id: 'src/a.js;unmapped' }, + }, + ], + }, + { + scope_type: 'MODULE', + name: 'app/src/b.js', + source_file: 'app/src/b.js', + language_specifics: { chunk_debug_ids: [DEBUG_ID_A] }, + scopes: [ + { + scope_type: 'FUNCTION', + name: 'outer', + source_file: 'app/src/b.js', + start_line: 1, + end_line: 8, + language_specifics: { function_id: 'src/b.js;outer' }, + scopes: [ + { + scope_type: 'FUNCTION', + name: '', + source_file: 'app/src/b.js', + start_line: 3, + end_line: 5, + language_specifics: { + function_id: 'src/b.js;@3:4:0', + }, + }, + ], + }, + { + scope_type: 'FUNCTION', + name: 'later', + source_file: 'app/src/b.js', + start_line: 10, + end_line: 12, + language_specifics: { function_id: 'src/b.js;later' }, + }, + ], + }, + ], + }); + }); + + test('Should leave out the version and chunk debug IDs when unknown.', () => { + const chunks = [getChunk(undefined, [getFunction('src/a.js;add', [1, 3])])]; + const { table } = buildSymbolTable(chunks, resolveSourceFile); + const payload = serializeSymbolTable(table, undefined); + + expect(payload).not.toHaveProperty('version'); + expect(payload).not.toHaveProperty('chunk_debug_ids'); + expect(payload.scopes[0]).not.toHaveProperty('language_specifics'); + }); +}); diff --git a/packages/plugins/live-debugger/src/symbol-table/table.ts b/packages/plugins/live-debugger/src/symbol-table/table.ts new file mode 100644 index 000000000..9a48e8eda --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/table.ts @@ -0,0 +1,159 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { parseFunctionId } from '../transform/functionId'; + +import type { + ChunkFunction, + ChunkFunctions, + SourcePosition, + SymbolTable, + SymbolTableEntry, +} from './types'; + +export type SymbolTableResult = { + table: SymbolTable; + // Module paths of functions dropped because they're outside the repository. + outsideModulePaths: Set; + // Function IDs that couldn't be parsed. + invalidFunctionIds: Set; +}; + +const formatPosition = (position: SourcePosition | undefined) => + position ? `${position.line}:${position.column}` : ''; + +// The same function can be in several chunks, or in several output formats of the same build. +// Functions that share an ID but are at different places are different functions (v1 IDs aren't +// unique), so the key includes the location. +const getEntryKey = (chunkFunction: ChunkFunction, sourceFile: string) => { + const start = formatPosition(chunkFunction.start); + const end = formatPosition(chunkFunction.end); + return [chunkFunction.functionId, sourceFile, start, end].join('\0'); +}; + +// In some chunks a copy of a function can't be mapped, while other copies can. A copy without a +// line range then adds nothing, so drop it and attach its children to its own parent. +const removeUnlocatedCopies = (entries: SymbolTableEntry[]): SymbolTableEntry[] => { + const getFunctionKey = (entry: SymbolTableEntry) => `${entry.functionId}\0${entry.sourceFile}`; + const locatedEntries = entries.filter((entry) => entry.start); + const locatedFunctionKeys = new Set(locatedEntries.map(getFunctionKey)); + const removedParentKeys = new Map(); + + const keptEntries = entries.filter((entry) => { + const functionKey = getFunctionKey(entry); + if (entry.start || !locatedFunctionKeys.has(functionKey)) { + return true; + } + removedParentKeys.set(entry.key, entry.parentKey); + return false; + }); + + const getKeptParentKey = (parentKey: string | undefined) => { + let keptParentKey = parentKey; + while (keptParentKey !== undefined && removedParentKeys.has(keptParentKey)) { + keptParentKey = removedParentKeys.get(keptParentKey); + } + return keptParentKey; + }; + + return keptEntries.map((entry) => { + const parentKey = getKeptParentKey(entry.parentKey); + return { ...entry, parentKey }; + }); +}; + +// Map the output's chunks to one entry per function, deduplicated across chunks. +export const buildSymbolTable = ( + chunks: ChunkFunctions[], + resolveSourceFile: (modulePath: string) => string | undefined, +): SymbolTableResult => { + const entries = new Map(); + const fileChunkDebugIds = new Map>(); + const chunkDebugIds = new Set(); + const outsideModulePaths = new Set(); + const invalidFunctionIds = new Set(); + let hasCompleteDebugIds = true; + + for (const chunk of chunks) { + if (chunk.debugId) { + chunkDebugIds.add(chunk.debugId); + } else if (chunk.functions.length > 0) { + hasCompleteDebugIds = false; + } + + // Per function of the chunk, by index. + const keys: (string | undefined)[] = []; + const sourceFiles: (string | undefined)[] = []; + + // The closest enclosing function that's in the same file. + const getParentKey = (chunkFunction: ChunkFunction, sourceFile: string) => { + let parentIndex = chunkFunction.parentIndex; + while (parentIndex !== undefined) { + if (sourceFiles[parentIndex] === sourceFile && keys[parentIndex]) { + return keys[parentIndex]; + } + parentIndex = chunk.functions[parentIndex].parentIndex; + } + return undefined; + }; + + for (const chunkFunction of chunk.functions) { + const parsedId = parseFunctionId(chunkFunction.functionId); + if (!parsedId) { + invalidFunctionIds.add(chunkFunction.functionId); + keys.push(undefined); + sourceFiles.push(undefined); + continue; + } + + const sourceFile = resolveSourceFile(parsedId.modulePath); + if (!sourceFile) { + outsideModulePaths.add(parsedId.modulePath); + keys.push(undefined); + sourceFiles.push(undefined); + continue; + } + + const key = getEntryKey(chunkFunction, sourceFile); + const parentKey = getParentKey(chunkFunction, sourceFile); + keys.push(key); + sourceFiles.push(sourceFile); + + if (chunk.debugId) { + const debugIds = fileChunkDebugIds.get(sourceFile) ?? new Set(); + debugIds.add(chunk.debugId); + fileChunkDebugIds.set(sourceFile, debugIds); + } + + if (!entries.has(key)) { + entries.set(key, { + key, + functionId: chunkFunction.functionId, + name: parsedId.name, + sourceFile, + start: chunkFunction.start, + end: chunkFunction.end, + parentKey, + }); + } + } + } + + const sortedChunkDebugIds = [...chunkDebugIds].sort(); + const sortedFileEntries = [...fileChunkDebugIds].map(([sourceFile, debugIds]) => { + const sortedDebugIds = [...debugIds].sort(); + return [sourceFile, sortedDebugIds] as const; + }); + const sortedFileChunkDebugIds = new Map(sortedFileEntries); + + // Never publish incomplete chunk membership. + const allEntries = [...entries.values()]; + const table: SymbolTable = { + entries: removeUnlocatedCopies(allEntries), + chunkDebugIds: hasCompleteDebugIds ? sortedChunkDebugIds : undefined, + fileChunkDebugIds: hasCompleteDebugIds ? sortedFileChunkDebugIds : undefined, + }; + + return { table, outsideModulePaths, invalidFunctionIds }; +}; diff --git a/packages/plugins/live-debugger/src/symbol-table/types.ts b/packages/plugins/live-debugger/src/symbol-table/types.ts new file mode 100644 index 000000000..26776b4ec --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/types.ts @@ -0,0 +1,49 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +// A position in an original source file: 1-based line, 0-based column. +export type SourcePosition = { + line: number; + column: number; +}; + +// An instrumented function found in an emitted chunk. +export type ChunkFunction = { + functionId: string; + // Index of the enclosing instrumented function in the same chunk. + parentIndex: number | undefined; + // Original positions of the function body's opening and closing brace. + // Both are undefined when either one can't be mapped. + start: SourcePosition | undefined; + end: SourcePosition | undefined; +}; + +export type ChunkFunctions = { + filepath: string; + // Undefined when the chunk has no debug ID. + debugId: string | undefined; + functions: ChunkFunction[]; +}; + +export type SymbolTableEntry = { + // Identifies the entry: the same function in several chunks has the same key. + key: string; + functionId: string; + name: string; + // Path relative to the repository root. + sourceFile: string; + start: SourcePosition | undefined; + end: SourcePosition | undefined; + // Key of the enclosing function's entry, when it's in the same file. + parentKey: string | undefined; +}; + +export type SymbolTable = { + entries: SymbolTableEntry[]; + // Debug IDs of the build's chunks. Undefined when a chunk with instrumented functions has none. + chunkDebugIds: string[] | undefined; + // For each source file, the debug IDs of the chunks that contain its functions. + // Undefined in the same case as `chunkDebugIds`. + fileChunkDebugIds: Map | undefined; +}; diff --git a/packages/plugins/live-debugger/src/symbolTable.integration.test.ts b/packages/plugins/live-debugger/src/symbolTable.integration.test.ts new file mode 100644 index 000000000..d1eb48702 --- /dev/null +++ b/packages/plugins/live-debugger/src/symbolTable.integration.test.ts @@ -0,0 +1,407 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { datadogRspackPlugin } from '@datadog/rspack-plugin'; +import { datadogWebpackPlugin } from '@datadog/webpack-plugin'; +import { existsSync, mkdir, outputFile, readFile, readFileSync, rm } from '@dd/core/helpers/fs'; +import type { BundlerName, Options } from '@dd/core/types'; +import { getOutDir, getTempWorkingDir } from '@dd/tests/_jest/helpers/env'; +import { defaultPluginOptions } from '@dd/tests/_jest/helpers/mocks'; +import { BUNDLERS, getBundlerConfig } from '@dd/tests/_jest/helpers/runBundlers'; +import { buildWithRspack, buildWithWebpack, configRspack, configWebpack } from '@dd/tools/bundlers'; +import fsp from 'fs/promises'; +import { glob } from 'glob'; +import path from 'path'; + +import { SYMBOL_TABLE_PATH_ENV } from './constants'; +import type { SymbolTablePayload, SymDbScope } from './symbol-table/serialize'; + +type XpackCompiler = { options: { module: { rules: object[] } } }; + +type TableFunction = { + functionId: string; + name: string; + sourceFile: string; + startLine: number | undefined; + endLine: number | undefined; + parentName: string | undefined; +}; + +// The repository root holds the git marker, the build root is its `app` directory. +const APP_DIR = 'app'; +const VIRTUAL_PEER_PATH = '.yarn/__virtual__/peer-virtual-0123456789/1/packages/peer/index.js'; +const XPACK_BUNDLERS: BundlerName[] = ['webpack', 'rspack']; +const PROBE_CALL_RX = /\$dd_probes\((['"`])(.*?)\1\)/g; + +const PEER_SOURCE = `export function peerHello() { + return 'hello from peer'; +} +`; + +const FIXTURE_FILES: Record = { + 'app/src/main.js': `import { libGreet } from '@fixture/lib'; +import { peerHello } from '../../${VIRTUAL_PEER_PATH}'; +import { add, triple } from './math.js'; + +export const values = [1, 2].map(function (value) { + return add(value, 1); +}); + +console.log(values, libGreet(), peerHello(), triple(2)); +`, + // webpack and rspack also cover a module ID with a query string. + 'app/src/main-xpack.js': `import { render } from './component.js?query'; +import { values } from './main.js'; + +console.log(render(values.join(','))); +`, + 'app/src/math.js': `export function add(a, b) { + const double = (value) => { + return value * 2; + }; + return double(a) + b; +} + +export const triple = (value) => + value * + 3; +`, + 'app/src/component.js': `export function render(label) { + return '

' + label + '

'; +} +`, + 'packages/lib/package.json': JSON.stringify({ + name: '@fixture/lib', + version: '1.0.0', + main: 'index.js', + }), + 'packages/lib/index.js': `export function libGreet() { + return 'hello from lib'; +} +`, + 'packages/peer/index.js': PEER_SOURCE, + // Yarn PnP loads packages with peer dependencies through virtual paths. + [VIRTUAL_PEER_PATH]: PEER_SOURCE, +}; + +// Real webpack and rspack builds run a loader producing a source map before the Datadog plugins, +// which unplugin needs to chain their source maps. +const SOURCE_MAP_LOADER = path.resolve( + __dirname, + '../../../tests/src/_jest/fixtures/circular_project/identity-source-map-loader.cjs', +); +const sourceMapLoaderPlugin = { + apply(compiler: XpackCompiler) { + compiler.options.module.rules.unshift({ test: /\.js$/, loader: SOURCE_MAP_LOADER }); + }, +}; + +const PLUGIN_OVERRIDES: Partial = { + enableGit: false, + liveDebugger: {}, + metadata: { version: '1.0.0' }, + sourcemaps: { debugId: true }, +}; + +const createFixture = async (workingDir: string) => { + const writes = Object.entries(FIXTURE_FILES).map(([file, content]) => { + const filepath = path.resolve(workingDir, file); + return outputFile(filepath, content); + }); + await Promise.all(writes); + + // A workspace package linked into node_modules. + const linkPath = path.resolve(workingDir, 'app/node_modules/@fixture/lib'); + const linkTarget = path.resolve(workingDir, 'packages/lib'); + const linkDir = path.dirname(linkPath); + await mkdir(linkDir); + await fsp.symlink(linkTarget, linkPath, 'dir'); + + // The symbol table only needs the repository root, which the `.git` entry marks. + const gitDir = path.resolve(workingDir, '.git'); + await mkdir(gitDir); +}; + +// Build with the symbol table dump enabled, and return the dump. +const buildWithSymbolTable = async (dumpPath: string, build: () => Promise) => { + process.env[SYMBOL_TABLE_PATH_ENV] = dumpPath; + try { + const errors = await build(); + expect(errors).toEqual([]); + } finally { + delete process.env[SYMBOL_TABLE_PATH_ENV]; + } + + const dump = await readFile(dumpPath); + const table: SymbolTablePayload = JSON.parse(dump); + return table; +}; + +const flattenScopes = ( + scopes: SymDbScope[] = [], + parentName: string | undefined = undefined, +): TableFunction[] => + scopes.flatMap((scope) => { + if (scope.scope_type === 'MODULE') { + return flattenScopes(scope.scopes); + } + + const tableFunction: TableFunction = { + functionId: scope.language_specifics?.function_id ?? '', + name: scope.name, + sourceFile: scope.source_file, + startLine: scope.start_line, + endLine: scope.end_line, + parentName, + }; + const children = flattenScopes(scope.scopes, scope.name); + return [tableFunction, ...children]; + }); + +const getOutputFunctionIds = async (outDir: string) => { + const files = await glob('**/*.{js,mjs}', { cwd: outDir, absolute: true }); + const contents = files.map((file) => readFileSync(file)); + const functionIds = contents.flatMap((content) => { + const matches = content.matchAll(PROBE_CALL_RX); + return [...matches].map(([, , functionId]) => functionId); + }); + return [...new Set(functionIds)].sort(); +}; + +describe('Live Debugger symbol table', () => { + const tables: Partial> = {}; + const outputFunctionIds: Partial> = {}; + const outputContents: Partial> = {}; + let workingDir: string; + let appDir: string; + + beforeAll(async () => { + const seed = `${Math.abs(jest.getSeed())}.symbol-table`; + workingDir = getTempWorkingDir(seed); + appDir = path.resolve(workingDir, APP_DIR); + await createFixture(workingDir); + const buildRoot = appDir; + + // Run the builds sequentially to ease the resources usage. + for (const { name: bundlerName, run } of BUNDLERS) { + const isXpack = XPACK_BUNDLERS.includes(bundlerName); + const entry = { main: isXpack ? './src/main-xpack.js' : './src/main.js' }; + const plugins = isXpack ? [sourceMapLoaderPlugin] : []; + const dumpPath = path.resolve(workingDir, `symbol-table.${bundlerName}.json`); + const outDir = getOutDir(appDir, bundlerName); + + // eslint-disable-next-line no-await-in-loop + tables[bundlerName] = await buildWithSymbolTable(dumpPath, async () => { + const config = getBundlerConfig(bundlerName, buildRoot, PLUGIN_OVERRIDES, { + entry, + plugins, + }); + const { errors } = await run(buildRoot, config); + return errors; + }); + // eslint-disable-next-line no-await-in-loop + outputFunctionIds[bundlerName] = await getOutputFunctionIds(outDir); + // eslint-disable-next-line no-await-in-loop + const outputFiles = await glob('**/*', { cwd: outDir, absolute: true, nodir: true }); + outputContents[bundlerName] = outputFiles.map((file) => readFileSync(file)); + } + }, 200000); + + afterAll(async () => { + if (!process.env.NO_CLEANUP) { + await rm(workingDir); + } + }); + + describe.each(BUNDLERS)('$name | $version', ({ name: bundlerName }) => { + const getTable = () => tables[bundlerName]; + const getFunctions = () => flattenScopes(getTable()?.scopes); + const getFunction = (sourceFile: string, name: string) => + getFunctions().find( + (tableFunction) => + tableFunction.sourceFile === sourceFile && tableFunction.name === name, + ); + + test('Should list exactly the function IDs present in the output.', () => { + const functionIds = getFunctions().map((tableFunction) => tableFunction.functionId); + const uniqueFunctionIds = [...new Set(functionIds)].sort(); + + expect(outputFunctionIds[bundlerName]?.length).toBeGreaterThan(0); + expect(uniqueFunctionIds).toEqual(outputFunctionIds[bundlerName]); + }); + + test('Should describe the build.', () => { + const table = getTable(); + expect(table?.language).toBe('browser'); + expect(table?.version).toBe('1.0.0'); + expect(table?.chunk_debug_ids?.length).toBeGreaterThan(0); + + for (const moduleScope of table?.scopes ?? []) { + expect(moduleScope.scope_type).toBe('MODULE'); + expect(moduleScope.start_line).toBeUndefined(); + expect(moduleScope.language_specifics?.chunk_debug_ids?.length).toBeGreaterThan(0); + } + }); + + test('Should give paths relative to the repository root, with original line ranges.', () => { + expect(getFunction('app/src/math.js', 'add')).toMatchObject({ + startLine: 1, + endLine: 6, + parentName: undefined, + }); + expect(getFunction('app/src/math.js', 'double')).toMatchObject({ + startLine: 2, + endLine: 4, + parentName: 'add', + }); + expect(getFunction('app/src/main.js', '')).toMatchObject({ + startLine: 5, + endLine: 7, + }); + }); + + test('Should give arrow functions with an expression body their line range.', () => { + expect(getFunction('app/src/math.js', 'triple')).toMatchObject({ + startLine: 9, + endLine: 10, + }); + }); + + test('Should resolve symlinked workspace packages to their repository path.', () => { + expect(getFunction('packages/lib/index.js', 'libGreet')).toMatchObject({ + startLine: 1, + endLine: 3, + }); + }); + + test('Should resolve Yarn PnP virtual paths to their repository path.', () => { + expect(getFunction('packages/peer/index.js', 'peerHello')).toMatchObject({ + startLine: 1, + endLine: 3, + }); + + const sourceFiles = getFunctions().map((tableFunction) => tableFunction.sourceFile); + const virtualSourceFiles = sourceFiles.filter((sourceFile) => + sourceFile.includes('__virtual__'), + ); + expect(virtualSourceFiles).toEqual([]); + }); + + test('Should not write the symbol table to the output directory.', () => { + const contents = outputContents[bundlerName] ?? []; + expect(contents.length).toBeGreaterThan(0); + + const tableContents = contents.filter((content) => content.includes('"scope_type"')); + expect(tableContents).toEqual([]); + }); + + if (XPACK_BUNDLERS.includes(bundlerName)) { + test('Should strip the query string of module IDs.', () => { + expect(getFunction('app/src/component.js', 'render')).toMatchObject({ + startLine: 1, + endLine: 3, + }); + }); + } + }); + + describe('webpack | warm filesystem cache', () => { + if (!BUNDLERS.some(({ name }) => name === 'webpack')) { + test.skip('webpack is not part of this run.', () => {}); + return; + } + + let coldTable: SymbolTablePayload | undefined; + let warmTable: SymbolTablePayload | undefined; + let cacheDirectory: string; + + beforeAll(async () => { + cacheDirectory = path.resolve(workingDir, 'webpack-cache'); + const outDir = getOutDir(appDir, 'webpack-cache'); + + const build = async () => { + const plugin = datadogWebpackPlugin({ + ...defaultPluginOptions, + ...PLUGIN_OVERRIDES, + }); + const config = configWebpack({ + workingDir: appDir, + outDir, + entry: { main: './src/main-xpack.js' }, + plugins: [sourceMapLoaderPlugin, plugin], + }); + config.cache = { type: 'filesystem', cacheDirectory }; + const { errors } = await buildWithWebpack(config); + return errors; + }; + + const coldDumpPath = path.resolve(workingDir, 'symbol-table.webpack-cold.json'); + coldTable = await buildWithSymbolTable(coldDumpPath, build); + const warmDumpPath = path.resolve(workingDir, 'symbol-table.webpack-warm.json'); + warmTable = await buildWithSymbolTable(warmDumpPath, build); + }, 200000); + + test('Should list the same functions when modules come from the cache.', () => { + expect(existsSync(cacheDirectory)).toBe(true); + expect(flattenScopes(coldTable?.scopes).length).toBeGreaterThan(0); + expect(warmTable).toEqual(coldTable); + }); + }); + // Minifiers rewrite the injected code and don't map every token, like web-ui's production builds. + describe('rspack | minified production build', () => { + if (!BUNDLERS.some(({ name }) => name === 'rspack')) { + test.skip('rspack is not part of this run.', () => {}); + return; + } + + let minifiedTable: SymbolTablePayload | undefined; + + beforeAll(async () => { + const outDir = getOutDir(appDir, 'rspack-production'); + const dumpPath = path.resolve(workingDir, 'symbol-table.rspack-production.json'); + + minifiedTable = await buildWithSymbolTable(dumpPath, async () => { + const plugin = datadogRspackPlugin({ + ...defaultPluginOptions, + ...PLUGIN_OVERRIDES, + rum: { privacy: {} }, + }); + const config = configRspack({ + workingDir: appDir, + outDir, + entry: { main: './src/main-xpack.js' }, + plugins: [sourceMapLoaderPlugin, plugin], + }); + const { errors } = await buildWithRspack({ + ...config, + mode: 'production', + optimization: { ...config.optimization, minimize: true }, + }); + return errors; + }); + }, 200000); + + test('Should give every function its original line range.', () => { + const functions = flattenScopes(minifiedTable?.scopes); + const withoutLines = functions.filter((tableFunction) => !tableFunction.endLine); + + expect(functions.length).toBeGreaterThan(0); + expect(withoutLines).toEqual([]); + expect(functions).toContainEqual( + expect.objectContaining({ + sourceFile: 'app/src/math.js', + name: 'add', + startLine: 1, + endLine: 6, + }), + ); + + // The end of an expression body is the last token the minifier keeps mapped. + const triple = functions.find((tableFunction) => tableFunction.name === 'triple'); + expect(triple?.startLine).toBe(9); + expect(triple?.endLine).toBeGreaterThanOrEqual(9); + expect(triple?.endLine).toBeLessThanOrEqual(10); + }); + }); +}); diff --git a/packages/plugins/live-debugger/src/transform/functionId.test.ts b/packages/plugins/live-debugger/src/transform/functionId.test.ts index e48ac2c7f..84ddce0d9 100644 --- a/packages/plugins/live-debugger/src/transform/functionId.test.ts +++ b/packages/plugins/live-debugger/src/transform/functionId.test.ts @@ -7,7 +7,7 @@ import type * as t from '@babel/types'; import type { BabelPath } from './babel-path.types'; import { resolveCjsDefaultExport } from './cjs-interop'; -import { generateFunctionId, getFunctionName } from './functionId'; +import { generateFunctionId, getFunctionName, parseFunctionId } from './functionId'; // eslint-disable-next-line @typescript-eslint/no-var-requires, global-require const babelTypes = require('@babel/types') as typeof import('@babel/types'); @@ -312,3 +312,37 @@ describe('generateFunctionId', () => { }); }); }); + +describe('parseFunctionId', () => { + const cases = [ + { + description: 'split a named function ID', + input: 'src/utils.js;add', + expected: { modulePath: 'src/utils.js', name: 'add' }, + }, + { + description: 'keep separators that are part of the function name', + input: 'src/utils.js;get;user', + expected: { modulePath: 'src/utils.js', name: 'get;user' }, + }, + { + description: 'drop the location of anonymous functions', + input: 'src/widget.js;@4:16:0', + expected: { modulePath: 'src/widget.js', name: '' }, + }, + { + description: 'keep the query string of the module path', + input: 'src/Cart.vue?vue&type=script;total', + expected: { modulePath: 'src/Cart.vue?vue&type=script', name: 'total' }, + }, + { + description: 'reject an ID without a module path', + input: ';add', + expected: undefined, + }, + ]; + + test.each(cases)('Should $description.', ({ input, expected }) => { + expect(parseFunctionId(input)).toEqual(expected); + }); +}); diff --git a/packages/plugins/live-debugger/src/transform/functionId.ts b/packages/plugins/live-debugger/src/transform/functionId.ts index 16f2def97..f5aaa7a9f 100644 --- a/packages/plugins/live-debugger/src/transform/functionId.ts +++ b/packages/plugins/live-debugger/src/transform/functionId.ts @@ -5,8 +5,34 @@ import type * as t from '@babel/types'; import path from 'path'; +import { ANONYMOUS_FUNCTION_NAME, FUNCTION_ID_SEPARATOR } from '../constants'; + import type { BabelPath, BabelTypesModule } from './babel-path.types'; +export type ParsedFunctionId = { + // The module ID relative to the build root, including any query string. + modulePath: string; + // The function name, without the location suffix of anonymous functions. + name: string; +}; + +/** + * Split a function ID generated by `generateFunctionId` into its module path and function name. + * Names can contain the separator (e.g. string keys), so the ID is split at its first occurrence. + */ +export function parseFunctionId(functionId: string): ParsedFunctionId | undefined { + const separatorIndex = functionId.indexOf(FUNCTION_ID_SEPARATOR); + if (separatorIndex <= 0) { + return undefined; + } + + const modulePath = functionId.slice(0, separatorIndex); + const functionName = functionId.slice(separatorIndex + FUNCTION_ID_SEPARATOR.length); + const isAnonymous = functionName.startsWith(`${ANONYMOUS_FUNCTION_NAME}@`); + + return { modulePath, name: isAnonymous ? ANONYMOUS_FUNCTION_NAME : functionName }; +} + /** * Generate a stable, unique function ID * Format (POC): ; @@ -26,12 +52,12 @@ export function generateFunctionId( const functionName = getFunctionName(functionPath, typesModule); if (functionName) { - return `${relativePath};${functionName}`; + return `${relativePath}${FUNCTION_ID_SEPARATOR}${functionName}`; } const locationSuffix = getFunctionLocation(functionPath); - return `${relativePath};@${locationSuffix}:${anonymousSiblingIndex}`; + return `${relativePath}${FUNCTION_ID_SEPARATOR}${ANONYMOUS_FUNCTION_NAME}@${locationSuffix}:${anonymousSiblingIndex}`; } /** diff --git a/packages/plugins/live-debugger/src/transform/index.ts b/packages/plugins/live-debugger/src/transform/index.ts index f39b634a7..139a14072 100644 --- a/packages/plugins/live-debugger/src/transform/index.ts +++ b/packages/plugins/live-debugger/src/transform/index.ts @@ -9,7 +9,7 @@ import { LIVE_DEBUGGER_PROBES_FUNCTION } from '@dd/core/constants'; import type MagicStringType from 'magic-string'; import type { SourceMap } from 'magic-string'; -import { SKIP_INSTRUMENTATION_COMMENT } from '../constants'; +import { LIVE_DEBUGGER_THROW_FUNCTION, SKIP_INSTRUMENTATION_COMMENT } from '../constants'; import type { DecoratorSyntax, FunctionKind } from '../types'; import type { BabelPath, BabelTypesModule } from './babel-path.types'; @@ -511,7 +511,7 @@ function injectInstrumentation(s: MagicStringType, code: string, target: Functio // collect, and keeps the bare call whose function ID RUM privacy leaves inline. const probeDecl = `let ${probeVarName} = globalThis.${LIVE_DEBUGGER_PROBES_FUNCTION} !== void 0 && ${LIVE_DEBUGGER_PROBES_FUNCTION}('${escapedFunctionId}');`; const entryCall = `if (${probeVarName}) ${probeVarName} = $dd_entry(${probeVarName}, ${receiverArg}${argsArg});`; - const catchBlock = `catch($dd_e) { if (${probeVarName}) $dd_throw(${probeVarName}, $dd_e, ${receiverArg}${argsArg}); throw $dd_e; }`; + const catchBlock = `catch($dd_e) { if (${probeVarName}) ${LIVE_DEBUGGER_THROW_FUNCTION}(${probeVarName}, $dd_e, ${receiverArg}${argsArg}); throw $dd_e; }`; if (isExpressionBody) { // Arrow expression body: (a) => expr diff --git a/packages/published/esbuild-plugin/package.json b/packages/published/esbuild-plugin/package.json index ce4212a88..fe9b98e85 100644 --- a/packages/published/esbuild-plugin/package.json +++ b/packages/published/esbuild-plugin/package.json @@ -52,6 +52,7 @@ "dependencies": { "@datadog/js-instrumentation-wasm": "1.1.0", "@jridgewell/remapping": "2.3.5", + "@jridgewell/trace-mapping": "0.3.31", "async-retry": "1.3.3", "chalk": "2.3.1", "esbuild": "0.25.8", diff --git a/packages/published/rollup-plugin/package.json b/packages/published/rollup-plugin/package.json index a43d5304b..7ad860f4e 100644 --- a/packages/published/rollup-plugin/package.json +++ b/packages/published/rollup-plugin/package.json @@ -55,6 +55,7 @@ "dependencies": { "@datadog/js-instrumentation-wasm": "1.1.0", "@jridgewell/remapping": "2.3.5", + "@jridgewell/trace-mapping": "0.3.31", "async-retry": "1.3.3", "chalk": "2.3.1", "esbuild": "0.25.8", diff --git a/packages/published/rspack-plugin/package.json b/packages/published/rspack-plugin/package.json index 69b4f5440..b26a2d26e 100644 --- a/packages/published/rspack-plugin/package.json +++ b/packages/published/rspack-plugin/package.json @@ -52,6 +52,7 @@ "dependencies": { "@datadog/js-instrumentation-wasm": "1.1.0", "@jridgewell/remapping": "2.3.5", + "@jridgewell/trace-mapping": "0.3.31", "async-retry": "1.3.3", "chalk": "2.3.1", "esbuild": "0.25.8", diff --git a/packages/published/vite-plugin/package.json b/packages/published/vite-plugin/package.json index 296e013fd..c51cf8587 100644 --- a/packages/published/vite-plugin/package.json +++ b/packages/published/vite-plugin/package.json @@ -52,6 +52,7 @@ "dependencies": { "@datadog/js-instrumentation-wasm": "1.1.0", "@jridgewell/remapping": "2.3.5", + "@jridgewell/trace-mapping": "0.3.31", "async-retry": "1.3.3", "chalk": "2.3.1", "esbuild": "0.25.8", diff --git a/packages/published/webpack-plugin/package.json b/packages/published/webpack-plugin/package.json index 426b4cf1d..19ce3efbf 100644 --- a/packages/published/webpack-plugin/package.json +++ b/packages/published/webpack-plugin/package.json @@ -52,6 +52,7 @@ "dependencies": { "@datadog/js-instrumentation-wasm": "1.1.0", "@jridgewell/remapping": "2.3.5", + "@jridgewell/trace-mapping": "0.3.31", "async-retry": "1.3.3", "chalk": "2.3.1", "esbuild": "0.25.8", diff --git a/yarn.lock b/yarn.lock index efb0d6b2b..c340821b0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1690,6 +1690,7 @@ __metadata: "@dd/factory": "workspace:*" "@dd/tools": "workspace:*" "@jridgewell/remapping": "npm:2.3.5" + "@jridgewell/trace-mapping": "npm:0.3.31" "@rollup/plugin-babel": "npm:6.0.4" "@rollup/plugin-commonjs": "npm:28.0.1" "@rollup/plugin-esm-shim": "npm:0.1.8" @@ -1750,6 +1751,7 @@ __metadata: "@dd/factory": "workspace:*" "@dd/tools": "workspace:*" "@jridgewell/remapping": "npm:2.3.5" + "@jridgewell/trace-mapping": "npm:0.3.31" "@rollup/plugin-babel": "npm:6.0.4" "@rollup/plugin-commonjs": "npm:28.0.1" "@rollup/plugin-esm-shim": "npm:0.1.8" @@ -1803,6 +1805,7 @@ __metadata: "@dd/factory": "workspace:*" "@dd/tools": "workspace:*" "@jridgewell/remapping": "npm:2.3.5" + "@jridgewell/trace-mapping": "npm:0.3.31" "@rollup/plugin-babel": "npm:6.0.4" "@rollup/plugin-commonjs": "npm:28.0.1" "@rollup/plugin-esm-shim": "npm:0.1.8" @@ -1859,6 +1862,7 @@ __metadata: "@dd/factory": "workspace:*" "@dd/tools": "workspace:*" "@jridgewell/remapping": "npm:2.3.5" + "@jridgewell/trace-mapping": "npm:0.3.31" "@rollup/plugin-babel": "npm:6.0.4" "@rollup/plugin-commonjs": "npm:28.0.1" "@rollup/plugin-esm-shim": "npm:0.1.8" @@ -1912,6 +1916,7 @@ __metadata: "@dd/factory": "workspace:*" "@dd/tools": "workspace:*" "@jridgewell/remapping": "npm:2.3.5" + "@jridgewell/trace-mapping": "npm:0.3.31" "@rollup/plugin-babel": "npm:6.0.4" "@rollup/plugin-commonjs": "npm:28.0.1" "@rollup/plugin-esm-shim": "npm:0.1.8"