From 6eae7ad7f5365af03331c086324de8b61b1cec44 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 15 Oct 2026 14:03:37 -0400 Subject: [PATCH 1/5] Generate a Live Debugger symbol table from the build output Live Debugger needs to know which instrumented functions a build ships and where they are in the repository, so the UI and MCP can copy a function's ID from SymDB instead of computing it. This adds the generation step (DEBUG-6327); the upload comes in DEBUG-6328. After a one-off build, the plugin scans each emitted chunk for the `$dd_probes('')` call at the start of every instrumented function and the `$dd_throw(` call at its end, pairs them like brackets, and maps both through the chunk's final source map to get the original line range. It doesn't parse the output or rely on the transform hook, so it works with persistent bundler caches and only lists functions that survive tree-shaking. File paths and display names come from the function ID. Query strings are stripped, Yarn PnP virtual paths and symlinks are resolved, and paths are made relative to the git root. The table also records the build's chunk debug IDs and which chunks contain each file, when every instrumented chunk has one. The table is serialized as SymDB scopes (a MODULE per file and nested FUNCTION scopes, language `browser`). For now its only consumer is an opt-in debug dump enabled by `DD_LIVE_DEBUGGER_SYMBOL_TABLE_PATH`; without it, the plugin adds no hooks and no build time. Generation never fails the build and never writes inside the output directory. Also make core's `outputJson` resolve once the file is written instead of when the data has been read, which the new tests exposed as a race. Environment: Datadog workspace Co-Authored-By: Claude Sonnet 5.5 --- packages/core/src/helpers/fs.ts | 16 +- .../plugins/live-debugger/CONTRIBUTING.md | 28 ++ packages/plugins/live-debugger/package.json | 2 +- .../plugins/live-debugger/src/constants.ts | 13 + packages/plugins/live-debugger/src/index.ts | 2 + .../src/symbol-table/build-mode.ts | 43 +++ .../src/symbol-table/index.test.ts | 61 ++++ .../live-debugger/src/symbol-table/index.ts | 190 ++++++++++ .../live-debugger/src/symbol-table/locate.ts | 122 +++++++ .../src/symbol-table/paths.test.ts | 124 +++++++ .../live-debugger/src/symbol-table/paths.ts | 85 +++++ .../src/symbol-table/scan.test.ts | 117 ++++++ .../live-debugger/src/symbol-table/scan.ts | 244 +++++++++++++ .../src/symbol-table/serialize.ts | 126 +++++++ .../src/symbol-table/table.test.ts | 209 +++++++++++ .../live-debugger/src/symbol-table/table.ts | 127 +++++++ .../live-debugger/src/symbol-table/types.ts | 49 +++ .../src/symbolTable.integration.test.ts | 339 ++++++++++++++++++ .../src/transform/functionId.test.ts | 36 +- .../live-debugger/src/transform/functionId.ts | 30 +- .../live-debugger/src/transform/index.ts | 4 +- .../published/esbuild-plugin/package.json | 1 + packages/published/rollup-plugin/package.json | 1 + packages/published/rspack-plugin/package.json | 1 + packages/published/vite-plugin/package.json | 1 + .../published/webpack-plugin/package.json | 1 + yarn.lock | 5 + 27 files changed, 1970 insertions(+), 7 deletions(-) create mode 100644 packages/plugins/live-debugger/src/symbol-table/build-mode.ts create mode 100644 packages/plugins/live-debugger/src/symbol-table/index.test.ts create mode 100644 packages/plugins/live-debugger/src/symbol-table/index.ts create mode 100644 packages/plugins/live-debugger/src/symbol-table/locate.ts create mode 100644 packages/plugins/live-debugger/src/symbol-table/paths.test.ts create mode 100644 packages/plugins/live-debugger/src/symbol-table/paths.ts create mode 100644 packages/plugins/live-debugger/src/symbol-table/scan.test.ts create mode 100644 packages/plugins/live-debugger/src/symbol-table/scan.ts create mode 100644 packages/plugins/live-debugger/src/symbol-table/serialize.ts create mode 100644 packages/plugins/live-debugger/src/symbol-table/table.test.ts create mode 100644 packages/plugins/live-debugger/src/symbol-table/table.ts create mode 100644 packages/plugins/live-debugger/src/symbol-table/types.ts create mode 100644 packages/plugins/live-debugger/src/symbolTable.integration.test.ts diff --git a/packages/core/src/helpers/fs.ts b/packages/core/src/helpers/fs.ts index e9f9c12dd..04b0728a3 100644 --- a/packages/core/src/helpers/fs.ts +++ b/packages/core/src/helpers/fs.ts @@ -47,10 +47,15 @@ export const outputJson = async (filepath: string, data: any) => { const writable = fs.createWriteStream(filepath); const readable = new JsonStreamStringify(data, undefined, 2); const prom = new Promise((resolve, reject) => { - readable.on('end', () => { + // Resolve once the data is on disk, not when it's been read. + writable.on('finish', () => { resolve(); }); + writable.on('error', (err) => { + reject(err); + }); + readable.on('error', (err) => { reject(err); }); @@ -59,6 +64,15 @@ export const outputJson = async (filepath: string, data: any) => { return prom; }; +// Return the canonical path, with symlinks resolved, or the input path if it doesn't exist. +export const realpathSync = (filepath: string) => { + try { + return fs.realpathSync(filepath); + } catch { + return filepath; + } +}; + export const outputJsonSync = (filepath: string, data: any) => { // FIXME: This will crash on strings too long. const dataString = JSON.stringify(data, null, 4); diff --git a/packages/plugins/live-debugger/CONTRIBUTING.md b/packages/plugins/live-debugger/CONTRIBUTING.md index 6b00adb5c..e962daf52 100644 --- a/packages/plugins/live-debugger/CONTRIBUTING.md +++ b/packages/plugins/live-debugger/CONTRIBUTING.md @@ -6,6 +6,7 @@ Developer notes for the Live Debugger plugin. - [Development workflow](#development-workflow) +- [Symbol table](#symbol-table) - [Consumer build canary](#consumer-build-canary) - [Runtime benchmark](#runtime-benchmark) - [Running it](#running-it) @@ -41,6 +42,33 @@ Generated code should keep the dormant runtime path small: call `$dd_probes(func When adding or changing `liveDebugger` configuration, update [`src/types.ts`](./src/types.ts), [`src/validate.ts`](./src/validate.ts), [`src/validate.test.ts`](./src/validate.test.ts), and the consumer-facing [`README.md`](./README.md). +## Symbol table + +The plugin can list the instrumented functions that ship in a build, for upload to Live Debugger's symbol database (SymDB). Each entry has the function ID, a display name, the path relative to the git repository root, and the original line range. The table lives in [`src/symbol-table`](./src/symbol-table). + +For now, the only consumer is an opt-in debug dump. Set `DD_LIVE_DEBUGGER_SYMBOL_TABLE_PATH` to a file path to generate the table and write it there: + +```bash +DD_LIVE_DEBUGGER_SYMBOL_TABLE_PATH=/tmp/symbol-table.json yarn build +``` + +Without that variable, the plugin doesn't add any hook or build time. The dump is never written inside the bundler's output directory, and generation never fails the build: problems are logged as warnings. + +How it works: + +- After a one-off build (not in watch mode or on a dev server), the plugin scans the text of every emitted JavaScript chunk, without parsing it. It doesn't use the transform hook, which doesn't run for modules restored from a persistent cache and doesn't know which functions tree-shaking removes. +- Every instrumented function has exactly one `$dd_probes('')` call at the start of its body and one `$dd_throw(` call at its end. The scan pairs them like brackets, which also gives each function's parent. +- The transform anchors both calls' source map segments to the original body's opening and closing brace, so looking them up in the chunk's final source map gives the line range. +- The path and display name come from the function ID. Query strings are removed, Yarn PnP virtual paths and symlinks are resolved, and the path is made relative to the git root. Without a git repository, the table is skipped. +- Each chunk's debug ID is read from the runtime bootstrap, when debug IDs are enabled, to record which chunks contain each file. + +When changing instrumentation output, keep exactly one `$dd_probes` call at the start and one `$dd_throw` call at the end of every instrumented function, anchored to the body's braces. [`src/symbolTable.integration.test.ts`](./src/symbolTable.integration.test.ts) checks the table on every bundler. + +Known limitations: + +- esbuild doesn't tell plugins whether it's watching, so its rebuilds always regenerate the table. +- On webpack and Rspack, files that reach the plugin without a source map from an earlier loader get shifted line numbers ([DEBUG-6343](https://datadoghq.atlassian.net/browse/DEBUG-6343)). + ## Consumer build canary Canary targets build real consumer projects with local Datadog build plugins. diff --git a/packages/plugins/live-debugger/package.json b/packages/plugins/live-debugger/package.json index b4dfb40da..542edd4b4 100644 --- a/packages/plugins/live-debugger/package.json +++ b/packages/plugins/live-debugger/package.json @@ -25,6 +25,7 @@ "dependencies": { "@dd/core": "workspace:*", "@jridgewell/remapping": "2.3.5", + "@jridgewell/trace-mapping": "0.3.31", "chalk": "2.3.1", "picomatch": "4.0.7" }, @@ -32,7 +33,6 @@ "@babel/parser": "7.24.5", "@babel/traverse": "7.24.5", "@babel/types": "7.24.5", - "@jridgewell/trace-mapping": "0.3.31", "@types/picomatch": "4.0.2", "magic-string": "0.30.21", "typescript": "5.4.3" diff --git a/packages/plugins/live-debugger/src/constants.ts b/packages/plugins/live-debugger/src/constants.ts index 0c12cbbb3..2544e2adc 100644 --- a/packages/plugins/live-debugger/src/constants.ts +++ b/packages/plugins/live-debugger/src/constants.ts @@ -10,6 +10,19 @@ export const PLUGIN_NAME: PluginName = 'datadog-live-debugger-plugin' as const; // Skip instrumentation comment export const SKIP_INSTRUMENTATION_COMMENT = '@dd-no-instrumentation'; +// Runtime global that instrumented functions call when they throw. +// The symbol table relies on it marking the end of every instrumented function's body. +export const LIVE_DEBUGGER_THROW_FUNCTION = '$dd_throw'; + +// Function IDs are `;`. +export const FUNCTION_ID_SEPARATOR = ';'; +// Function name used in the IDs of anonymous functions, followed by their location. +export const ANONYMOUS_FUNCTION_NAME = ''; + +// Path where an opt-in debug dump of the symbol table is written. +// Until the symbol table upload exists, setting it is also what enables the symbol table. +export const SYMBOL_TABLE_PATH_ENV = 'DD_LIVE_DEBUGGER_SYMBOL_TABLE_PATH'; + export const DEFAULT_FILE_EXTENSIONS = [ '.js', '.jsx', diff --git a/packages/plugins/live-debugger/src/index.ts b/packages/plugins/live-debugger/src/index.ts index d297b5df6..71a4776bb 100644 --- a/packages/plugins/live-debugger/src/index.ts +++ b/packages/plugins/live-debugger/src/index.ts @@ -12,6 +12,7 @@ import type { SourceMapCompact, UnpluginBuildContext } from 'unplugin'; import { CONFIG_KEY, PLUGIN_NAME } from './constants'; import { createFileExtensionPattern, createFileFilter } from './filter'; import { getRuntimeBootstrap } from './runtime-bootstrap'; +import { getSymbolTableHooks } from './symbol-table'; import { transformCode } from './transform'; import type { LiveDebuggerOptions, LiveDebuggerOptionsWithDefaults } from './types'; import { validateOptions } from './validate'; @@ -117,6 +118,7 @@ export const getLiveDebuggerPlugin = ( } }, }, + ...getSymbolTableHooks(pluginOptions, context, log), buildEnd: () => { if (totalFunctions > 0) { log.info( diff --git a/packages/plugins/live-debugger/src/symbol-table/build-mode.ts b/packages/plugins/live-debugger/src/symbol-table/build-mode.ts new file mode 100644 index 000000000..97dc53a1d --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/build-mode.ts @@ -0,0 +1,43 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import type { PluginOptions } from '@dd/core/types'; + +export type BuildModeTracker = { + hooks: Pick; + // False in watch mode and on dev servers. esbuild doesn't tell plugins whether it's watching, + // so its builds always count as one-off builds. + isOneOffBuild: () => boolean; +}; + +// Track whether the build is a one-off build, as opposed to watch mode or a dev server. +export const createBuildModeTracker = (): BuildModeTracker => { + let isContinuousBuild = false; + let isCompilerWatching = () => false; + + return { + hooks: { + webpack(compiler) { + isCompilerWatching = () => compiler.watchMode; + }, + rspack(compiler) { + isCompilerWatching = () => compiler.watchMode; + }, + rollup: { + buildStart() { + isContinuousBuild ||= this.meta.watchMode; + }, + }, + vite: { + configResolved(config) { + isContinuousBuild ||= config.command === 'serve'; + }, + buildStart() { + isContinuousBuild ||= this.meta.watchMode; + }, + }, + }, + isOneOffBuild: () => !isContinuousBuild && !isCompilerWatching(), + }; +}; diff --git a/packages/plugins/live-debugger/src/symbol-table/index.test.ts b/packages/plugins/live-debugger/src/symbol-table/index.test.ts new file mode 100644 index 000000000..abb914c00 --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/index.test.ts @@ -0,0 +1,61 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { existsSync, rm } from '@dd/core/helpers/fs'; +import { getTempWorkingDir } from '@dd/tests/_jest/helpers/env'; +import { getMockLogger, mockLogFn } from '@dd/tests/_jest/helpers/mocks'; +import path from 'path'; + +import { generateSymbolTable, writeSymbolTable } from './index'; + +const PAYLOAD = { language: 'browser' as const, scopes: [] }; + +describe('Live Debugger symbol table', () => { + let workingDir: string; + + beforeEach(() => { + const seed = `${Math.abs(jest.getSeed())}.symbol-table-unit.${Date.now()}`; + workingDir = getTempWorkingDir(seed); + }); + + afterEach(async () => { + await rm(workingDir); + }); + + test('Should not write the dump inside the output directory.', async () => { + const outDir = path.resolve(workingDir, 'dist'); + const dumpPath = path.resolve(outDir, 'symbol-table.json'); + + await writeSymbolTable(PAYLOAD, dumpPath, outDir, getMockLogger()); + + expect(existsSync(dumpPath)).toBe(false); + expect(mockLogFn).toHaveBeenCalledWith( + expect.stringContaining('inside the output directory'), + 'warn', + ); + }); + + test('Should write the dump outside the output directory.', async () => { + const outDir = path.resolve(workingDir, 'dist'); + const dumpPath = path.resolve(workingDir, 'symbol-table.json'); + + await writeSymbolTable(PAYLOAD, dumpPath, outDir, getMockLogger()); + + expect(existsSync(dumpPath)).toBe(true); + }); + + test('Should skip the table outside a git repository.', async () => { + const table = await generateSymbolTable({ + outputs: [], + buildRoot: workingDir, + log: getMockLogger(), + }); + + expect(table).toBeUndefined(); + expect(mockLogFn).toHaveBeenCalledWith( + expect.stringContaining("isn't in a git repository"), + 'warn', + ); + }); +}); diff --git a/packages/plugins/live-debugger/src/symbol-table/index.ts b/packages/plugins/live-debugger/src/symbol-table/index.ts new file mode 100644 index 000000000..6ce98358f --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/index.ts @@ -0,0 +1,190 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { outputJson, readFile } from '@dd/core/helpers/fs'; +import type { BuildReport, GlobalContext, Logger, Output, PluginOptions } from '@dd/core/types'; +import path from 'path'; + +import { SYMBOL_TABLE_PATH_ENV } from '../constants'; +import type { LiveDebuggerOptionsWithDefaults } from '../types'; + +import { createBuildModeTracker } from './build-mode'; +import { loadSourceMap, locateFunctions } from './locate'; +import { createSourceFileResolver, getRepositoryRoot, isInsideDirectory } from './paths'; +import { readChunkDebugId, scanChunk } from './scan'; +import type { SymbolTablePayload } from './serialize'; +import { serializeSymbolTable } from './serialize'; +import { buildSymbolTable } from './table'; +import type { ChunkFunction, ChunkFunctions, SymbolTable } from './types'; + +const LOG_PREFIX = 'Live Debugger symbol table'; +const JS_OUTPUT_TYPES = new Set(['js', 'mjs', 'cjs']); + +type GenerateSymbolTableArgs = { + outputs: Output[]; + buildRoot: string; + log: Logger; +}; + +const getFirst = (values: Set) => values.values().next().value; + +const byFilepath = (a: Output, b: Output) => { + if (a.filepath === b.filepath) { + return 0; + } + return a.filepath < b.filepath ? -1 : 1; +}; + +/** + * List the instrumented functions that ship in the build's output, with their path relative to + * the repository root and their original line range. + */ +export const generateSymbolTable = async ({ + outputs, + buildRoot, + log, +}: GenerateSymbolTableArgs): Promise => { + const repositoryRoot = getRepositoryRoot(buildRoot); + if (!repositoryRoot) { + log.warn(`Skipping the ${LOG_PREFIX}: the build root isn't in a git repository.`); + return undefined; + } + + const chunkOutputs = outputs.filter((output) => JS_OUTPUT_TYPES.has(output.type)); + chunkOutputs.sort(byFilepath); + + const chunks: ChunkFunctions[] = []; + let chunksWithoutSourceMap = 0; + let unpairedChunks = 0; + let chunksWithoutDebugId = 0; + + // One chunk at a time, so at most one source map is in memory. + for (const output of chunkOutputs) { + // eslint-disable-next-line no-await-in-loop + const code = await readFile(output.filepath); + const scan = scanChunk(code); + const debugId = readChunkDebugId(code); + let functions: ChunkFunction[] = []; + + if (scan.functions.length > 0) { + // eslint-disable-next-line no-await-in-loop + const traceMap = await loadSourceMap(output.filepath, code); + functions = locateFunctions(code, scan, traceMap); + chunksWithoutSourceMap += Number(!traceMap); + unpairedChunks += Number(!scan.paired); + chunksWithoutDebugId += Number(!debugId); + } + + chunks.push({ filepath: output.filepath, debugId, functions }); + } + + const resolveSourceFile = createSourceFileResolver(buildRoot, repositoryRoot); + const { table, outsideModulePaths, invalidFunctionIds } = buildSymbolTable( + chunks, + resolveSourceFile, + ); + + if (chunksWithoutSourceMap > 0) { + log.warn( + `${LOG_PREFIX}: ${chunksWithoutSourceMap} chunk(s) have no source map, so their functions have no line numbers.`, + ); + } + if (unpairedChunks > 0) { + log.warn( + `${LOG_PREFIX}: the instrumentation markers of ${unpairedChunks} chunk(s) don't pair up, so their functions have no line numbers.`, + ); + } + if (chunksWithoutDebugId > 0) { + log.warn( + `${LOG_PREFIX}: ${chunksWithoutDebugId} chunk(s) have no debug ID, so chunk debug IDs are left out. Enable "rum.sourceCodeContext.debugId" to include them.`, + ); + } + if (outsideModulePaths.size > 0) { + const exampleModulePath = getFirst(outsideModulePaths); + log.warn( + `${LOG_PREFIX}: left out the functions of ${outsideModulePaths.size} file(s) outside the git repository, such as "${exampleModulePath}".`, + ); + } + if (invalidFunctionIds.size > 0) { + const exampleFunctionId = getFirst(invalidFunctionIds); + log.warn( + `${LOG_PREFIX}: left out ${invalidFunctionIds.size} function(s) with an unexpected ID, such as "${exampleFunctionId}".`, + ); + } + + return table; +}; + +// Write the opt-in debug dump. It must never end up in the deployed output. +export const writeSymbolTable = async ( + payload: SymbolTablePayload, + dumpPath: string, + outDir: string, + log: Logger, +) => { + const absoluteDumpPath = path.resolve(dumpPath); + if (isInsideDirectory(absoluteDumpPath, outDir)) { + log.warn( + `${LOG_PREFIX}: not writing it to "${absoluteDumpPath}", because that's inside the output directory.`, + ); + return; + } + + await outputJson(absoluteDumpPath, payload); + log.info(`${LOG_PREFIX}: wrote ${payload.scopes.length} file(s) to "${absoluteDumpPath}".`); +}; + +/** + * Hooks that generate the symbol table after one-off builds. They're only added when something + * consumes the table, so other builds don't do any extra work. The table never fails the build. + */ +export const getSymbolTableHooks = ( + pluginOptions: LiveDebuggerOptionsWithDefaults, + context: GlobalContext, + log: Logger, +): Partial => { + const dumpPath = process.env[SYMBOL_TABLE_PATH_ENV]; + if (!dumpPath) { + return {}; + } + + const buildMode = createBuildModeTracker(); + + const handleSymbolTable = async (report: BuildReport) => { + // Wait for the injection plugin to finish rewriting esbuild's output. + await context.artifactsReady; + const timer = log.time('generate symbol table'); + try { + const table = await generateSymbolTable({ + outputs: report.outputs ?? [], + buildRoot: context.buildRoot, + log, + }); + if (table) { + const payload = serializeSymbolTable(table, pluginOptions.version); + await writeSymbolTable(payload, dumpPath, context.bundler.outDir, log); + } + } catch (error) { + log.warn(`Could not generate the ${LOG_PREFIX}: ${error}`); + } finally { + timer.end(); + } + }; + + return { + ...buildMode.hooks, + async buildReport(report) { + if (!buildMode.isOneOffBuild()) { + return; + } + + const handling = handleSymbolTable(report); + if (context.artifactsPending) { + context.queue(handling); + return; + } + await handling; + }, + }; +}; diff --git a/packages/plugins/live-debugger/src/symbol-table/locate.ts b/packages/plugins/live-debugger/src/symbol-table/locate.ts new file mode 100644 index 000000000..b27026b4a --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/locate.ts @@ -0,0 +1,122 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { existsSync, readFile } from '@dd/core/helpers/fs'; +import { originalPositionFor, TraceMap } from '@jridgewell/trace-mapping'; +import path from 'path'; + +import type { ChunkScan } from './scan'; +import type { ChunkFunction, SourcePosition } from './types'; + +const SOURCE_MAPPING_URL_RX = /[#@]\s*sourceMappingURL=(\S+)\s*$/; +const DATA_URL_RX = /^data:[^,]*?(;base64)?,(.*)$/; +// The `sourceMappingURL` comment is at the end of the chunk. +const SOURCE_MAPPING_URL_LOOKBEHIND = 4096; + +// Load the chunk's final source map: `.map`, or the one its `sourceMappingURL` points to. +export const loadSourceMap = async ( + filepath: string, + code: string, +): Promise => { + try { + const siblingMapPath = `${filepath}.map`; + if (existsSync(siblingMapPath)) { + const siblingMap = await readFile(siblingMapPath); + return new TraceMap(siblingMap); + } + + const tail = code.slice(-SOURCE_MAPPING_URL_LOOKBEHIND); + const url = SOURCE_MAPPING_URL_RX.exec(tail)?.[1]; + if (!url) { + return undefined; + } + + const dataUrl = DATA_URL_RX.exec(url); + if (dataUrl) { + const [, base64, payload] = dataUrl; + const inlineMap = base64 + ? Buffer.from(payload, 'base64').toString('utf-8') + : decodeURIComponent(payload); + return new TraceMap(inlineMap); + } + + const chunkDir = path.dirname(filepath); + const linkedMapPath = path.resolve(chunkDir, url); + if (!existsSync(linkedMapPath)) { + return undefined; + } + const linkedMap = await readFile(linkedMapPath); + return new TraceMap(linkedMap); + } catch { + return undefined; + } +}; + +// Return a function that converts an offset in `code` to a 1-based line and 0-based column. +const createOffsetConverter = (code: string) => { + const lineStarts = [0]; + let newline = code.indexOf('\n'); + while (newline !== -1) { + lineStarts.push(newline + 1); + newline = code.indexOf('\n', newline + 1); + } + + return (offset: number): SourcePosition => { + let low = 0; + let high = lineStarts.length - 1; + while (low < high) { + const middle = Math.ceil((low + high) / 2); + if (lineStarts[middle] <= offset) { + low = middle; + } else { + high = middle - 1; + } + } + + return { line: low + 1, column: offset - lineStarts[low] }; + }; +}; + +const getOriginalPosition = ( + traceMap: TraceMap, + generated: SourcePosition, +): SourcePosition | undefined => { + const original = originalPositionFor(traceMap, generated); + if (original.line === null || original.column === null) { + return undefined; + } + + return { line: original.line, column: original.column }; +}; + +/** + * Map the scanned functions to their original positions. The transform anchors the `$dd_probes` + * call to the body's opening brace and the `$dd_throw` call to its closing brace. + */ +export const locateFunctions = ( + code: string, + scan: ChunkScan, + traceMap: TraceMap | undefined, +): ChunkFunction[] => { + const toPosition = createOffsetConverter(code); + + return scan.functions.map((scannedFunction) => { + const { functionId, probeOffset, throwOffset } = scannedFunction; + const parentIndex = scan.paired ? scannedFunction.parentIndex : undefined; + + if (!traceMap || !scan.paired || throwOffset === undefined) { + return { functionId, parentIndex, start: undefined, end: undefined }; + } + + const generatedStart = toPosition(probeOffset); + const generatedEnd = toPosition(throwOffset); + const start = getOriginalPosition(traceMap, generatedStart); + const end = getOriginalPosition(traceMap, generatedEnd); + if (!start || !end || end.line < start.line) { + return { functionId, parentIndex, start: undefined, end: undefined }; + } + + return { functionId, parentIndex, start, end }; + }); +}; diff --git a/packages/plugins/live-debugger/src/symbol-table/paths.test.ts b/packages/plugins/live-debugger/src/symbol-table/paths.test.ts new file mode 100644 index 000000000..3bfdc0e23 --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/paths.test.ts @@ -0,0 +1,124 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { + createSourceFileResolver, + isInsideDirectory, + resolveYarnVirtualPath, + stripQuery, +} from './paths'; + +// Paths that don't exist, so resolving symlinks leaves them unchanged. +const REPOSITORY_ROOT = '/does-not-exist/repo'; +const BUILD_ROOT = `${REPOSITORY_ROOT}/app`; + +describe('stripQuery', () => { + const cases = [ + { description: 'keep a path without a query', input: 'src/a.js', expected: 'src/a.js' }, + { + description: 'remove a loader query', + input: 'src/a.js?inline', + expected: 'src/a.js', + }, + { + description: 'remove a Vue single-file component query', + input: 'src/Cart.vue?vue&type=script&lang.ts', + expected: 'src/Cart.vue', + }, + ]; + + test.each(cases)('Should $description.', ({ input, expected }) => { + expect(stripQuery(input)).toBe(expected); + }); +}); + +describe('resolveYarnVirtualPath', () => { + const cases = [ + { + description: 'keep a regular path', + input: '/repo/packages/a/index.js', + expected: '/repo/packages/a/index.js', + }, + { + description: 'resolve a web-ui workspace package', + input: '/web-ui/.yarn/__virtual__/hash-05a436a78a514e39/1/packages/apps/databases/toolkit/group.ts', + expected: '/web-ui/packages/apps/databases/toolkit/group.ts', + }, + { + description: 'go up as many levels as the depth', + input: '/repo/sub/.yarn/__virtual__/pkg-virtual-abc/2/packages/a.js', + expected: '/repo/packages/a.js', + }, + { + description: 'stay in the virtual folder parent with a depth of 0', + input: '/repo/.yarn/__virtual__/pkg-virtual-abc/0/cache/a.js', + expected: '/repo/.yarn/cache/a.js', + }, + { + description: 'resolve the name used by older Yarn versions', + input: '/repo/.yarn/$$virtual/pkg-virtual-abc/1/packages/a.js', + expected: '/repo/packages/a.js', + }, + { + description: 'resolve nested virtual paths', + input: '/repo/.yarn/__virtual__/a-virtual-1/1/.yarn/__virtual__/b-virtual-2/1/packages/b.js', + expected: '/repo/packages/b.js', + }, + ]; + + test.each(cases)('Should $description.', ({ input, expected }) => { + expect(resolveYarnVirtualPath(input)).toBe(expected); + }); +}); + +describe('createSourceFileResolver', () => { + const cases = [ + { + description: 'make a path relative to the repository root', + input: 'src/a.js', + expected: 'app/src/a.js', + }, + { + description: 'strip the query string', + input: 'src/Cart.vue?vue&type=script', + expected: 'app/src/Cart.vue', + }, + { + description: 'resolve a workspace package outside the build root', + input: '../packages/lib/index.js', + expected: 'packages/lib/index.js', + }, + { + description: 'resolve a Yarn virtual path', + input: '../.yarn/__virtual__/lib-virtual-abc/1/packages/lib/index.js', + expected: 'packages/lib/index.js', + }, + { + description: 'leave out a file outside the repository', + input: '../../shared/index.js', + expected: undefined, + }, + ]; + + test.each(cases)('Should $description.', ({ input, expected }) => { + const resolveSourceFile = createSourceFileResolver(BUILD_ROOT, REPOSITORY_ROOT); + expect(resolveSourceFile(input)).toBe(expected); + }); +}); + +describe('isInsideDirectory', () => { + const cases = [ + { description: 'accept a nested file', input: '/out/dir/a.json', expected: true }, + { description: 'reject a sibling file', input: '/out/a.json', expected: false }, + { + description: 'reject a directory that only shares a prefix', + input: '/out/dir-2/a.json', + expected: false, + }, + ]; + + test.each(cases)('Should $description.', ({ input, expected }) => { + expect(isInsideDirectory(input, '/out/dir')).toBe(expected); + }); +}); diff --git a/packages/plugins/live-debugger/src/symbol-table/paths.ts b/packages/plugins/live-debugger/src/symbol-table/paths.ts new file mode 100644 index 000000000..6dfd7ef6b --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/paths.ts @@ -0,0 +1,85 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { realpathSync } from '@dd/core/helpers/fs'; +import { getClosest } from '@dd/core/helpers/paths'; +import path from 'path'; + +// `/__virtual__///`, as created by Yarn PnP for packages with peer +// dependencies. `$$virtual` is the name used by older Yarn versions. +const YARN_VIRTUAL_PATH_RX = /^(.*\/(?:__virtual__|\$\$virtual))\/[^/]+\/(\d+)(\/.*)?$/; + +const toPosixPath = (filepath: string) => filepath.replace(/\\/g, '/'); + +// Remove the query string of a module ID, e.g. `Cart.vue?vue&type=script`. +export const stripQuery = (moduleId: string): string => { + const queryStart = moduleId.indexOf('?'); + return queryStart === -1 ? moduleId : moduleId.slice(0, queryStart); +}; + +/** + * Resolve a Yarn PnP virtual path to the path it stands for, the way Yarn does: for + * `/__virtual__///`, go up `depth` levels from `` and append ``. + */ +export const resolveYarnVirtualPath = (filepath: string): string => { + const posixPath = toPosixPath(filepath); + const match = YARN_VIRTUAL_PATH_RX.exec(posixPath); + if (!match) { + return filepath; + } + + const [, virtualDir, depth, rest = ''] = match; + const target = path.posix.dirname(virtualDir); + const backtrack = '../'.repeat(Number(depth)); + const resolved = path.posix.join(target, backtrack, rest); + + // A resolved path can itself be virtual. + return resolveYarnVirtualPath(resolved); +}; + +// Return the canonical root of the git repository that contains the build root. +export const getRepositoryRoot = (buildRoot: string): string | undefined => { + const gitPath = getClosest(buildRoot, '.git'); + if (!gitPath) { + return undefined; + } + + const repositoryRoot = path.dirname(gitPath); + return realpathSync(repositoryRoot); +}; + +const isOutside = (relativePath: string) => + relativePath === '' || + relativePath === '..' || + relativePath.startsWith(`..${path.sep}`) || + path.isAbsolute(relativePath); + +export const isInsideDirectory = (filepath: string, directory: string): boolean => { + const relativePath = path.relative(directory, filepath); + return relativePath === '' || !isOutside(relativePath); +}; + +/** + * Return a function that converts the module path of a function ID to a path relative to the + * repository root, or to undefined when the module is outside the repository. + */ +export const createSourceFileResolver = (buildRoot: string, repositoryRoot: string) => { + const cache = new Map(); + + return (modulePath: string): string | undefined => { + if (cache.has(modulePath)) { + return cache.get(modulePath); + } + + const moduleFile = stripQuery(modulePath); + const absolutePath = path.resolve(buildRoot, moduleFile); + const unvirtualizedPath = resolveYarnVirtualPath(absolutePath); + const realPath = realpathSync(unvirtualizedPath); + const relativePath = path.relative(repositoryRoot, realPath); + const sourceFile = isOutside(relativePath) ? undefined : toPosixPath(relativePath); + + cache.set(modulePath, sourceFile); + return sourceFile; + }; +}; diff --git a/packages/plugins/live-debugger/src/symbol-table/scan.test.ts b/packages/plugins/live-debugger/src/symbol-table/scan.test.ts new file mode 100644 index 000000000..c6f4f7726 --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/scan.test.ts @@ -0,0 +1,117 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { readChunkDebugId, scanChunk } from './scan'; + +const DEBUG_ID = '0f5e4a5c-3b1d-4c8e-9a7f-1b2c3d4e5f60'; + +describe('scanChunk', () => { + const cases = [ + { + description: 'read single, double and backtick quoted function IDs', + code: `$dd_probes('a.js;one');$dd_throw(p);$dd_probes("a.js;two");$dd_throw(p);$dd_probes(\`a.js;three\`);$dd_throw(p);`, + expected: { + functionIds: ['a.js;one', 'a.js;two', 'a.js;three'], + parentIndexes: [undefined, undefined, undefined], + paired: true, + }, + }, + { + description: 'decode escape sequences', + code: String.raw`$dd_probes('a.js;it\'s');$dd_throw(p);$dd_probes("a.js;\x41\u0042\u{43}\\");$dd_throw(p);`, + expected: { + functionIds: ["a.js;it's", 'a.js;ABC\\'], + parentIndexes: [undefined, undefined], + paired: true, + }, + }, + { + description: 'nest functions like brackets', + code: `function a(){let p=$dd_probes('a.js;outer');try{const b=()=>{let q=$dd_probes('a.js;inner');try{}catch(e){$dd_throw(q,e)}};}catch(e){$dd_throw(p,e)}}function c(){let r=$dd_probes('a.js;sibling');try{}catch(e){$dd_throw(r,e)}}`, + expected: { + functionIds: ['a.js;outer', 'a.js;inner', 'a.js;sibling'], + parentIndexes: [undefined, 0, undefined], + paired: true, + }, + }, + { + description: 'accept whitespace around the argument', + code: `$dd_probes ( 'a.js;spaced' ) ;$dd_throw (p);`, + expected: { + functionIds: ['a.js;spaced'], + parentIndexes: [undefined], + paired: true, + }, + }, + { + description: 'ignore member calls, declarations and non-constant arguments', + code: `obj.$dd_probes('a.js;member');function $dd_throw(a){}x$dd_probes('a.js;prefixed');$dd_probes(id);$dd_probes(\`a.js;\${name}\`);$dd_probes('a.js;' + name);`, + expected: { + functionIds: [], + parentIndexes: [], + paired: true, + }, + }, + { + description: 'report a closing marker without an opening one', + code: `$dd_throw(p);$dd_probes('a.js;one');$dd_throw(p);`, + expected: { + functionIds: ['a.js;one'], + parentIndexes: [undefined], + paired: false, + }, + }, + { + description: 'report an opening marker without a closing one', + code: `$dd_probes('a.js;one');`, + expected: { + functionIds: ['a.js;one'], + parentIndexes: [undefined], + paired: false, + }, + }, + ]; + + test.each(cases)('Should $description.', ({ code, expected }) => { + const scan = scanChunk(code); + const functionIds = scan.functions.map((scanned) => scanned.functionId); + const parentIndexes = scan.functions.map((scanned) => scanned.parentIndex); + + expect(functionIds).toEqual(expected.functionIds); + expect(parentIndexes).toEqual(expected.parentIndexes); + expect(scan.paired).toBe(expected.paired); + }); + + test('Should record the offsets of both markers.', () => { + const code = `let p=$dd_probes('a.js;one');try{}catch(e){$dd_throw(p,e)}`; + const [scanned] = scanChunk(code).functions; + + expect(scanned.probeOffset).toBe(code.indexOf('$dd_probes')); + expect(scanned.throwOffset).toBe(code.indexOf('$dd_throw')); + }); +}); + +describe('readChunkDebugId', () => { + const cases = [ + { + description: 'read the debug ID of the runtime bootstrap', + code: `globalThis.__DD_LIVE_DEBUGGER_BUILD__={"version":"1.0.0","debugId":"${DEBUG_ID}"}`, + expected: DEBUG_ID, + }, + { + description: 'read the debug ID of a minified runtime bootstrap', + code: `globalThis.__DD_LIVE_DEBUGGER_BUILD__={version:"1.0.0",debugId:'${DEBUG_ID}'}`, + expected: DEBUG_ID, + }, + { + description: 'return undefined without a debug ID', + code: `globalThis.__DD_LIVE_DEBUGGER_BUILD__={"version":"1.0.0"}`, + expected: undefined, + }, + ]; + + test.each(cases)('Should $description.', ({ code, expected }) => { + expect(readChunkDebugId(code)).toBe(expected); + }); +}); diff --git a/packages/plugins/live-debugger/src/symbol-table/scan.ts b/packages/plugins/live-debugger/src/symbol-table/scan.ts new file mode 100644 index 000000000..447616750 --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/scan.ts @@ -0,0 +1,244 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { LIVE_DEBUGGER_PROBES_FUNCTION } from '@dd/core/constants'; + +import { LIVE_DEBUGGER_THROW_FUNCTION } from '../constants'; + +// An instrumented function found in a chunk's code. +export type ScannedFunction = { + functionId: string; + // Offset of the `$dd_probes('')` call at the start of the function's body. + probeOffset: number; + // Offset of the `$dd_throw(` call at the end of the function's body. + throwOffset: number | undefined; + // Index of the enclosing instrumented function in the same list. + parentIndex: number | undefined; +}; + +export type ChunkScan = { + functions: ScannedFunction[]; + // False when the markers don't nest, so `throwOffset` and `parentIndex` can't be trusted. + paired: boolean; +}; + +const escapeRegExp = (value: string) => value.replace(/[$()*+.?[\\\]^{|}]/g, '\\$&'); + +const MARKER_RX = new RegExp( + `(${escapeRegExp(LIVE_DEBUGGER_PROBES_FUNCTION)}|${escapeRegExp(LIVE_DEBUGGER_THROW_FUNCTION)})\\s*\\(`, + 'g', +); +// A marker preceded by one of these is another identifier or a member access, not a call. +const NON_CALL_PREFIX_RX = /[\w$.]/; +// A marker preceded by this is a function declaration, not a call. +const FUNCTION_DECLARATION_SUFFIX_RX = /\bfunction\s*\*?\s*$/; +const FUNCTION_DECLARATION_LOOKBEHIND = 16; +// The runtime bootstrap injected into every chunk records the chunk's debug ID. +// Minifiers can drop the quotes around the key and change the quotes around the value. +const BUILD_METADATA_DEBUG_ID_RX = + /__DD_LIVE_DEBUGGER_BUILD__\s*=\s*\{[^}]*?["']?debugId["']?\s*:\s*["']([0-9a-fA-F-]{36})["']/; + +const QUOTES = new Set(["'", '"', '`']); +const WHITESPACE_RX = /\s/; +const HEX_RX = /^[0-9a-fA-F]+$/; +const SINGLE_CHARACTER_ESCAPES: Record = { + b: '\b', + f: '\f', + n: '\n', + r: '\r', + t: '\t', + v: '\v', + 0: '\0', +}; +const LINE_CONTINUATIONS = new Set(['\n', '\u2028', '\u2029']); + +type Decoded = { + value: string; + // Offset right after the decoded source text. + end: number; +}; + +const skipWhitespace = (code: string, offset: number): number => { + let index = offset; + while (index < code.length && WHITESPACE_RX.test(code[index])) { + index++; + } + return index; +}; + +const isCallSite = (code: string, offset: number): boolean => { + if (offset > 0 && NON_CALL_PREFIX_RX.test(code[offset - 1])) { + return false; + } + + const lookbehindStart = Math.max(0, offset - FUNCTION_DECLARATION_LOOKBEHIND); + const lookbehind = code.slice(lookbehindStart, offset); + return !FUNCTION_DECLARATION_SUFFIX_RX.test(lookbehind); +}; + +const decodeCodePoint = (hex: string, end: number): Decoded | undefined => { + if (!HEX_RX.test(hex)) { + return undefined; + } + + const codePoint = parseInt(hex, 16); + if (codePoint > 0x10ffff) { + return undefined; + } + + return { value: String.fromCodePoint(codePoint), end }; +}; + +// Decode the escape sequence whose backslash is right before `offset`. +const decodeEscape = (code: string, offset: number): Decoded | undefined => { + const char = code[offset]; + if (char === undefined) { + return undefined; + } + + if (char === 'x') { + const hex = code.slice(offset + 1, offset + 3); + return hex.length === 2 ? decodeCodePoint(hex, offset + 3) : undefined; + } + + if (char === 'u') { + if (code[offset + 1] === '{') { + const closingBrace = code.indexOf('}', offset + 2); + if (closingBrace === -1) { + return undefined; + } + const hex = code.slice(offset + 2, closingBrace); + return decodeCodePoint(hex, closingBrace + 1); + } + + const hex = code.slice(offset + 1, offset + 5); + return hex.length === 4 ? decodeCodePoint(hex, offset + 5) : undefined; + } + + if (char === '\r') { + const end = code[offset + 1] === '\n' ? offset + 2 : offset + 1; + return { value: '', end }; + } + + if (LINE_CONTINUATIONS.has(char)) { + return { value: '', end: offset + 1 }; + } + + return { value: SINGLE_CHARACTER_ESCAPES[char] ?? char, end: offset + 1 }; +}; + +// Decode the string literal starting at `offset`, which must be its opening quote. +const decodeStringLiteral = (code: string, offset: number): Decoded | undefined => { + const quote = code[offset]; + if (!QUOTES.has(quote)) { + return undefined; + } + + const isTemplate = quote === '`'; + let value = ''; + let index = offset + 1; + let segmentStart = index; + + while (index < code.length) { + const char = code[index]; + + if (char === quote) { + value += code.slice(segmentStart, index); + return { value, end: index + 1 }; + } + + if (char === '\\') { + value += code.slice(segmentStart, index); + const escape = decodeEscape(code, index + 1); + if (!escape) { + return undefined; + } + value += escape.value; + index = escape.end; + segmentStart = index; + continue; + } + + // Only constant strings are function IDs. + if (isTemplate && char === '$' && code[index + 1] === '{') { + return undefined; + } + + if (!isTemplate && (char === '\n' || char === '\r')) { + return undefined; + } + + index++; + } + + return undefined; +}; + +// Read the function ID of a `$dd_probes('')` call, from right after its opening parenthesis. +const readFunctionIdArgument = (code: string, offset: number): string | undefined => { + const literalStart = skipWhitespace(code, offset); + const literal = decodeStringLiteral(code, literalStart); + if (!literal) { + return undefined; + } + + const closingParenthesis = skipWhitespace(code, literal.end); + return code[closingParenthesis] === ')' ? literal.value : undefined; +}; + +/** + * Find the instrumented functions in a chunk's code, without parsing it. + * + * Every instrumented function contains exactly one `$dd_probes('')` call at the start of its + * body and one `$dd_throw(` call at its end, in the injected `catch`. Minifiers keep both, because + * they are calls to globals. Pairing them like brackets gives each function's extent, and the + * nesting gives its enclosing function. + */ +export const scanChunk = (code: string): ChunkScan => { + const functions: ScannedFunction[] = []; + const openFunctions: number[] = []; + let paired = true; + + for (const match of code.matchAll(MARKER_RX)) { + const offset = match.index; + if (!isCallSite(code, offset)) { + continue; + } + + if (match[1] === LIVE_DEBUGGER_PROBES_FUNCTION) { + const argumentOffset = offset + match[0].length; + const functionId = readFunctionIdArgument(code, argumentOffset); + if (functionId === undefined) { + continue; + } + + functions.push({ + functionId, + probeOffset: offset, + throwOffset: undefined, + parentIndex: openFunctions.at(-1), + }); + openFunctions.push(functions.length - 1); + continue; + } + + const closedFunction = openFunctions.pop(); + if (closedFunction === undefined) { + paired = false; + continue; + } + functions[closedFunction].throwOffset = offset; + } + + if (openFunctions.length > 0) { + paired = false; + } + + return { functions, paired }; +}; + +// Read the chunk's debug ID from the runtime bootstrap. +export const readChunkDebugId = (code: string): string | undefined => { + return BUILD_METADATA_DEBUG_ID_RX.exec(code)?.[1]; +}; diff --git a/packages/plugins/live-debugger/src/symbol-table/serialize.ts b/packages/plugins/live-debugger/src/symbol-table/serialize.ts new file mode 100644 index 000000000..b2d1867fd --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/serialize.ts @@ -0,0 +1,126 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import type { SourcePosition, SymbolTable, SymbolTableEntry } from './types'; + +// SymDB language of browser symbol tables. +export const SYMBOL_TABLE_LANGUAGE = 'browser'; + +// A SymDB scope, in the format the backend SDKs upload. +export type SymDbScope = { + scope_type: 'MODULE' | 'FUNCTION'; + name: string; + source_file: string; + start_line?: number; + end_line?: number; + language_specifics?: { + function_id?: string; + chunk_debug_ids?: string[]; + }; + scopes?: SymDbScope[]; +}; + +// The symbol table in the format of the upload's file part. The upload adds the fields that are +// specific to each request: `service`, `upload_id`, `batch_num` and `final`. +export type SymbolTablePayload = { + language: typeof SYMBOL_TABLE_LANGUAGE; + version?: string; + chunk_debug_ids?: string[]; + scopes: SymDbScope[]; +}; + +const comparePositions = (a: SourcePosition | undefined, b: SourcePosition | undefined) => { + if (!a || !b) { + // Functions without a position go last. + return Number(!a) - Number(!b); + } + return a.line - b.line || a.column - b.column; +}; + +const compareStrings = (a: string, b: string) => (a < b ? -1 : Number(a > b)); + +// Source order, so enclosing functions come before the functions they contain. +const compareEntries = (a: SymbolTableEntry, b: SymbolTableEntry) => + comparePositions(a.start, b.start) || + compareStrings(a.functionId, b.functionId) || + comparePositions(a.end, b.end); + +const toFunctionScope = ( + entry: SymbolTableEntry, + childrenByParentKey: Map, +): SymDbScope => { + const children = childrenByParentKey.get(entry.key) ?? []; + const scope: SymDbScope = { + scope_type: 'FUNCTION', + name: entry.name, + source_file: entry.sourceFile, + }; + + if (entry.start && entry.end) { + scope.start_line = entry.start.line; + scope.end_line = entry.end.line; + } + + scope.language_specifics = { function_id: entry.functionId }; + + if (children.length > 0) { + scope.scopes = children.map((child) => toFunctionScope(child, childrenByParentKey)); + } + + return scope; +}; + +/** + * Convert the symbol table to SymDB scopes: one `MODULE` scope per file, without a line range, and + * one `FUNCTION` scope per function, nested under its enclosing function, in source order. + */ +export const serializeSymbolTable = ( + table: SymbolTable, + version: string | undefined, +): SymbolTablePayload => { + const sortedEntries = [...table.entries].sort(compareEntries); + const topLevelEntriesByFile = new Map(); + const childrenByParentKey = new Map(); + + for (const entry of sortedEntries) { + const siblings = entry.parentKey + ? childrenByParentKey.get(entry.parentKey) ?? [] + : topLevelEntriesByFile.get(entry.sourceFile) ?? []; + siblings.push(entry); + + if (entry.parentKey) { + childrenByParentKey.set(entry.parentKey, siblings); + } else { + topLevelEntriesByFile.set(entry.sourceFile, siblings); + } + } + + const sourceFiles = [...topLevelEntriesByFile.keys()].sort(compareStrings); + const scopes = sourceFiles.map((sourceFile): SymDbScope => { + const topLevelEntries = topLevelEntriesByFile.get(sourceFile) ?? []; + const chunkDebugIds = table.fileChunkDebugIds?.get(sourceFile); + const moduleScope: SymDbScope = { + scope_type: 'MODULE', + name: sourceFile, + source_file: sourceFile, + }; + + if (chunkDebugIds) { + moduleScope.language_specifics = { chunk_debug_ids: chunkDebugIds }; + } + + moduleScope.scopes = topLevelEntries.map((entry) => + toFunctionScope(entry, childrenByParentKey), + ); + + return moduleScope; + }); + + return { + language: SYMBOL_TABLE_LANGUAGE, + ...(version ? { version } : {}), + ...(table.chunkDebugIds ? { chunk_debug_ids: table.chunkDebugIds } : {}), + scopes, + }; +}; diff --git a/packages/plugins/live-debugger/src/symbol-table/table.test.ts b/packages/plugins/live-debugger/src/symbol-table/table.test.ts new file mode 100644 index 000000000..37bff5f9f --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/table.test.ts @@ -0,0 +1,209 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { serializeSymbolTable } from './serialize'; +import { buildSymbolTable } from './table'; +import type { ChunkFunction, ChunkFunctions } from './types'; + +const DEBUG_ID_A = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'; +const DEBUG_ID_B = 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb'; + +// Module paths are relative to the build root, `app` in the repository. +const resolveSourceFile = (modulePath: string) => + modulePath.startsWith('../../') ? undefined : `app/${modulePath.replace(/\?.*$/, '')}`; + +const getFunction = ( + functionId: string, + lines: [number, number] | undefined, + parentIndex?: number, +): ChunkFunction => ({ + functionId, + parentIndex, + start: lines ? { line: lines[0], column: 0 } : undefined, + end: lines ? { line: lines[1], column: 0 } : undefined, +}); + +const getChunk = ( + debugId: string | undefined, + functions: ChunkFunction[], + filepath = 'out.js', +): ChunkFunctions => ({ filepath, debugId, functions }); + +describe('buildSymbolTable', () => { + const cases = [ + { + description: 'list the same function once when several chunks contain it', + chunks: [ + getChunk(DEBUG_ID_A, [getFunction('src/a.js;add', [1, 3])]), + getChunk(DEBUG_ID_B, [getFunction('src/a.js;add', [1, 3])]), + ], + expected: { functionIds: ['src/a.js;add'] }, + }, + { + description: 'keep functions that share an ID at different places', + chunks: [ + getChunk(DEBUG_ID_A, [ + getFunction('src/a.js;render', [1, 3]), + getFunction('src/a.js;render', [5, 7]), + ]), + ], + expected: { functionIds: ['src/a.js;render', 'src/a.js;render'] }, + }, + { + description: 'leave out functions outside the repository and invalid IDs', + chunks: [ + getChunk(DEBUG_ID_A, [ + getFunction('../../shared/a.js;outside', [1, 3]), + getFunction('no-separator', [1, 3]), + getFunction('src/a.js;inside', [1, 3]), + ]), + ], + expected: { + functionIds: ['src/a.js;inside'], + outsideModulePaths: ['../../shared/a.js'], + invalidFunctionIds: ['no-separator'], + }, + }, + ]; + + test.each(cases)('Should $description.', ({ chunks, expected }) => { + const result = buildSymbolTable(chunks, resolveSourceFile); + const functionIds = result.table.entries.map((entry) => entry.functionId); + const outsideModulePaths = [...result.outsideModulePaths]; + const invalidFunctionIds = [...result.invalidFunctionIds]; + + expect(functionIds).toEqual(expected.functionIds); + expect(outsideModulePaths).toEqual(expected.outsideModulePaths ?? []); + expect(invalidFunctionIds).toEqual(expected.invalidFunctionIds ?? []); + }); + + test('Should record which chunks contain each file.', () => { + const chunks = [ + getChunk(DEBUG_ID_B, [getFunction('src/a.js;add', [1, 3])]), + getChunk(DEBUG_ID_A, [ + getFunction('src/a.js;add', [1, 3]), + getFunction('src/b.js;sub', [1, 3]), + ]), + getChunk(DEBUG_ID_A, []), + ]; + const { table } = buildSymbolTable(chunks, resolveSourceFile); + const fileChunkDebugIds = Object.fromEntries(table.fileChunkDebugIds ?? []); + + expect(table.chunkDebugIds).toEqual([DEBUG_ID_A, DEBUG_ID_B]); + expect(fileChunkDebugIds).toEqual({ + 'app/src/a.js': [DEBUG_ID_A, DEBUG_ID_B], + 'app/src/b.js': [DEBUG_ID_A], + }); + }); + + test('Should leave out chunk debug IDs when a chunk with functions has none.', () => { + const chunks = [ + getChunk(DEBUG_ID_A, [getFunction('src/a.js;add', [1, 3])]), + getChunk(undefined, [getFunction('src/b.js;sub', [1, 3])]), + ]; + const { table } = buildSymbolTable(chunks, resolveSourceFile); + + expect(table.chunkDebugIds).toBeUndefined(); + expect(table.fileChunkDebugIds).toBeUndefined(); + }); + + test('Should only nest functions under enclosing functions of the same file.', () => { + const chunks = [ + getChunk(DEBUG_ID_A, [ + getFunction('src/a.js;outer', [1, 9]), + // Inlined into `outer` by a minifier. + getFunction('src/b.js;inlined', [1, 3], 0), + getFunction('src/a.js;inner', [2, 4], 1), + ]), + ]; + const { table } = buildSymbolTable(chunks, resolveSourceFile); + const [outer, inlined, inner] = table.entries; + + expect(outer.parentKey).toBeUndefined(); + expect(inlined.parentKey).toBeUndefined(); + expect(inner.parentKey).toBe(outer.key); + }); +}); + +describe('serializeSymbolTable', () => { + test('Should convert the table to SymDB scopes in source order.', () => { + const chunks = [ + getChunk(DEBUG_ID_A, [ + getFunction('src/b.js;later', [10, 12]), + getFunction('src/b.js;outer', [1, 8]), + getFunction('src/b.js;@3:4:0', [3, 5], 1), + getFunction('src/a.js;unmapped', undefined), + ]), + ]; + const { table } = buildSymbolTable(chunks, resolveSourceFile); + + expect(serializeSymbolTable(table, '1.0.0')).toEqual({ + language: 'browser', + version: '1.0.0', + chunk_debug_ids: [DEBUG_ID_A], + scopes: [ + { + scope_type: 'MODULE', + name: 'app/src/a.js', + source_file: 'app/src/a.js', + language_specifics: { chunk_debug_ids: [DEBUG_ID_A] }, + scopes: [ + { + scope_type: 'FUNCTION', + name: 'unmapped', + source_file: 'app/src/a.js', + language_specifics: { function_id: 'src/a.js;unmapped' }, + }, + ], + }, + { + scope_type: 'MODULE', + name: 'app/src/b.js', + source_file: 'app/src/b.js', + language_specifics: { chunk_debug_ids: [DEBUG_ID_A] }, + scopes: [ + { + scope_type: 'FUNCTION', + name: 'outer', + source_file: 'app/src/b.js', + start_line: 1, + end_line: 8, + language_specifics: { function_id: 'src/b.js;outer' }, + scopes: [ + { + scope_type: 'FUNCTION', + name: '', + source_file: 'app/src/b.js', + start_line: 3, + end_line: 5, + language_specifics: { + function_id: 'src/b.js;@3:4:0', + }, + }, + ], + }, + { + scope_type: 'FUNCTION', + name: 'later', + source_file: 'app/src/b.js', + start_line: 10, + end_line: 12, + language_specifics: { function_id: 'src/b.js;later' }, + }, + ], + }, + ], + }); + }); + + test('Should leave out the version and chunk debug IDs when unknown.', () => { + const chunks = [getChunk(undefined, [getFunction('src/a.js;add', [1, 3])])]; + const { table } = buildSymbolTable(chunks, resolveSourceFile); + const payload = serializeSymbolTable(table, undefined); + + expect(payload).not.toHaveProperty('version'); + expect(payload).not.toHaveProperty('chunk_debug_ids'); + expect(payload.scopes[0]).not.toHaveProperty('language_specifics'); + }); +}); diff --git a/packages/plugins/live-debugger/src/symbol-table/table.ts b/packages/plugins/live-debugger/src/symbol-table/table.ts new file mode 100644 index 000000000..329930dde --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/table.ts @@ -0,0 +1,127 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { parseFunctionId } from '../transform/functionId'; + +import type { + ChunkFunction, + ChunkFunctions, + SourcePosition, + SymbolTable, + SymbolTableEntry, +} from './types'; + +export type SymbolTableResult = { + table: SymbolTable; + // Module paths of functions dropped because they're outside the repository. + outsideModulePaths: Set; + // Function IDs that couldn't be parsed. + invalidFunctionIds: Set; +}; + +const formatPosition = (position: SourcePosition | undefined) => + position ? `${position.line}:${position.column}` : ''; + +// The same function can be in several chunks, or in several output formats of the same build. +// Functions that share an ID but are at different places are different functions (v1 IDs aren't +// unique), so the key includes the location. +const getEntryKey = (chunkFunction: ChunkFunction, sourceFile: string) => { + const start = formatPosition(chunkFunction.start); + const end = formatPosition(chunkFunction.end); + return [chunkFunction.functionId, sourceFile, start, end].join('\0'); +}; + +// Map the output's chunks to one entry per function, deduplicated across chunks. +export const buildSymbolTable = ( + chunks: ChunkFunctions[], + resolveSourceFile: (modulePath: string) => string | undefined, +): SymbolTableResult => { + const entries = new Map(); + const fileChunkDebugIds = new Map>(); + const chunkDebugIds = new Set(); + const outsideModulePaths = new Set(); + const invalidFunctionIds = new Set(); + let hasCompleteDebugIds = true; + + for (const chunk of chunks) { + if (chunk.debugId) { + chunkDebugIds.add(chunk.debugId); + } else if (chunk.functions.length > 0) { + hasCompleteDebugIds = false; + } + + // Per function of the chunk, by index. + const keys: (string | undefined)[] = []; + const sourceFiles: (string | undefined)[] = []; + + // The closest enclosing function that's in the same file. + const getParentKey = (chunkFunction: ChunkFunction, sourceFile: string) => { + let parentIndex = chunkFunction.parentIndex; + while (parentIndex !== undefined) { + if (sourceFiles[parentIndex] === sourceFile && keys[parentIndex]) { + return keys[parentIndex]; + } + parentIndex = chunk.functions[parentIndex].parentIndex; + } + return undefined; + }; + + for (const chunkFunction of chunk.functions) { + const parsedId = parseFunctionId(chunkFunction.functionId); + if (!parsedId) { + invalidFunctionIds.add(chunkFunction.functionId); + keys.push(undefined); + sourceFiles.push(undefined); + continue; + } + + const sourceFile = resolveSourceFile(parsedId.modulePath); + if (!sourceFile) { + outsideModulePaths.add(parsedId.modulePath); + keys.push(undefined); + sourceFiles.push(undefined); + continue; + } + + const key = getEntryKey(chunkFunction, sourceFile); + const parentKey = getParentKey(chunkFunction, sourceFile); + keys.push(key); + sourceFiles.push(sourceFile); + + if (chunk.debugId) { + const debugIds = fileChunkDebugIds.get(sourceFile) ?? new Set(); + debugIds.add(chunk.debugId); + fileChunkDebugIds.set(sourceFile, debugIds); + } + + if (!entries.has(key)) { + entries.set(key, { + key, + functionId: chunkFunction.functionId, + name: parsedId.name, + sourceFile, + start: chunkFunction.start, + end: chunkFunction.end, + parentKey, + }); + } + } + } + + const sortedChunkDebugIds = [...chunkDebugIds].sort(); + const sortedFileEntries = [...fileChunkDebugIds].map(([sourceFile, debugIds]) => { + const sortedDebugIds = [...debugIds].sort(); + return [sourceFile, sortedDebugIds] as const; + }); + const sortedFileChunkDebugIds = new Map(sortedFileEntries); + + // Never publish incomplete chunk membership. + const table: SymbolTable = { + entries: [...entries.values()], + chunkDebugIds: hasCompleteDebugIds ? sortedChunkDebugIds : undefined, + fileChunkDebugIds: hasCompleteDebugIds ? sortedFileChunkDebugIds : undefined, + }; + + return { table, outsideModulePaths, invalidFunctionIds }; +}; diff --git a/packages/plugins/live-debugger/src/symbol-table/types.ts b/packages/plugins/live-debugger/src/symbol-table/types.ts new file mode 100644 index 000000000..26776b4ec --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/types.ts @@ -0,0 +1,49 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +// A position in an original source file: 1-based line, 0-based column. +export type SourcePosition = { + line: number; + column: number; +}; + +// An instrumented function found in an emitted chunk. +export type ChunkFunction = { + functionId: string; + // Index of the enclosing instrumented function in the same chunk. + parentIndex: number | undefined; + // Original positions of the function body's opening and closing brace. + // Both are undefined when either one can't be mapped. + start: SourcePosition | undefined; + end: SourcePosition | undefined; +}; + +export type ChunkFunctions = { + filepath: string; + // Undefined when the chunk has no debug ID. + debugId: string | undefined; + functions: ChunkFunction[]; +}; + +export type SymbolTableEntry = { + // Identifies the entry: the same function in several chunks has the same key. + key: string; + functionId: string; + name: string; + // Path relative to the repository root. + sourceFile: string; + start: SourcePosition | undefined; + end: SourcePosition | undefined; + // Key of the enclosing function's entry, when it's in the same file. + parentKey: string | undefined; +}; + +export type SymbolTable = { + entries: SymbolTableEntry[]; + // Debug IDs of the build's chunks. Undefined when a chunk with instrumented functions has none. + chunkDebugIds: string[] | undefined; + // For each source file, the debug IDs of the chunks that contain its functions. + // Undefined in the same case as `chunkDebugIds`. + fileChunkDebugIds: Map | undefined; +}; diff --git a/packages/plugins/live-debugger/src/symbolTable.integration.test.ts b/packages/plugins/live-debugger/src/symbolTable.integration.test.ts new file mode 100644 index 000000000..9f9334ec9 --- /dev/null +++ b/packages/plugins/live-debugger/src/symbolTable.integration.test.ts @@ -0,0 +1,339 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { datadogWebpackPlugin } from '@datadog/webpack-plugin'; +import { existsSync, mkdir, outputFile, readFile, readFileSync, rm } from '@dd/core/helpers/fs'; +import type { BundlerName, Options } from '@dd/core/types'; +import { getOutDir, getTempWorkingDir } from '@dd/tests/_jest/helpers/env'; +import { defaultPluginOptions } from '@dd/tests/_jest/helpers/mocks'; +import { BUNDLERS, getBundlerConfig } from '@dd/tests/_jest/helpers/runBundlers'; +import { buildWithWebpack, configWebpack } from '@dd/tools/bundlers'; +import fsp from 'fs/promises'; +import { glob } from 'glob'; +import path from 'path'; + +import { SYMBOL_TABLE_PATH_ENV } from './constants'; +import type { SymbolTablePayload, SymDbScope } from './symbol-table/serialize'; + +type XpackCompiler = { options: { module: { rules: object[] } } }; + +type TableFunction = { + functionId: string; + name: string; + sourceFile: string; + startLine: number | undefined; + endLine: number | undefined; + parentName: string | undefined; +}; + +// The repository root holds the git marker, the build root is its `app` directory. +const APP_DIR = 'app'; +const VIRTUAL_PEER_PATH = '.yarn/__virtual__/peer-virtual-0123456789/1/packages/peer/index.js'; +const XPACK_BUNDLERS: BundlerName[] = ['webpack', 'rspack']; +const PROBE_CALL_RX = /\$dd_probes\((['"`])(.*?)\1\)/g; + +const PEER_SOURCE = `export function peerHello() { + return 'hello from peer'; +} +`; + +const FIXTURE_FILES: Record = { + 'app/src/main.js': `import { libGreet } from '@fixture/lib'; +import { peerHello } from '../../${VIRTUAL_PEER_PATH}'; +import { add } from './math.js'; + +export const values = [1, 2].map(function (value) { + return add(value, 1); +}); + +console.log(values, libGreet(), peerHello()); +`, + // webpack and rspack also cover a module ID with a query string. + 'app/src/main-xpack.js': `import { render } from './component.js?query'; +import { values } from './main.js'; + +console.log(render(values.join(','))); +`, + 'app/src/math.js': `export function add(a, b) { + const double = (value) => { + return value * 2; + }; + return double(a) + b; +} +`, + 'app/src/component.js': `export function render(label) { + return '

' + label + '

'; +} +`, + 'packages/lib/package.json': JSON.stringify({ + name: '@fixture/lib', + version: '1.0.0', + main: 'index.js', + }), + 'packages/lib/index.js': `export function libGreet() { + return 'hello from lib'; +} +`, + 'packages/peer/index.js': PEER_SOURCE, + // Yarn PnP loads packages with peer dependencies through virtual paths. + [VIRTUAL_PEER_PATH]: PEER_SOURCE, +}; + +// Real webpack and rspack builds run a loader producing a source map before the Datadog plugins, +// which unplugin needs to chain their source maps. +const SOURCE_MAP_LOADER = path.resolve( + __dirname, + '../../../tests/src/_jest/fixtures/circular_project/identity-source-map-loader.cjs', +); +const sourceMapLoaderPlugin = { + apply(compiler: XpackCompiler) { + compiler.options.module.rules.unshift({ test: /\.js$/, loader: SOURCE_MAP_LOADER }); + }, +}; + +const PLUGIN_OVERRIDES: Partial = { + enableGit: false, + liveDebugger: {}, + metadata: { version: '1.0.0' }, + sourcemaps: { debugId: true }, +}; + +const createFixture = async (workingDir: string) => { + const writes = Object.entries(FIXTURE_FILES).map(([file, content]) => { + const filepath = path.resolve(workingDir, file); + return outputFile(filepath, content); + }); + await Promise.all(writes); + + // A workspace package linked into node_modules. + const linkPath = path.resolve(workingDir, 'app/node_modules/@fixture/lib'); + const linkTarget = path.resolve(workingDir, 'packages/lib'); + const linkDir = path.dirname(linkPath); + await mkdir(linkDir); + await fsp.symlink(linkTarget, linkPath, 'dir'); + + // The symbol table only needs the repository root, which the `.git` entry marks. + const gitDir = path.resolve(workingDir, '.git'); + await mkdir(gitDir); +}; + +// Build with the symbol table dump enabled, and return the dump. +const buildWithSymbolTable = async (dumpPath: string, build: () => Promise) => { + process.env[SYMBOL_TABLE_PATH_ENV] = dumpPath; + try { + const errors = await build(); + expect(errors).toEqual([]); + } finally { + delete process.env[SYMBOL_TABLE_PATH_ENV]; + } + + const dump = await readFile(dumpPath); + const table: SymbolTablePayload = JSON.parse(dump); + return table; +}; + +const flattenScopes = ( + scopes: SymDbScope[] = [], + parentName: string | undefined = undefined, +): TableFunction[] => + scopes.flatMap((scope) => { + if (scope.scope_type === 'MODULE') { + return flattenScopes(scope.scopes); + } + + const tableFunction: TableFunction = { + functionId: scope.language_specifics?.function_id ?? '', + name: scope.name, + sourceFile: scope.source_file, + startLine: scope.start_line, + endLine: scope.end_line, + parentName, + }; + const children = flattenScopes(scope.scopes, scope.name); + return [tableFunction, ...children]; + }); + +const getOutputFunctionIds = async (outDir: string) => { + const files = await glob('**/*.{js,mjs}', { cwd: outDir, absolute: true }); + const contents = files.map((file) => readFileSync(file)); + const functionIds = contents.flatMap((content) => { + const matches = content.matchAll(PROBE_CALL_RX); + return [...matches].map(([, , functionId]) => functionId); + }); + return [...new Set(functionIds)].sort(); +}; + +describe('Live Debugger symbol table', () => { + const tables: Partial> = {}; + const outputFunctionIds: Partial> = {}; + const outputContents: Partial> = {}; + let workingDir: string; + let appDir: string; + + beforeAll(async () => { + const seed = `${Math.abs(jest.getSeed())}.symbol-table`; + workingDir = getTempWorkingDir(seed); + appDir = path.resolve(workingDir, APP_DIR); + await createFixture(workingDir); + const buildRoot = appDir; + + // Run the builds sequentially to ease the resources usage. + for (const { name: bundlerName, run } of BUNDLERS) { + const isXpack = XPACK_BUNDLERS.includes(bundlerName); + const entry = { main: isXpack ? './src/main-xpack.js' : './src/main.js' }; + const plugins = isXpack ? [sourceMapLoaderPlugin] : []; + const dumpPath = path.resolve(workingDir, `symbol-table.${bundlerName}.json`); + const outDir = getOutDir(appDir, bundlerName); + + // eslint-disable-next-line no-await-in-loop + tables[bundlerName] = await buildWithSymbolTable(dumpPath, async () => { + const config = getBundlerConfig(bundlerName, buildRoot, PLUGIN_OVERRIDES, { + entry, + plugins, + }); + const { errors } = await run(buildRoot, config); + return errors; + }); + // eslint-disable-next-line no-await-in-loop + outputFunctionIds[bundlerName] = await getOutputFunctionIds(outDir); + // eslint-disable-next-line no-await-in-loop + const outputFiles = await glob('**/*', { cwd: outDir, absolute: true, nodir: true }); + outputContents[bundlerName] = outputFiles.map((file) => readFileSync(file)); + } + }, 200000); + + afterAll(async () => { + if (!process.env.NO_CLEANUP) { + await rm(workingDir); + } + }); + + describe.each(BUNDLERS)('$name | $version', ({ name: bundlerName }) => { + const getTable = () => tables[bundlerName]; + const getFunctions = () => flattenScopes(getTable()?.scopes); + const getFunction = (sourceFile: string, name: string) => + getFunctions().find( + (tableFunction) => + tableFunction.sourceFile === sourceFile && tableFunction.name === name, + ); + + test('Should list exactly the function IDs present in the output.', () => { + const functionIds = getFunctions().map((tableFunction) => tableFunction.functionId); + const uniqueFunctionIds = [...new Set(functionIds)].sort(); + + expect(outputFunctionIds[bundlerName]?.length).toBeGreaterThan(0); + expect(uniqueFunctionIds).toEqual(outputFunctionIds[bundlerName]); + }); + + test('Should describe the build.', () => { + const table = getTable(); + expect(table?.language).toBe('browser'); + expect(table?.version).toBe('1.0.0'); + expect(table?.chunk_debug_ids?.length).toBeGreaterThan(0); + + for (const moduleScope of table?.scopes ?? []) { + expect(moduleScope.scope_type).toBe('MODULE'); + expect(moduleScope.start_line).toBeUndefined(); + expect(moduleScope.language_specifics?.chunk_debug_ids?.length).toBeGreaterThan(0); + } + }); + + test('Should give paths relative to the repository root, with original line ranges.', () => { + expect(getFunction('app/src/math.js', 'add')).toMatchObject({ + startLine: 1, + endLine: 6, + parentName: undefined, + }); + expect(getFunction('app/src/math.js', 'double')).toMatchObject({ + startLine: 2, + endLine: 4, + parentName: 'add', + }); + expect(getFunction('app/src/main.js', '')).toMatchObject({ + startLine: 5, + endLine: 7, + }); + }); + + test('Should resolve symlinked workspace packages to their repository path.', () => { + expect(getFunction('packages/lib/index.js', 'libGreet')).toMatchObject({ + startLine: 1, + endLine: 3, + }); + }); + + test('Should resolve Yarn PnP virtual paths to their repository path.', () => { + expect(getFunction('packages/peer/index.js', 'peerHello')).toMatchObject({ + startLine: 1, + endLine: 3, + }); + + const sourceFiles = getFunctions().map((tableFunction) => tableFunction.sourceFile); + const virtualSourceFiles = sourceFiles.filter((sourceFile) => + sourceFile.includes('__virtual__'), + ); + expect(virtualSourceFiles).toEqual([]); + }); + + test('Should not write the symbol table to the output directory.', () => { + const contents = outputContents[bundlerName] ?? []; + expect(contents.length).toBeGreaterThan(0); + + const tableContents = contents.filter((content) => content.includes('"scope_type"')); + expect(tableContents).toEqual([]); + }); + + if (XPACK_BUNDLERS.includes(bundlerName)) { + test('Should strip the query string of module IDs.', () => { + expect(getFunction('app/src/component.js', 'render')).toMatchObject({ + startLine: 1, + endLine: 3, + }); + }); + } + }); + + describe('webpack | warm filesystem cache', () => { + if (!BUNDLERS.some(({ name }) => name === 'webpack')) { + test.skip('webpack is not part of this run.', () => {}); + return; + } + + let coldTable: SymbolTablePayload | undefined; + let warmTable: SymbolTablePayload | undefined; + let cacheDirectory: string; + + beforeAll(async () => { + cacheDirectory = path.resolve(workingDir, 'webpack-cache'); + const outDir = getOutDir(appDir, 'webpack-cache'); + + const build = async () => { + const plugin = datadogWebpackPlugin({ + ...defaultPluginOptions, + ...PLUGIN_OVERRIDES, + }); + const config = configWebpack({ + workingDir: appDir, + outDir, + entry: { main: './src/main-xpack.js' }, + plugins: [sourceMapLoaderPlugin, plugin], + }); + config.cache = { type: 'filesystem', cacheDirectory }; + const { errors } = await buildWithWebpack(config); + return errors; + }; + + const coldDumpPath = path.resolve(workingDir, 'symbol-table.webpack-cold.json'); + coldTable = await buildWithSymbolTable(coldDumpPath, build); + const warmDumpPath = path.resolve(workingDir, 'symbol-table.webpack-warm.json'); + warmTable = await buildWithSymbolTable(warmDumpPath, build); + }, 200000); + + test('Should list the same functions when modules come from the cache.', () => { + expect(existsSync(cacheDirectory)).toBe(true); + expect(flattenScopes(coldTable?.scopes).length).toBeGreaterThan(0); + expect(warmTable).toEqual(coldTable); + }); + }); +}); diff --git a/packages/plugins/live-debugger/src/transform/functionId.test.ts b/packages/plugins/live-debugger/src/transform/functionId.test.ts index e48ac2c7f..84ddce0d9 100644 --- a/packages/plugins/live-debugger/src/transform/functionId.test.ts +++ b/packages/plugins/live-debugger/src/transform/functionId.test.ts @@ -7,7 +7,7 @@ import type * as t from '@babel/types'; import type { BabelPath } from './babel-path.types'; import { resolveCjsDefaultExport } from './cjs-interop'; -import { generateFunctionId, getFunctionName } from './functionId'; +import { generateFunctionId, getFunctionName, parseFunctionId } from './functionId'; // eslint-disable-next-line @typescript-eslint/no-var-requires, global-require const babelTypes = require('@babel/types') as typeof import('@babel/types'); @@ -312,3 +312,37 @@ describe('generateFunctionId', () => { }); }); }); + +describe('parseFunctionId', () => { + const cases = [ + { + description: 'split a named function ID', + input: 'src/utils.js;add', + expected: { modulePath: 'src/utils.js', name: 'add' }, + }, + { + description: 'keep separators that are part of the function name', + input: 'src/utils.js;get;user', + expected: { modulePath: 'src/utils.js', name: 'get;user' }, + }, + { + description: 'drop the location of anonymous functions', + input: 'src/widget.js;@4:16:0', + expected: { modulePath: 'src/widget.js', name: '' }, + }, + { + description: 'keep the query string of the module path', + input: 'src/Cart.vue?vue&type=script;total', + expected: { modulePath: 'src/Cart.vue?vue&type=script', name: 'total' }, + }, + { + description: 'reject an ID without a module path', + input: ';add', + expected: undefined, + }, + ]; + + test.each(cases)('Should $description.', ({ input, expected }) => { + expect(parseFunctionId(input)).toEqual(expected); + }); +}); diff --git a/packages/plugins/live-debugger/src/transform/functionId.ts b/packages/plugins/live-debugger/src/transform/functionId.ts index 16f2def97..f5aaa7a9f 100644 --- a/packages/plugins/live-debugger/src/transform/functionId.ts +++ b/packages/plugins/live-debugger/src/transform/functionId.ts @@ -5,8 +5,34 @@ import type * as t from '@babel/types'; import path from 'path'; +import { ANONYMOUS_FUNCTION_NAME, FUNCTION_ID_SEPARATOR } from '../constants'; + import type { BabelPath, BabelTypesModule } from './babel-path.types'; +export type ParsedFunctionId = { + // The module ID relative to the build root, including any query string. + modulePath: string; + // The function name, without the location suffix of anonymous functions. + name: string; +}; + +/** + * Split a function ID generated by `generateFunctionId` into its module path and function name. + * Names can contain the separator (e.g. string keys), so the ID is split at its first occurrence. + */ +export function parseFunctionId(functionId: string): ParsedFunctionId | undefined { + const separatorIndex = functionId.indexOf(FUNCTION_ID_SEPARATOR); + if (separatorIndex <= 0) { + return undefined; + } + + const modulePath = functionId.slice(0, separatorIndex); + const functionName = functionId.slice(separatorIndex + FUNCTION_ID_SEPARATOR.length); + const isAnonymous = functionName.startsWith(`${ANONYMOUS_FUNCTION_NAME}@`); + + return { modulePath, name: isAnonymous ? ANONYMOUS_FUNCTION_NAME : functionName }; +} + /** * Generate a stable, unique function ID * Format (POC): ; @@ -26,12 +52,12 @@ export function generateFunctionId( const functionName = getFunctionName(functionPath, typesModule); if (functionName) { - return `${relativePath};${functionName}`; + return `${relativePath}${FUNCTION_ID_SEPARATOR}${functionName}`; } const locationSuffix = getFunctionLocation(functionPath); - return `${relativePath};@${locationSuffix}:${anonymousSiblingIndex}`; + return `${relativePath}${FUNCTION_ID_SEPARATOR}${ANONYMOUS_FUNCTION_NAME}@${locationSuffix}:${anonymousSiblingIndex}`; } /** diff --git a/packages/plugins/live-debugger/src/transform/index.ts b/packages/plugins/live-debugger/src/transform/index.ts index c503f1d33..6694bb0c4 100644 --- a/packages/plugins/live-debugger/src/transform/index.ts +++ b/packages/plugins/live-debugger/src/transform/index.ts @@ -9,7 +9,7 @@ import { LIVE_DEBUGGER_PROBES_FUNCTION } from '@dd/core/constants'; import type MagicStringType from 'magic-string'; import type { SourceMap } from 'magic-string'; -import { SKIP_INSTRUMENTATION_COMMENT } from '../constants'; +import { LIVE_DEBUGGER_THROW_FUNCTION, SKIP_INSTRUMENTATION_COMMENT } from '../constants'; import type { DecoratorSyntax, FunctionKind } from '../types'; import type { BabelPath, BabelTypesModule } from './babel-path.types'; @@ -496,7 +496,7 @@ function injectInstrumentation(s: MagicStringType, code: string, target: Functio // every exit hook pairs with its own entry state, not a concurrent call's. const probeDecl = `let ${probeVarName} = ${LIVE_DEBUGGER_PROBES_FUNCTION}('${escapedFunctionId}');`; const entryCall = `if (${probeVarName}) ${probeVarName} = $dd_entry(${probeVarName}, ${receiverArg}${argsArg});`; - const catchBlock = `catch(e) { if (${probeVarName}) $dd_throw(${probeVarName}, e, ${receiverArg}${argsArg}); throw e; }`; + const catchBlock = `catch(e) { if (${probeVarName}) ${LIVE_DEBUGGER_THROW_FUNCTION}(${probeVarName}, e, ${receiverArg}${argsArg}); throw e; }`; if (isExpressionBody) { // Arrow expression body: (a) => expr diff --git a/packages/published/esbuild-plugin/package.json b/packages/published/esbuild-plugin/package.json index 6023eb0b9..150c8ba0a 100644 --- a/packages/published/esbuild-plugin/package.json +++ b/packages/published/esbuild-plugin/package.json @@ -52,6 +52,7 @@ "dependencies": { "@datadog/js-instrumentation-wasm": "1.1.0", "@jridgewell/remapping": "2.3.5", + "@jridgewell/trace-mapping": "0.3.31", "async-retry": "1.3.3", "chalk": "2.3.1", "esbuild": "0.25.8", diff --git a/packages/published/rollup-plugin/package.json b/packages/published/rollup-plugin/package.json index 4e0a518aa..4425de8bf 100644 --- a/packages/published/rollup-plugin/package.json +++ b/packages/published/rollup-plugin/package.json @@ -55,6 +55,7 @@ "dependencies": { "@datadog/js-instrumentation-wasm": "1.1.0", "@jridgewell/remapping": "2.3.5", + "@jridgewell/trace-mapping": "0.3.31", "async-retry": "1.3.3", "chalk": "2.3.1", "esbuild": "0.25.8", diff --git a/packages/published/rspack-plugin/package.json b/packages/published/rspack-plugin/package.json index 5ea679d1a..df8005532 100644 --- a/packages/published/rspack-plugin/package.json +++ b/packages/published/rspack-plugin/package.json @@ -52,6 +52,7 @@ "dependencies": { "@datadog/js-instrumentation-wasm": "1.1.0", "@jridgewell/remapping": "2.3.5", + "@jridgewell/trace-mapping": "0.3.31", "async-retry": "1.3.3", "chalk": "2.3.1", "esbuild": "0.25.8", diff --git a/packages/published/vite-plugin/package.json b/packages/published/vite-plugin/package.json index dffc0f42e..aebedad10 100644 --- a/packages/published/vite-plugin/package.json +++ b/packages/published/vite-plugin/package.json @@ -52,6 +52,7 @@ "dependencies": { "@datadog/js-instrumentation-wasm": "1.1.0", "@jridgewell/remapping": "2.3.5", + "@jridgewell/trace-mapping": "0.3.31", "async-retry": "1.3.3", "chalk": "2.3.1", "esbuild": "0.25.8", diff --git a/packages/published/webpack-plugin/package.json b/packages/published/webpack-plugin/package.json index e7bcc733f..dd8c83aa7 100644 --- a/packages/published/webpack-plugin/package.json +++ b/packages/published/webpack-plugin/package.json @@ -52,6 +52,7 @@ "dependencies": { "@datadog/js-instrumentation-wasm": "1.1.0", "@jridgewell/remapping": "2.3.5", + "@jridgewell/trace-mapping": "0.3.31", "async-retry": "1.3.3", "chalk": "2.3.1", "esbuild": "0.25.8", diff --git a/yarn.lock b/yarn.lock index f16217b08..95e8e4df3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1689,6 +1689,7 @@ __metadata: "@dd/factory": "workspace:*" "@dd/tools": "workspace:*" "@jridgewell/remapping": "npm:2.3.5" + "@jridgewell/trace-mapping": "npm:0.3.31" "@rollup/plugin-babel": "npm:6.0.4" "@rollup/plugin-commonjs": "npm:28.0.1" "@rollup/plugin-esm-shim": "npm:0.1.8" @@ -1748,6 +1749,7 @@ __metadata: "@dd/factory": "workspace:*" "@dd/tools": "workspace:*" "@jridgewell/remapping": "npm:2.3.5" + "@jridgewell/trace-mapping": "npm:0.3.31" "@rollup/plugin-babel": "npm:6.0.4" "@rollup/plugin-commonjs": "npm:28.0.1" "@rollup/plugin-esm-shim": "npm:0.1.8" @@ -1800,6 +1802,7 @@ __metadata: "@dd/factory": "workspace:*" "@dd/tools": "workspace:*" "@jridgewell/remapping": "npm:2.3.5" + "@jridgewell/trace-mapping": "npm:0.3.31" "@rollup/plugin-babel": "npm:6.0.4" "@rollup/plugin-commonjs": "npm:28.0.1" "@rollup/plugin-esm-shim": "npm:0.1.8" @@ -1855,6 +1858,7 @@ __metadata: "@dd/factory": "workspace:*" "@dd/tools": "workspace:*" "@jridgewell/remapping": "npm:2.3.5" + "@jridgewell/trace-mapping": "npm:0.3.31" "@rollup/plugin-babel": "npm:6.0.4" "@rollup/plugin-commonjs": "npm:28.0.1" "@rollup/plugin-esm-shim": "npm:0.1.8" @@ -1907,6 +1911,7 @@ __metadata: "@dd/factory": "workspace:*" "@dd/tools": "workspace:*" "@jridgewell/remapping": "npm:2.3.5" + "@jridgewell/trace-mapping": "npm:0.3.31" "@rollup/plugin-babel": "npm:6.0.4" "@rollup/plugin-commonjs": "npm:28.0.1" "@rollup/plugin-esm-shim": "npm:0.1.8" From d77dd2c7e105fa026b1b4c755b24ad0b89b63db2 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 15 Oct 2026 16:12:17 -0400 Subject: [PATCH 2/5] Find the end line of functions whose closing code isn't mapped On a web-ui production build, 45% of the symbol table's functions had no line range. The transform appends the closing code of an arrow function's expression body without a source map segment, and minifiers leave some rewritten tokens unmapped, so looking up the `$dd_throw` call alone often found nothing. Use the last mapped position of the same source between the two markers instead. For block bodies that's still the closing brace. For expression bodies, it's the last mapped token of the expression. On the same web-ui build, functions without a line range drop to 1.8%. The integration test fixture now has a multi-line arrow function with an expression body, and a minified Rspack production build. Environment: Datadog workspace Co-Authored-By: Claude Opus 5.5 --- .../plugins/live-debugger/CONTRIBUTING.md | 4 +- .../live-debugger/src/symbol-table/locate.ts | 83 ++++++++++++++++--- .../src/symbolTable.integration.test.ts | 74 ++++++++++++++++- 3 files changed, 147 insertions(+), 14 deletions(-) diff --git a/packages/plugins/live-debugger/CONTRIBUTING.md b/packages/plugins/live-debugger/CONTRIBUTING.md index e962daf52..ad97dac42 100644 --- a/packages/plugins/live-debugger/CONTRIBUTING.md +++ b/packages/plugins/live-debugger/CONTRIBUTING.md @@ -58,7 +58,7 @@ How it works: - After a one-off build (not in watch mode or on a dev server), the plugin scans the text of every emitted JavaScript chunk, without parsing it. It doesn't use the transform hook, which doesn't run for modules restored from a persistent cache and doesn't know which functions tree-shaking removes. - Every instrumented function has exactly one `$dd_probes('')` call at the start of its body and one `$dd_throw(` call at its end. The scan pairs them like brackets, which also gives each function's parent. -- The transform anchors both calls' source map segments to the original body's opening and closing brace, so looking them up in the chunk's final source map gives the line range. +- The transform anchors the `$dd_probes` call to the original body's opening brace, which gives the start line through the chunk's final source map. The end line is the last mapped position up to the `$dd_throw` call: for block bodies that's the closing brace. The code appended after an arrow function's expression body has no mapping, and minifiers leave some tokens unmapped, so the closest mapped token before it is used instead. - The path and display name come from the function ID. Query strings are removed, Yarn PnP virtual paths and symlinks are resolved, and the path is made relative to the git root. Without a git repository, the table is skipped. - Each chunk's debug ID is read from the runtime bootstrap, when debug IDs are enabled, to record which chunks contain each file. @@ -67,6 +67,8 @@ When changing instrumentation output, keep exactly one `$dd_probes` call at the Known limitations: - esbuild doesn't tell plugins whether it's watching, so its rebuilds always regenerate the table. +- In minified builds, the end line of an arrow function's expression body is the last line the minifier keeps mapped, which can be before the real end. +- Helpers that compilers inject, such as `_define_property`, have no original source, so they have no line range. - On webpack and Rspack, files that reach the plugin without a source map from an earlier loader get shifted line numbers ([DEBUG-6343](https://datadoghq.atlassian.net/browse/DEBUG-6343)). ## Consumer build canary diff --git a/packages/plugins/live-debugger/src/symbol-table/locate.ts b/packages/plugins/live-debugger/src/symbol-table/locate.ts index b27026b4a..5faea7a28 100644 --- a/packages/plugins/live-debugger/src/symbol-table/locate.ts +++ b/packages/plugins/live-debugger/src/symbol-table/locate.ts @@ -3,7 +3,8 @@ // Copyright 2019-Present Datadog, Inc. import { existsSync, readFile } from '@dd/core/helpers/fs'; -import { originalPositionFor, TraceMap } from '@jridgewell/trace-mapping'; +import { decodedMappings, TraceMap, traceSegment } from '@jridgewell/trace-mapping'; +import type { SourceMapSegment } from '@jridgewell/trace-mapping'; import path from 'path'; import type { ChunkScan } from './scan'; @@ -78,21 +79,73 @@ const createOffsetConverter = (code: string) => { }; }; -const getOriginalPosition = ( +type MappedPosition = SourcePosition & { + sourceIndex: number; +}; + +// The original position of the closest segment at or before a generated position. +const getMappedPosition = ( traceMap: TraceMap, generated: SourcePosition, -): SourcePosition | undefined => { - const original = originalPositionFor(traceMap, generated); - if (original.line === null || original.column === null) { +): MappedPosition | undefined => { + const segment = traceSegment(traceMap, generated.line - 1, generated.column); + if (!segment || segment.length === 1) { return undefined; } - return { line: original.line, column: original.column }; + return { line: segment[2] + 1, column: segment[3], sourceIndex: segment[1] }; +}; + +// Index of the first segment at or after `column`. +const findSegmentIndex = (segments: readonly SourceMapSegment[], column: number): number => { + let low = 0; + let high = segments.length; + while (low < high) { + const middle = Math.floor((low + high) / 2); + if (segments[middle][0] < column) { + low = middle + 1; + } else { + high = middle; + } + } + return low; +}; + +/** + * The original position of the last mapped segment of `sourceIndex` between two generated + * positions, both included. Some tokens have no mapping: the code the transform appends after + * an arrow function's expression body, and tokens that minifiers rewrite. The closest mapped token + * before the `$dd_throw` call is then the best estimate of where the function ends. + */ +const getLastMappedPosition = ( + traceMap: TraceMap, + from: SourcePosition, + to: SourcePosition, + sourceIndex: number, +): SourcePosition | undefined => { + const lines = decodedMappings(traceMap); + + for (let line = to.line; line >= from.line; line--) { + const segments = lines[line - 1] ?? []; + const lastIndex = + line === to.line ? findSegmentIndex(segments, to.column + 1) - 1 : segments.length - 1; + const firstColumn = line === from.line ? from.column : 0; + + for (let index = lastIndex; index >= 0 && segments[index][0] >= firstColumn; index--) { + const segment = segments[index]; + if (segment.length !== 1 && segment[1] === sourceIndex) { + return { line: segment[2] + 1, column: segment[3] }; + } + } + } + + return undefined; }; /** * Map the scanned functions to their original positions. The transform anchors the `$dd_probes` - * call to the body's opening brace and the `$dd_throw` call to its closing brace. + * call to the body's opening brace, and the `$dd_throw` call to its closing brace when the body + * is a block. */ export const locateFunctions = ( code: string, @@ -111,9 +164,19 @@ export const locateFunctions = ( const generatedStart = toPosition(probeOffset); const generatedEnd = toPosition(throwOffset); - const start = getOriginalPosition(traceMap, generatedStart); - const end = getOriginalPosition(traceMap, generatedEnd); - if (!start || !end || end.line < start.line) { + const mappedStart = getMappedPosition(traceMap, generatedStart); + if (!mappedStart) { + return { functionId, parentIndex, start: undefined, end: undefined }; + } + + const start = { line: mappedStart.line, column: mappedStart.column }; + const end = getLastMappedPosition( + traceMap, + generatedStart, + generatedEnd, + mappedStart.sourceIndex, + ); + if (!end || end.line < start.line) { return { functionId, parentIndex, start: undefined, end: undefined }; } diff --git a/packages/plugins/live-debugger/src/symbolTable.integration.test.ts b/packages/plugins/live-debugger/src/symbolTable.integration.test.ts index 9f9334ec9..d1eb48702 100644 --- a/packages/plugins/live-debugger/src/symbolTable.integration.test.ts +++ b/packages/plugins/live-debugger/src/symbolTable.integration.test.ts @@ -2,13 +2,14 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. +import { datadogRspackPlugin } from '@datadog/rspack-plugin'; import { datadogWebpackPlugin } from '@datadog/webpack-plugin'; import { existsSync, mkdir, outputFile, readFile, readFileSync, rm } from '@dd/core/helpers/fs'; import type { BundlerName, Options } from '@dd/core/types'; import { getOutDir, getTempWorkingDir } from '@dd/tests/_jest/helpers/env'; import { defaultPluginOptions } from '@dd/tests/_jest/helpers/mocks'; import { BUNDLERS, getBundlerConfig } from '@dd/tests/_jest/helpers/runBundlers'; -import { buildWithWebpack, configWebpack } from '@dd/tools/bundlers'; +import { buildWithRspack, buildWithWebpack, configRspack, configWebpack } from '@dd/tools/bundlers'; import fsp from 'fs/promises'; import { glob } from 'glob'; import path from 'path'; @@ -41,13 +42,13 @@ const PEER_SOURCE = `export function peerHello() { const FIXTURE_FILES: Record = { 'app/src/main.js': `import { libGreet } from '@fixture/lib'; import { peerHello } from '../../${VIRTUAL_PEER_PATH}'; -import { add } from './math.js'; +import { add, triple } from './math.js'; export const values = [1, 2].map(function (value) { return add(value, 1); }); -console.log(values, libGreet(), peerHello()); +console.log(values, libGreet(), peerHello(), triple(2)); `, // webpack and rspack also cover a module ID with a query string. 'app/src/main-xpack.js': `import { render } from './component.js?query'; @@ -61,6 +62,10 @@ console.log(render(values.join(','))); }; return double(a) + b; } + +export const triple = (value) => + value * + 3; `, 'app/src/component.js': `export function render(label) { return '

' + label + '

'; @@ -256,6 +261,13 @@ describe('Live Debugger symbol table', () => { }); }); + test('Should give arrow functions with an expression body their line range.', () => { + expect(getFunction('app/src/math.js', 'triple')).toMatchObject({ + startLine: 9, + endLine: 10, + }); + }); + test('Should resolve symlinked workspace packages to their repository path.', () => { expect(getFunction('packages/lib/index.js', 'libGreet')).toMatchObject({ startLine: 1, @@ -336,4 +348,60 @@ describe('Live Debugger symbol table', () => { expect(warmTable).toEqual(coldTable); }); }); + // Minifiers rewrite the injected code and don't map every token, like web-ui's production builds. + describe('rspack | minified production build', () => { + if (!BUNDLERS.some(({ name }) => name === 'rspack')) { + test.skip('rspack is not part of this run.', () => {}); + return; + } + + let minifiedTable: SymbolTablePayload | undefined; + + beforeAll(async () => { + const outDir = getOutDir(appDir, 'rspack-production'); + const dumpPath = path.resolve(workingDir, 'symbol-table.rspack-production.json'); + + minifiedTable = await buildWithSymbolTable(dumpPath, async () => { + const plugin = datadogRspackPlugin({ + ...defaultPluginOptions, + ...PLUGIN_OVERRIDES, + rum: { privacy: {} }, + }); + const config = configRspack({ + workingDir: appDir, + outDir, + entry: { main: './src/main-xpack.js' }, + plugins: [sourceMapLoaderPlugin, plugin], + }); + const { errors } = await buildWithRspack({ + ...config, + mode: 'production', + optimization: { ...config.optimization, minimize: true }, + }); + return errors; + }); + }, 200000); + + test('Should give every function its original line range.', () => { + const functions = flattenScopes(minifiedTable?.scopes); + const withoutLines = functions.filter((tableFunction) => !tableFunction.endLine); + + expect(functions.length).toBeGreaterThan(0); + expect(withoutLines).toEqual([]); + expect(functions).toContainEqual( + expect.objectContaining({ + sourceFile: 'app/src/math.js', + name: 'add', + startLine: 1, + endLine: 6, + }), + ); + + // The end of an expression body is the last token the minifier keeps mapped. + const triple = functions.find((tableFunction) => tableFunction.name === 'triple'); + expect(triple?.startLine).toBe(9); + expect(triple?.endLine).toBeGreaterThanOrEqual(9); + expect(triple?.endLine).toBeLessThanOrEqual(10); + }); + }); }); From a2d67f99884267eccbb759e430b048fef224eef1 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 15 Oct 2026 16:12:20 -0400 Subject: [PATCH 3/5] Recover more Live Debugger function line ranges in minified builds On a web-ui production build, 1.8% of the symbol table's functions still had no line range after the previous fix: - Minifiers can also leave the `$dd_probes` call unmapped. Fall back to the first mapped token between the two markers for the start, like the end already falls back to the last one. - A module bundled into many chunks can be mapped in some chunks and not in others, which left a copy without a line range next to the mapped ones. Drop such copies, and attach their nested functions to their parent. Functions without a line range drop to 0.98% on the same build, mostly helpers that compilers inject, such as `_define_property`, which have no original source. Unit tests cover the lookups with hand-built source maps. Environment: Datadog workspace Co-Authored-By: Claude Opus 5.5 --- .../src/symbol-table/locate.test.ts | 95 +++++++++++++++++++ .../live-debugger/src/symbol-table/locate.ts | 36 ++++++- .../src/symbol-table/table.test.ts | 25 +++++ .../live-debugger/src/symbol-table/table.ts | 34 ++++++- 4 files changed, 186 insertions(+), 4 deletions(-) create mode 100644 packages/plugins/live-debugger/src/symbol-table/locate.test.ts diff --git a/packages/plugins/live-debugger/src/symbol-table/locate.test.ts b/packages/plugins/live-debugger/src/symbol-table/locate.test.ts new file mode 100644 index 000000000..5dd621146 --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/locate.test.ts @@ -0,0 +1,95 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import type { SourceMapSegment } from '@jridgewell/trace-mapping'; +import { TraceMap } from '@jridgewell/trace-mapping'; + +import { locateFunctions } from './locate'; +import { scanChunk } from './scan'; + +// A minified function: `body` stands for the original code of its body. +const CODE = `let p=$dd_probes('a.js;fn');try{body}catch(e){throw p&&$dd_throw(p,e),e}`; +const PROBE = CODE.indexOf('$dd_probes'); +const BODY = CODE.indexOf('body'); +const THROW = CODE.indexOf('$dd_throw'); +const CATCH = CODE.indexOf('catch'); + +// Segments map a generated column to `[source, original line (0-based), original column]`, +// or to nothing. +type Mapping = [column: number, source?: number, line?: number]; + +const getTraceMap = (mappings: Mapping[]) => { + const segments = mappings.map(([column, source, line]): SourceMapSegment => { + if (source === undefined || line === undefined) { + return [column]; + } + return [column, source, line, 0]; + }); + + return new TraceMap({ + version: 3, + sources: ['a.js', 'b.js'], + names: [], + mappings: [segments], + }); +}; + +describe('locateFunctions', () => { + const cases = [ + { + description: 'use the original positions of both markers', + mappings: [ + [PROBE, 0, 0], + [BODY, 0, 1], + [THROW, 0, 4], + ], + expected: { start: 1, end: 5 }, + }, + { + description: 'end at the last mapped token when the closing marker is not mapped', + mappings: [[PROBE, 0, 0], [BODY, 0, 2], [THROW - 8], [THROW]], + expected: { start: 1, end: 3 }, + }, + { + description: 'start at the first mapped token when the opening marker is not mapped', + mappings: [[0], [PROBE], [BODY, 0, 6], [THROW, 0, 8]], + expected: { start: 7, end: 9 }, + }, + { + description: 'ignore tokens of another source when looking for the end', + mappings: [[PROBE, 0, 0], [BODY, 0, 2], [CATCH, 1, 40], [THROW]], + expected: { start: 1, end: 3 }, + }, + { + description: 'leave out positions when nothing in the function is mapped', + mappings: [[0], [PROBE], [THROW]], + expected: undefined, + }, + ] satisfies { description: string; mappings: Mapping[]; expected: unknown }[]; + + test.each(cases)('Should $description.', ({ mappings, expected }) => { + const traceMap = getTraceMap(mappings); + const scan = scanChunk(CODE); + const [located] = locateFunctions(CODE, scan, traceMap); + const lines = located.start && + located.end && { + start: located.start.line, + end: located.end.line, + }; + + expect(lines).toEqual(expected); + }); + + test('Should leave out positions without a source map.', () => { + const scan = scanChunk(CODE); + const [located] = locateFunctions(CODE, scan, undefined); + + expect(located).toEqual({ + functionId: 'a.js;fn', + parentIndex: undefined, + start: undefined, + end: undefined, + }); + }); +}); diff --git a/packages/plugins/live-debugger/src/symbol-table/locate.ts b/packages/plugins/live-debugger/src/symbol-table/locate.ts index 5faea7a28..1efe4c99a 100644 --- a/packages/plugins/live-debugger/src/symbol-table/locate.ts +++ b/packages/plugins/live-debugger/src/symbol-table/locate.ts @@ -83,7 +83,7 @@ type MappedPosition = SourcePosition & { sourceIndex: number; }; -// The original position of the closest segment at or before a generated position. +// The original position of the closest segment at or before a generated position, if it's mapped. const getMappedPosition = ( traceMap: TraceMap, generated: SourcePosition, @@ -111,6 +111,34 @@ const findSegmentIndex = (segments: readonly SourceMapSegment[], column: number) return low; }; +// The original position of the first mapped segment between two generated positions, both included. +const getFirstMappedPosition = ( + traceMap: TraceMap, + from: SourcePosition, + to: SourcePosition, +): MappedPosition | undefined => { + const lines = decodedMappings(traceMap); + + for (let line = from.line; line <= to.line; line++) { + const segments = lines[line - 1] ?? []; + const firstIndex = line === from.line ? findSegmentIndex(segments, from.column) : 0; + const lastColumn = line === to.line ? to.column : Infinity; + + for ( + let index = firstIndex; + index < segments.length && segments[index][0] <= lastColumn; + index++ + ) { + const segment = segments[index]; + if (segment.length !== 1) { + return { line: segment[2] + 1, column: segment[3], sourceIndex: segment[1] }; + } + } + } + + return undefined; +}; + /** * The original position of the last mapped segment of `sourceIndex` between two generated * positions, both included. Some tokens have no mapping: the code the transform appends after @@ -145,7 +173,7 @@ const getLastMappedPosition = ( /** * Map the scanned functions to their original positions. The transform anchors the `$dd_probes` * call to the body's opening brace, and the `$dd_throw` call to its closing brace when the body - * is a block. + * is a block. When a marker isn't mapped, the closest mapped token inside the function is used. */ export const locateFunctions = ( code: string, @@ -164,7 +192,9 @@ export const locateFunctions = ( const generatedStart = toPosition(probeOffset); const generatedEnd = toPosition(throwOffset); - const mappedStart = getMappedPosition(traceMap, generatedStart); + const mappedStart = + getMappedPosition(traceMap, generatedStart) ?? + getFirstMappedPosition(traceMap, generatedStart, generatedEnd); if (!mappedStart) { return { functionId, parentIndex, start: undefined, end: undefined }; } diff --git a/packages/plugins/live-debugger/src/symbol-table/table.test.ts b/packages/plugins/live-debugger/src/symbol-table/table.test.ts index 37bff5f9f..4c0bda07c 100644 --- a/packages/plugins/live-debugger/src/symbol-table/table.test.ts +++ b/packages/plugins/live-debugger/src/symbol-table/table.test.ts @@ -78,6 +78,31 @@ describe('buildSymbolTable', () => { expect(invalidFunctionIds).toEqual(expected.invalidFunctionIds ?? []); }); + test('Should drop copies without a line range when another copy has one.', () => { + const chunks = [ + getChunk(DEBUG_ID_A, [ + getFunction('src/a.js;outer', undefined), + getFunction('src/a.js;inner', [2, 4], 0), + ]), + getChunk(DEBUG_ID_B, [ + getFunction('src/a.js;outer', [1, 9]), + getFunction('src/a.js;unmapped', undefined), + ]), + ]; + const { table } = buildSymbolTable(chunks, resolveSourceFile); + const entries = table.entries.map(({ functionId, start, parentKey }) => ({ + functionId, + located: Boolean(start), + hasParent: Boolean(parentKey), + })); + + expect(entries).toEqual([ + { functionId: 'src/a.js;inner', located: true, hasParent: false }, + { functionId: 'src/a.js;outer', located: true, hasParent: false }, + { functionId: 'src/a.js;unmapped', located: false, hasParent: false }, + ]); + }); + test('Should record which chunks contain each file.', () => { const chunks = [ getChunk(DEBUG_ID_B, [getFunction('src/a.js;add', [1, 3])]), diff --git a/packages/plugins/live-debugger/src/symbol-table/table.ts b/packages/plugins/live-debugger/src/symbol-table/table.ts index 329930dde..9a48e8eda 100644 --- a/packages/plugins/live-debugger/src/symbol-table/table.ts +++ b/packages/plugins/live-debugger/src/symbol-table/table.ts @@ -32,6 +32,37 @@ const getEntryKey = (chunkFunction: ChunkFunction, sourceFile: string) => { return [chunkFunction.functionId, sourceFile, start, end].join('\0'); }; +// In some chunks a copy of a function can't be mapped, while other copies can. A copy without a +// line range then adds nothing, so drop it and attach its children to its own parent. +const removeUnlocatedCopies = (entries: SymbolTableEntry[]): SymbolTableEntry[] => { + const getFunctionKey = (entry: SymbolTableEntry) => `${entry.functionId}\0${entry.sourceFile}`; + const locatedEntries = entries.filter((entry) => entry.start); + const locatedFunctionKeys = new Set(locatedEntries.map(getFunctionKey)); + const removedParentKeys = new Map(); + + const keptEntries = entries.filter((entry) => { + const functionKey = getFunctionKey(entry); + if (entry.start || !locatedFunctionKeys.has(functionKey)) { + return true; + } + removedParentKeys.set(entry.key, entry.parentKey); + return false; + }); + + const getKeptParentKey = (parentKey: string | undefined) => { + let keptParentKey = parentKey; + while (keptParentKey !== undefined && removedParentKeys.has(keptParentKey)) { + keptParentKey = removedParentKeys.get(keptParentKey); + } + return keptParentKey; + }; + + return keptEntries.map((entry) => { + const parentKey = getKeptParentKey(entry.parentKey); + return { ...entry, parentKey }; + }); +}; + // Map the output's chunks to one entry per function, deduplicated across chunks. export const buildSymbolTable = ( chunks: ChunkFunctions[], @@ -117,8 +148,9 @@ export const buildSymbolTable = ( const sortedFileChunkDebugIds = new Map(sortedFileEntries); // Never publish incomplete chunk membership. + const allEntries = [...entries.values()]; const table: SymbolTable = { - entries: [...entries.values()], + entries: removeUnlocatedCopies(allEntries), chunkDebugIds: hasCompleteDebugIds ? sortedChunkDebugIds : undefined, fileChunkDebugIds: hasCompleteDebugIds ? sortedFileChunkDebugIds : undefined, }; From 48ad60de76b28fa09c40d5fe7c63e82221cf295a Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Fri, 16 Oct 2026 10:35:28 -0400 Subject: [PATCH 4/5] Word Live Debugger symbol logs and docs for users Customers see the build logs, where "symbol table" is an internal term. Call them Live Debugger symbols, like Datadog's documentation does for symbol uploads, and remove noise from the contributing guide. Environment: Datadog workspace Co-Authored-By: Claude Opus 5.5 --- packages/plugins/live-debugger/CONTRIBUTING.md | 6 +++--- packages/plugins/live-debugger/src/symbol-table/index.ts | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/plugins/live-debugger/CONTRIBUTING.md b/packages/plugins/live-debugger/CONTRIBUTING.md index ad97dac42..c8aa3c88f 100644 --- a/packages/plugins/live-debugger/CONTRIBUTING.md +++ b/packages/plugins/live-debugger/CONTRIBUTING.md @@ -44,7 +44,7 @@ When adding or changing `liveDebugger` configuration, update [`src/types.ts`](./ ## Symbol table -The plugin can list the instrumented functions that ship in a build, for upload to Live Debugger's symbol database (SymDB). Each entry has the function ID, a display name, the path relative to the git repository root, and the original line range. The table lives in [`src/symbol-table`](./src/symbol-table). +The plugin can list the instrumented functions that ship in a build, for upload to Datadog's Symbol Database. Each entry has the function ID, a display name, the path relative to the git repository root, and the original line range. The table lives in [`src/symbol-table`](./src/symbol-table). For now, the only consumer is an opt-in debug dump. Set `DD_LIVE_DEBUGGER_SYMBOL_TABLE_PATH` to a file path to generate the table and write it there: @@ -56,7 +56,7 @@ Without that variable, the plugin doesn't add any hook or build time. The dump i How it works: -- After a one-off build (not in watch mode or on a dev server), the plugin scans the text of every emitted JavaScript chunk, without parsing it. It doesn't use the transform hook, which doesn't run for modules restored from a persistent cache and doesn't know which functions tree-shaking removes. +- After a one-off build, outside watch mode and dev servers, the plugin scans the text of every emitted JavaScript chunk, without parsing it. It doesn't use the transform hook, which doesn't run for modules restored from a persistent cache and doesn't know which functions tree-shaking removes. - Every instrumented function has exactly one `$dd_probes('')` call at the start of its body and one `$dd_throw(` call at its end. The scan pairs them like brackets, which also gives each function's parent. - The transform anchors the `$dd_probes` call to the original body's opening brace, which gives the start line through the chunk's final source map. The end line is the last mapped position up to the `$dd_throw` call: for block bodies that's the closing brace. The code appended after an arrow function's expression body has no mapping, and minifiers leave some tokens unmapped, so the closest mapped token before it is used instead. - The path and display name come from the function ID. Query strings are removed, Yarn PnP virtual paths and symlinks are resolved, and the path is made relative to the git root. Without a git repository, the table is skipped. @@ -69,7 +69,7 @@ Known limitations: - esbuild doesn't tell plugins whether it's watching, so its rebuilds always regenerate the table. - In minified builds, the end line of an arrow function's expression body is the last line the minifier keeps mapped, which can be before the real end. - Helpers that compilers inject, such as `_define_property`, have no original source, so they have no line range. -- On webpack and Rspack, files that reach the plugin without a source map from an earlier loader get shifted line numbers ([DEBUG-6343](https://datadoghq.atlassian.net/browse/DEBUG-6343)). +- On webpack and Rspack, files that reach the plugin without a source map from an earlier loader get shifted line numbers. ## Consumer build canary diff --git a/packages/plugins/live-debugger/src/symbol-table/index.ts b/packages/plugins/live-debugger/src/symbol-table/index.ts index 6ce98358f..38dfff69d 100644 --- a/packages/plugins/live-debugger/src/symbol-table/index.ts +++ b/packages/plugins/live-debugger/src/symbol-table/index.ts @@ -18,7 +18,7 @@ import { serializeSymbolTable } from './serialize'; import { buildSymbolTable } from './table'; import type { ChunkFunction, ChunkFunctions, SymbolTable } from './types'; -const LOG_PREFIX = 'Live Debugger symbol table'; +const LOG_PREFIX = 'Live Debugger symbols'; const JS_OUTPUT_TYPES = new Set(['js', 'mjs', 'cjs']); type GenerateSymbolTableArgs = { @@ -47,7 +47,7 @@ export const generateSymbolTable = async ({ }: GenerateSymbolTableArgs): Promise => { const repositoryRoot = getRepositoryRoot(buildRoot); if (!repositoryRoot) { - log.warn(`Skipping the ${LOG_PREFIX}: the build root isn't in a git repository.`); + log.warn(`Skipping ${LOG_PREFIX}: the build root isn't in a git repository.`); return undefined; } @@ -126,7 +126,7 @@ export const writeSymbolTable = async ( const absoluteDumpPath = path.resolve(dumpPath); if (isInsideDirectory(absoluteDumpPath, outDir)) { log.warn( - `${LOG_PREFIX}: not writing it to "${absoluteDumpPath}", because that's inside the output directory.`, + `${LOG_PREFIX}: not writing them to "${absoluteDumpPath}", because that's inside the output directory.`, ); return; } @@ -166,7 +166,7 @@ export const getSymbolTableHooks = ( await writeSymbolTable(payload, dumpPath, context.bundler.outDir, log); } } catch (error) { - log.warn(`Could not generate the ${LOG_PREFIX}: ${error}`); + log.warn(`Could not collect ${LOG_PREFIX}: ${error}`); } finally { timer.end(); } From b58ce2f8115401f514d75ce129890bf176e86535 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Fri, 16 Oct 2026 10:41:07 -0400 Subject: [PATCH 5/5] Simplify a SymDB scope comment Environment: Datadog workspace Co-Authored-By: Claude Opus 5.5 --- packages/plugins/live-debugger/src/symbol-table/serialize.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/plugins/live-debugger/src/symbol-table/serialize.ts b/packages/plugins/live-debugger/src/symbol-table/serialize.ts index b2d1867fd..db6887e07 100644 --- a/packages/plugins/live-debugger/src/symbol-table/serialize.ts +++ b/packages/plugins/live-debugger/src/symbol-table/serialize.ts @@ -7,7 +7,7 @@ import type { SourcePosition, SymbolTable, SymbolTableEntry } from './types'; // SymDB language of browser symbol tables. export const SYMBOL_TABLE_LANGUAGE = 'browser'; -// A SymDB scope, in the format the backend SDKs upload. +// A SymDB scope. export type SymDbScope = { scope_type: 'MODULE' | 'FUNCTION'; name: string;