diff --git a/README.md b/README.md index bf95eac79..21b9eefba 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ To interact with Datadog directly from your builds. - [`gitRepositoryUrl`](#gitrepositoryurl) - [`logLevel`](#loglevel) - [`metadata.name`](#metadataname) + - [`metadata.service`](#metadataservice) - [`metadata.version`](#metadataversion) - [`sourcemaps`](#sourcemaps) - [Features](#features) @@ -105,6 +106,7 @@ Follow the specific documentation for each bundler: logLevel?: 'debug' | 'info' | 'warn' | 'error' | 'none', metadata?: { name?: string; + service?: string; version?: string; }; sourcemaps?: @@ -178,7 +180,7 @@ Follow the specific documentation for each bundler: } | { debugId?: false; - service: string; + service?: string; version?: string; }; }; @@ -344,6 +346,12 @@ Which level of log do you want to show. The name of the build.
This is used to identify the build in logs, metrics and spans. +### `metadata.service` +> default: `null` + +The service of the deployed application, as set in the Datadog browser SDKs.
+This is the canonical place to declare the service once. Plugins that need a service (Live Debugger's symbol table upload, RUM source code context, source map upload, etc.) read it from here unless they're given a more specific override. + ### `metadata.version` > default: `null` @@ -502,7 +510,7 @@ datadogWebpackPlugin({ } | { debugId?: false, - service: string, + service?: string, version?: string, }, } diff --git a/packages/core/src/helpers/plugins.test.ts b/packages/core/src/helpers/plugins.test.ts index 2aa8ed4e3..c78e56dcc 100644 --- a/packages/core/src/helpers/plugins.test.ts +++ b/packages/core/src/helpers/plugins.test.ts @@ -105,6 +105,26 @@ describe('plugins', () => { input: { ...pluginOptions, enableGit: false }, result: false, }, + { + description: 'true with Live Debugger and an API key', + input: { ...pluginOptions, liveDebugger: {} }, + result: true, + }, + { + description: 'false with Live Debugger and no API key', + input: { ...pluginOptions, auth: { site: 'datadoghq.com' }, liveDebugger: {} }, + result: false, + }, + { + description: 'false with Live Debugger disabled', + input: { ...pluginOptions, liveDebugger: { enable: false } }, + result: false, + }, + { + description: 'false if disabled globaly with Live Debugger', + input: { ...pluginOptions, enableGit: false, liveDebugger: {} }, + result: false, + }, { description: 'false if disabled globaly with sourcemaps', input: { diff --git a/packages/core/src/helpers/plugins.ts b/packages/core/src/helpers/plugins.ts index d1d1c4089..25f336121 100644 --- a/packages/core/src/helpers/plugins.ts +++ b/packages/core/src/helpers/plugins.ts @@ -181,11 +181,15 @@ export const unserializeBuildReport = (report: SerializedBuildReport): BuildRepo }; // Verify that we should get the git information based on the options. -// Only get git information if sourcemaps are enabled and git is enabled. +// Only get git information if something uploads it and git is enabled. export const shouldGetGitInfo = (options: Options): boolean => { - // If we don't have sourcemaps enabled, we don't need git. + // Source map uploads need git. const gitEnabledFromSourcemaps = !!options.errorTracking?.sourcemaps; + // Live Debugger symbol table uploads need git, and only happen with an API key. + const liveDebuggerConfig = options.liveDebugger; + const isLiveDebuggerEnabled = !!liveDebuggerConfig && !!(liveDebuggerConfig.enable ?? true); + const gitEnabledFromLiveDebugger = isLiveDebuggerEnabled && !!options.auth?.apiKey; // If we have the 'enableGit' configuration at the root, use it and default to `true`. const gitEnabledFromRoot = options.enableGit ?? true; - return gitEnabledFromSourcemaps && gitEnabledFromRoot; + return (gitEnabledFromSourcemaps || gitEnabledFromLiveDebugger) && gitEnabledFromRoot; }; diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 950440681..49f230a6f 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -92,6 +92,7 @@ export type Timer = { export type BuildMetadata = { name?: string; + service?: string; version?: string; }; @@ -198,10 +199,15 @@ export type GlobalContext = { env: GlobalData['env']; getLogger: GetLogger; git?: RepositoryData; + // Pending while the git plugin collects git data. Resolved once it's done, with or without data. + gitPending: boolean; + gitReady: Promise; hook: TriggerHook; inject: (item: ToInjectItem) => void; markArtifactsPending: () => void; markArtifactsReady: (error?: unknown) => void; + markGitPending: () => void; + markGitReady: () => void; pluginNames: string[]; plugins: (PluginOptions | CustomPluginOptions)[]; queue: (promise: Promise) => void; diff --git a/packages/factory/src/helpers/context.ts b/packages/factory/src/helpers/context.ts index 5e58550bd..156a5029e 100644 --- a/packages/factory/src/helpers/context.ts +++ b/packages/factory/src/helpers/context.ts @@ -52,6 +52,8 @@ export const getContext = ({ buildRoot, env: data.env, getLogger: getLoggerFactory(data, stores, options.logLevel), + gitPending: false, + gitReady: Promise.resolve(), // This will be updated in the injection plugin on initialization. asyncHook: () => { throw new Error('AsyncHook function called before it was initialized.'); @@ -65,6 +67,8 @@ export const getContext = ({ }, markArtifactsPending: () => {}, markArtifactsReady: () => {}, + markGitPending: () => {}, + markGitReady: () => {}, plugins: [], // This will be updated in the async-queue plugin on initialization. queue: () => { @@ -105,5 +109,25 @@ export const getContext = ({ } }; + let resolveGit: (() => void) | undefined; + + context.markGitPending = () => { + if (resolveGit) { + return; + } + + context.gitReady = new Promise((resolve) => { + resolveGit = resolve; + }); + context.gitPending = true; + }; + + context.markGitReady = () => { + const resolve = resolveGit; + resolveGit = undefined; + context.gitPending = false; + resolve?.(); + }; + return context; }; diff --git a/packages/factory/src/validate.test.ts b/packages/factory/src/validate.test.ts index 642870371..e260cb911 100644 --- a/packages/factory/src/validate.test.ts +++ b/packages/factory/src/validate.test.ts @@ -37,6 +37,11 @@ describe('factory validateOptions', () => { expect(result.metadata).toEqual({ version: '1.0.0' }); }); + it('should accept metadata with a service', () => { + const result = validateOptions({ metadata: { service: 'checkout' } }); + expect(result.metadata).toEqual({ service: 'checkout' }); + }); + it('should accept an empty metadata block', () => { const result = validateOptions({ metadata: {} }); expect(result.metadata).toEqual({}); @@ -284,6 +289,11 @@ describe('factory validateOptions', () => { input: { metadata: { version: null } }, errorPattern: /metadata\.version.*must be a string/, }, + { + description: 'reject metadata.service when not a string', + input: { metadata: { service: 123 } }, + errorPattern: /metadata\.service.*must be a string/, + }, ]; test.each(cases)('should $description', ({ input, errorPattern }) => { diff --git a/packages/factory/src/validate.ts b/packages/factory/src/validate.ts index 052e3a696..8ef30bc4c 100644 --- a/packages/factory/src/validate.ts +++ b/packages/factory/src/validate.ts @@ -58,6 +58,9 @@ const validateMetadata = (metadata: BuildMetadata | undefined): string[] => { if (metadata.version !== undefined && typeof metadata.version !== 'string') { errors.push('metadata.version must be a string'); } + if (metadata.service !== undefined && typeof metadata.service !== 'string') { + errors.push('metadata.service must be a string'); + } return errors; }; diff --git a/packages/plugins/error-tracking/README.md b/packages/plugins/error-tracking/README.md index 3a9a5aac0..55ca29567 100644 --- a/packages/plugins/error-tracking/README.md +++ b/packages/plugins/error-tracking/README.md @@ -132,10 +132,12 @@ If omitted, the plugin falls back to the shared top-level `metadata.version`. At ### errorTracking.sourcemaps.service -> required +> required (or set via [`metadata.service`](/README.md#metadataservice)) Should be set as the name of the service you're uploading sourcemaps for, and Datadog will use this service name to find the corresponding sourcemaps based on the `service` tag set on the RUM SDK. +If omitted, the plugin falls back to the shared top-level `metadata.service`. At least one of the two must be set. + > [!NOTE] > > If you get the error `Error: No git remotes available`, diff --git a/packages/plugins/error-tracking/src/types.ts b/packages/plugins/error-tracking/src/types.ts index a771d6cd3..17f724f67 100644 --- a/packages/plugins/error-tracking/src/types.ts +++ b/packages/plugins/error-tracking/src/types.ts @@ -24,7 +24,9 @@ type ServiceVersionSourcemapsOptions = SourcemapsUploadOptions & { // Optional: when omitted, the validator falls back to the shared // top-level `metadata.version`. At least one of the two must be set. releaseVersion?: string; - service: string; + // Optional: when omitted, the validator falls back to the shared + // top-level `metadata.service`. At least one of the two must be set. + service?: string; }; export type SourcemapsOptions = DebugIdSourcemapsOptions | ServiceVersionSourcemapsOptions; diff --git a/packages/plugins/error-tracking/src/validate.test.ts b/packages/plugins/error-tracking/src/validate.test.ts index 2430b8ee2..0261397a8 100644 --- a/packages/plugins/error-tracking/src/validate.test.ts +++ b/packages/plugins/error-tracking/src/validate.test.ts @@ -58,7 +58,7 @@ describe('Error Tracking Plugins validate', () => { expect(errors).toHaveLength(3); const expectedErrors = [ 'sourcemaps.releaseVersion is required (set it directly or via metadata.version).', - 'sourcemaps.service is required.', + 'sourcemaps.service is required (set it directly or via metadata.service).', 'sourcemaps.minifiedPathPrefix is required.', ]; expect(errors.map(stripAnsi)).toEqual(expectedErrors); @@ -247,6 +247,34 @@ describe('Error Tracking Plugins validate', () => { ); }); + test('Should fall back to metadata.service when sourcemaps.service is unset', () => { + const { config, errors } = validateSourcemapsOptions({ + metadata: { service: 'checkout' }, + errorTracking: { + sourcemaps: getMinimalSourcemapsConfiguration({ + service: undefined, + }), + }, + }); + + expect(errors).toHaveLength(0); + expect(config).toEqual(expect.objectContaining({ service: 'checkout' })); + }); + + test('Should prefer an explicit sourcemaps.service over metadata.service', () => { + const { config, errors } = validateSourcemapsOptions({ + metadata: { service: 'checkout' }, + errorTracking: { + sourcemaps: getMinimalSourcemapsConfiguration({ + service: 'cart', + }), + }, + }); + + expect(errors).toHaveLength(0); + expect(config).toEqual(expect.objectContaining({ service: 'cart' })); + }); + test('Should return an error with a bad minifiedPathPrefix', () => { const { errors } = validateSourcemapsOptions({ errorTracking: { diff --git a/packages/plugins/error-tracking/src/validate.ts b/packages/plugins/error-tracking/src/validate.ts index 10325b753..e6e669161 100644 --- a/packages/plugins/error-tracking/src/validate.ts +++ b/packages/plugins/error-tracking/src/validate.ts @@ -112,6 +112,8 @@ export const validateSourcemapsOptions = ( // configure one canonical build version at the top level keeps every // consumer (live-debugger, sourcemaps, …) reading from the same place. const releaseVersion = sourcemapsCfg.releaseVersion || config.metadata?.version; + // Same for `service`, with the shared top-level `metadata.service`. + const service = sourcemapsCfg.service || config.metadata?.service; // Validate the configuration. if (!releaseVersion) { @@ -128,8 +130,10 @@ export const validateSourcemapsOptions = ( `${red('sourcemaps.releaseVersion')} must match ${red('metadata.version')} when both are configured.`, ); } - if (!sourcemapsCfg.service) { - toReturn.errors.push(`${red('sourcemaps.service')} is required.`); + if (!service) { + toReturn.errors.push( + `${red('sourcemaps.service')} is required (set it directly or via ${red('metadata.service')}).`, + ); } if (!sourcemapsCfg.minifiedPathPrefix) { toReturn.errors.push(`${red('sourcemaps.minifiedPathPrefix')} is required.`); @@ -145,10 +149,10 @@ export const validateSourcemapsOptions = ( ); } - // Build the resolved config only when `releaseVersion` actually - // resolves; otherwise an error has been recorded and the caller will - // throw before the config is read. - if (releaseVersion) { + // Build the resolved config only when `releaseVersion` and `service` + // actually resolve; otherwise an error has been recorded and the caller + // will throw before the config is read. + if (releaseVersion && service) { toReturn.config = { bailOnError: false, dryRun: false, @@ -156,6 +160,7 @@ export const validateSourcemapsOptions = ( ...sourcemapsCfg, debugId: false, releaseVersion, + service, }; } diff --git a/packages/plugins/git/src/index.ts b/packages/plugins/git/src/index.ts index 97a3f9899..47c0bdda9 100644 --- a/packages/plugins/git/src/index.ts +++ b/packages/plugins/git/src/index.ts @@ -30,11 +30,15 @@ export const getGitPlugins: GetInternalPlugins = (arg: GetPluginsArg) => { const repositoryData = await getRepositoryData(git, repositoryUrl); context.git = repositoryData; + // Let plugins that wait for git data continue without waiting for the hook's handlers. + context.markGitReady(); timeGit.end(); await context.asyncHook('git', context.git); } catch (e: any) { log.error(`Could not get git information: ${e.message}`); + } finally { + context.markGitReady(); } }; @@ -59,6 +63,7 @@ export const getGitPlugins: GetInternalPlugins = (arg: GetPluginsArg) => { // buildRoot hook is sync because xpack can't make it async. // So we queue the async part of the plugin. + context.markGitPending(); context.queue(processGit(gitDir)); } catch (e: any) { // We don't want to have the build fail for this. diff --git a/packages/plugins/live-debugger/CONTRIBUTING.md b/packages/plugins/live-debugger/CONTRIBUTING.md index 001135be1..86a033ba9 100644 --- a/packages/plugins/live-debugger/CONTRIBUTING.md +++ b/packages/plugins/live-debugger/CONTRIBUTING.md @@ -47,15 +47,15 @@ When adding or changing `liveDebugger` configuration, update [`src/types.ts`](./ ## Symbol table -The plugin can list the instrumented functions that ship in a build, for upload to Datadog's Symbol Database. Each entry has the function ID, a display name, the path relative to the git repository root, and the original line range. The table lives in [`src/symbol-table`](./src/symbol-table). +The plugin lists the instrumented functions that ship in a build and uploads them to Datadog's Symbol Database. Each entry has the function ID, a display name, the path relative to the git repository root, and the original line range. The code lives in [`src/symbol-table`](./src/symbol-table). -For now, the only consumer is an opt-in debug dump. Set `DD_LIVE_DEBUGGER_SYMBOL_TABLE_PATH` to a file path to generate the table and write it there: +The upload is on by default, and the README lists [what it needs](./README.md#symbol-upload). When it can't happen, the table isn't generated either, so the build does no extra work. To inspect the table, set `DD_LIVE_DEBUGGER_SYMBOL_TABLE_PATH` to a file path: the plugin then also writes the table there, in the format of the upload's file part. ```bash DD_LIVE_DEBUGGER_SYMBOL_TABLE_PATH=/tmp/symbol-table.json yarn build ``` -Without that variable, the plugin doesn't add any hook or build time. The dump is never written inside the bundler's output directory, and generation never fails the build: problems are logged as warnings. +The dump is never written inside the bundler's output directory, and neither generation nor upload ever fails the build: problems are logged as warnings. How it works: diff --git a/packages/plugins/live-debugger/README.md b/packages/plugins/live-debugger/README.md index 6418199d0..ca10bc202 100644 --- a/packages/plugins/live-debugger/README.md +++ b/packages/plugins/live-debugger/README.md @@ -11,11 +11,13 @@ Automatically instrument JavaScript functions at build time to enable Live Debug - [Prerequisites](#prerequisites) - [Source maps](#source-maps) + - [Service and version](#service-and-version) - [Required peer dependencies](#required-peer-dependencies) - [Limitations during Preview](#limitations-during-preview) - [Configuration](#configuration) - [How it works](#how-it-works) - [liveDebugger.enable](#livedebuggerenable) + - [metadata.service](#metadataservice) - [metadata.version](#metadataversion) - [liveDebugger.include](#livedebuggerinclude) - [liveDebugger.exclude](#livedebuggerexclude) @@ -24,6 +26,7 @@ Automatically instrument JavaScript functions at build time to enable Live Debug - [liveDebugger.functionTypes](#livedebuggerfunctiontypes) - [liveDebugger.namedOnly](#livedebuggernamedonly) - [liveDebugger.decorators](#livedebuggerdecorators) +- [Symbol upload](#symbol-upload) - [Skipped function types](#skipped-function-types) - [Bundle size impact](#bundle-size-impact) - [Runtime requirements](#runtime-requirements) @@ -43,6 +46,10 @@ Datadog resolves the deployed build and its source code through the [debug ID in - Keep [`enableGit`](/#enablegit) enabled (the default) and build from a Git checkout of the deployed commit. Set [`gitRepositoryUrl`](/#gitrepositoryurl) if CI clones through a mirror or proxy. - Set [`auth.site`](/#authsite) to the site that the [Browser Debugger SDK](https://github.com/DataDog/browser-sdk/tree/main/packages/browser-debugger#prerequisites) sends data to, and use an API key from the same organization as the SDK's client token. +### Service and version + +Set [`metadata.service`](#metadataservice) and [`metadata.version`](#metadataversion). The plugin needs both to [upload symbols](#symbol-upload). + ### Required peer dependencies The Live Debugger transform relies on Babel and `magic-string`. To keep the cost of @@ -88,7 +95,9 @@ liveDebugger?: { } ``` -Live Debugger also reads the build version from the top-level [`metadata.version`](#metadataversion) option. +Live Debugger also reads the build's service and version from the top-level [`metadata.service`](#metadataservice) and [`metadata.version`](#metadataversion) options. + +After each production build, the plugin [uploads symbols](#symbol-upload) to Datadog, which Live Debugger needs to add logpoints to your browser application. ## How it works @@ -135,6 +144,21 @@ Enable or disable the plugin without removing its configuration. Must be a boolean. Non-boolean values are coerced today but will be rejected in a future major release. +### metadata.service + +> default: `undefined` + +The service of the browser application, as reported by the Browser Debugger SDK. Set it at the top level of your Datadog plugin configuration: + +```ts +datadogBuildPlugins({ + metadata: { service: 'my-app', version: '1.0.0' }, + liveDebugger: {}, +}); +``` + +[Symbols](#symbol-upload) are uploaded for this service, and Live Debugger looks them up by the service the SDK reports, so the two must match. Live Debugger injects the value into the runtime-visible [build metadata](#build-metadata), so the Browser Debugger SDK uses it as the default `service` during `init()`. + ### metadata.version > default: `undefined` @@ -150,7 +174,7 @@ datadogBuildPlugins({ When set, Live Debugger injects the value into runtime-visible [build metadata](#build-metadata) so the Browser Debugger SDK uses it as the default `version` during `init()`. -Set `metadata.version` when you want version-specific probe targeting. +[Symbols](#symbol-upload) are only uploaded when `metadata.version` is set. It also enables version-specific probe targeting. ### liveDebugger.include @@ -266,6 +290,19 @@ liveDebugger: { } ``` +## Symbol upload + +To let you add logpoints to your browser application, Live Debugger needs to know which functions each build contains. After each production build, the plugin uploads nonsensitive symbols and metadata to Datadog: the names of the instrumented functions, the files they're in, and their line numbers. + +The upload needs: + +- an API key; +- a [service](#metadataservice) and a [version](#metadataversion); +- a git repository with a remote, and the [git integration](/README.md#enablegit) enabled; +- source maps, for the line numbers. + +When something is missing, the plugin logs a warning and skips the upload. Development servers and watch mode don't upload symbols. The upload never fails your build, and doesn't add anything to your build output. + ## Skipped function types The following function types are always skipped regardless of configuration: @@ -306,14 +343,19 @@ The plugin injects code into every output chunk that defines a `__DD_LIVE_DEBUGG ```javascript if (typeof globalThis.__DD_LIVE_DEBUGGER_BUILD__ === 'undefined') { - globalThis.__DD_LIVE_DEBUGGER_BUILD__ = { version: '1.0.0', debugId: '' }; + globalThis.__DD_LIVE_DEBUGGER_BUILD__ = { + version: '1.0.0', + service: 'my-app', + debugId: '', + }; } ``` - `version` is the [`metadata.version`](#metadataversion) value. It is omitted when `metadata.version` is not set. +- `service` is the [`metadata.service`](#metadataservice) value. It is omitted when `metadata.service` is not set. - `debugId` is the debug ID of the chunk that defined the global: the same value the [RUM plugin](/packages/plugins/rum#rumsourcecodecontextdebugid) injects into that chunk and that its source map is uploaded with. It is only included when debug IDs are enabled, with the top-level [`sourcemaps.debugId`](/#sourcemaps) option or with `rum.sourceCodeContext.debugId`. -The global is defined even when both fields are omitted. The first chunk to execute defines it and later chunks leave it untouched. All the source maps of a build are uploaded with the same git metadata, so the debug ID of any of its chunks identifies the build's repository and commit. +The global is defined even when all the fields are omitted. The first chunk to execute defines it and later chunks leave it untouched. All the source maps of a build are uploaded with the same git metadata, so the debug ID of any of its chunks identifies the build's repository and commit. ### Invocation handles diff --git a/packages/plugins/live-debugger/src/constants.ts b/packages/plugins/live-debugger/src/constants.ts index 2544e2adc..1f4c2a854 100644 --- a/packages/plugins/live-debugger/src/constants.ts +++ b/packages/plugins/live-debugger/src/constants.ts @@ -20,9 +20,14 @@ export const FUNCTION_ID_SEPARATOR = ';'; export const ANONYMOUS_FUNCTION_NAME = ''; // Path where an opt-in debug dump of the symbol table is written. -// Until the symbol table upload exists, setting it is also what enables the symbol table. export const SYMBOL_TABLE_PATH_ENV = 'DD_LIVE_DEBUGGER_SYMBOL_TABLE_PATH'; +// Intake that the symbol table is uploaded to. +export const SYMBOL_TABLE_INTAKE_SUBDOMAIN = 'debugger-intake'; +export const SYMBOL_TABLE_INTAKE_PATH = 'api/v2/debugger'; +// The intake accepts up to 50 MiB per request. Keep room for the request's event part. +export const SYMBOL_TABLE_MAX_PART_BYTES = 50 * 1024 * 1024 - 64 * 1024; + export const DEFAULT_FILE_EXTENSIONS = [ '.js', '.jsx', diff --git a/packages/plugins/live-debugger/src/index.test.ts b/packages/plugins/live-debugger/src/index.test.ts index 6327fc71e..65a7546ac 100644 --- a/packages/plugins/live-debugger/src/index.test.ts +++ b/packages/plugins/live-debugger/src/index.test.ts @@ -16,6 +16,8 @@ const makeOptions = ( overrides: Partial = {}, ): LiveDebuggerOptionsWithDefaults => ({ version: '1.0.0', + service: undefined, + enableGit: false, debugId: false, include: [], exclude: [/\/node_modules\//], @@ -507,22 +509,22 @@ describe('getPlugins', () => { const cases: { description: string; options: Partial; - expected: Pick; + expected: Pick; }[] = [ { description: 'inject the runtime bootstrap when an empty config is provided', options: { liveDebugger: {} }, - expected: { version: undefined, debugId: false }, + expected: { version: undefined, service: undefined, debugId: false }, }, { description: 'inject the version when metadata.version is provided', options: { liveDebugger: {}, metadata: { version: '1.0.0' } }, - expected: { version: '1.0.0', debugId: false }, + expected: { version: '1.0.0', service: undefined, debugId: false }, }, { description: 'not inject a version when only metadata.name is provided', options: { liveDebugger: {}, metadata: { name: 'my-build' } }, - expected: { version: undefined, debugId: false }, + expected: { version: undefined, service: undefined, debugId: false }, }, { description: 'inject the chunk debug ID when RUM debug IDs are enabled', @@ -530,7 +532,7 @@ describe('getPlugins', () => { liveDebugger: {}, rum: { sourceCodeContext: { debugId: true } }, }, - expected: { version: undefined, debugId: true }, + expected: { version: undefined, service: undefined, debugId: true }, }, { description: 'inject the version and the chunk debug ID', @@ -539,7 +541,7 @@ describe('getPlugins', () => { metadata: { version: '1.0.0' }, rum: { sourceCodeContext: { debugId: true } }, }, - expected: { version: '1.0.0', debugId: true }, + expected: { version: '1.0.0', service: undefined, debugId: true }, }, { description: 'not inject the chunk debug ID when RUM uses service and version', @@ -547,7 +549,12 @@ describe('getPlugins', () => { liveDebugger: {}, rum: { sourceCodeContext: { service: 'checkout' } }, }, - expected: { version: undefined, debugId: false }, + expected: { version: undefined, service: undefined, debugId: false }, + }, + { + description: 'inject the service when metadata.service is provided', + options: { liveDebugger: {}, metadata: { service: 'cart' } }, + expected: { version: undefined, service: 'cart', debugId: false }, }, ]; diff --git a/packages/plugins/live-debugger/src/runtime-bootstrap.test.ts b/packages/plugins/live-debugger/src/runtime-bootstrap.test.ts index c057208b0..f4d3d73e0 100644 --- a/packages/plugins/live-debugger/src/runtime-bootstrap.test.ts +++ b/packages/plugins/live-debugger/src/runtime-bootstrap.test.ts @@ -32,37 +32,43 @@ describe('getRuntimeBootstrap', () => { const cases = [ { description: 'expose the version and the chunk debug ID', - options: { version: '1.0.0', debugId: true }, + options: { version: '1.0.0', service: undefined, debugId: true }, chunk, expected: { version: '1.0.0', debugId: chunkDebugId }, }, { description: 'expose the chunk debug ID without a version', - options: { version: undefined, debugId: true }, + options: { version: undefined, service: undefined, debugId: true }, chunk, expected: { debugId: chunkDebugId }, }, { description: 'expose the version only when debug IDs are disabled', - options: { version: '1.0.0', debugId: false }, + options: { version: '1.0.0', service: undefined, debugId: false }, chunk, expected: { version: '1.0.0' }, }, + { + description: 'expose the service', + options: { version: '1.0.0', service: 'checkout', debugId: false }, + chunk, + expected: { version: '1.0.0', service: 'checkout' }, + }, { description: 'expose empty build metadata without a version or debug IDs', - options: { version: undefined, debugId: false }, + options: { version: undefined, service: undefined, debugId: false }, chunk, expected: {}, }, { description: 'not expose a debug ID without chunk information', - options: { version: '1.0.0', debugId: true }, + options: { version: '1.0.0', service: undefined, debugId: true }, chunk: undefined, expected: { version: '1.0.0' }, }, { description: 'safely serialize the version', - options: { version: `1.0.0"'\\\u2028`, debugId: false }, + options: { version: `1.0.0"'\\\u2028`, service: undefined, debugId: false }, chunk, expected: { version: `1.0.0"'\\\u2028` }, }, @@ -81,14 +87,22 @@ describe('getRuntimeBootstrap', () => { }); test('should be a per-chunk injected value', () => { - const bootstrap = getRuntimeBootstrap({ version: undefined, debugId: true }); + const bootstrap = getRuntimeBootstrap({ + version: undefined, + service: undefined, + debugId: true, + }); // The injection plugin only passes chunk info to functions with a single parameter. expect(bootstrap).toHaveLength(1); }); test('should keep the build metadata of the first chunk to execute', () => { - const bootstrap = getRuntimeBootstrap({ version: '1.0.0', debugId: true }); + const bootstrap = getRuntimeBootstrap({ + version: '1.0.0', + service: undefined, + debugId: true, + }); const firstChunk = getChunk('first chunk'); const secondChunk = getChunk('second chunk'); const firstChunkCode = bootstrap(firstChunk); diff --git a/packages/plugins/live-debugger/src/runtime-bootstrap.ts b/packages/plugins/live-debugger/src/runtime-bootstrap.ts index 37f105c1a..26fe6b853 100644 --- a/packages/plugins/live-debugger/src/runtime-bootstrap.ts +++ b/packages/plugins/live-debugger/src/runtime-bootstrap.ts @@ -11,6 +11,7 @@ const buildMetadataGlobal = '__DD_LIVE_DEBUGGER_BUILD__' as const; type BuildMetadata = { version?: string; + service?: string; debugId?: string; }; @@ -18,19 +19,22 @@ type BuildMetadata = { // instrumented functions only call once the SDK has defined it. It exposes build // metadata to the Browser Debugger SDK: // - `version` (from `metadata.version`) defaults the SDK's runtime version. +// - `service` (from `metadata.service`) defaults the SDK's service, so it +// matches the service the symbol table is uploaded with. // - `debugId` is the chunk's debug ID, the same one its sourcemap is uploaded // with. Every sourcemap of a build is uploaded with the same git metadata, // so any chunk's debug ID identifies the build's repository and commit. // The first chunk to execute defines the global; later chunks keep it. export const getRuntimeBootstrap = ({ version, + service, debugId, -}: Pick) => { +}: Pick) => { return (chunk?: ChunkInfo): string => { // Without chunk info there is no debug ID to match, so don't invent one. const chunkDebugId = debugId && chunk ? getChunkDebugId(chunk) : undefined; // Undefined fields are left out by JSON.stringify. - const buildMetadata: BuildMetadata = { version, debugId: chunkDebugId }; + const buildMetadata: BuildMetadata = { version, service, debugId: chunkDebugId }; const serializedBuildMetadata = JSON.stringify(buildMetadata); return `if(typeof globalThis.${buildMetadataGlobal}==='undefined'){globalThis.${buildMetadataGlobal}=${serializedBuildMetadata}}`; diff --git a/packages/plugins/live-debugger/src/symbol-table/index.test.ts b/packages/plugins/live-debugger/src/symbol-table/index.test.ts index abb914c00..3aa512f80 100644 --- a/packages/plugins/live-debugger/src/symbol-table/index.test.ts +++ b/packages/plugins/live-debugger/src/symbol-table/index.test.ts @@ -4,10 +4,12 @@ import { existsSync, rm } from '@dd/core/helpers/fs'; import { getTempWorkingDir } from '@dd/tests/_jest/helpers/env'; -import { getMockLogger, mockLogFn } from '@dd/tests/_jest/helpers/mocks'; +import { getContextMock, getMockLogger, mockLogFn } from '@dd/tests/_jest/helpers/mocks'; import path from 'path'; -import { generateSymbolTable, writeSymbolTable } from './index'; +import type { LiveDebuggerOptionsWithDefaults } from '../types'; + +import { generateSymbolTable, getUploadSetup, writeSymbolTable } from './index'; const PAYLOAD = { language: 'browser' as const, scopes: [] }; @@ -59,3 +61,68 @@ describe('Live Debugger symbol table', () => { ); }); }); + +describe('getUploadSetup', () => { + const options = { + version: '1.0.0', + service: 'shop', + enableGit: true, + } satisfies Partial; + + const cases = [ + { + description: 'upload with an API key, a service, a version and git', + options, + apiKey: 'api-key', + expected: { + settings: { + apiKey: 'api-key', + site: 'datadoghq.com', + service: 'shop', + version: '1.0.0', + }, + }, + }, + { + description: 'skip without an API key', + options, + apiKey: undefined, + expected: { skipReason: expect.stringContaining('API key') }, + }, + { + description: 'skip without a service', + options: { ...options, service: undefined }, + apiKey: 'api-key', + expected: { skipReason: expect.stringContaining('metadata.service') }, + }, + { + description: 'skip without a version', + options: { ...options, version: undefined }, + apiKey: 'api-key', + expected: { skipReason: expect.stringContaining('metadata.version') }, + }, + { + description: 'skip when git is disabled', + options: { ...options, enableGit: false }, + apiKey: 'api-key', + expected: { skipReason: expect.stringContaining('enableGit') }, + }, + ]; + + test.each(cases)('Should $description.', ({ options: pluginOptions, apiKey, expected }) => { + const context = getContextMock({ auth: { apiKey, site: 'datadoghq.com' } }); + const validatedOptions: LiveDebuggerOptionsWithDefaults = { + debugId: false, + include: [], + exclude: [], + fileExtensions: 'all', + honorSkipComments: true, + functionTypes: undefined, + namedOnly: false, + decorators: 'legacy', + ...pluginOptions, + }; + + expect(getUploadSetup(validatedOptions, context)).toEqual(expected); + }); +}); diff --git a/packages/plugins/live-debugger/src/symbol-table/index.ts b/packages/plugins/live-debugger/src/symbol-table/index.ts index 38dfff69d..d4c99da0a 100644 --- a/packages/plugins/live-debugger/src/symbol-table/index.ts +++ b/packages/plugins/live-debugger/src/symbol-table/index.ts @@ -6,7 +6,7 @@ import { outputJson, readFile } from '@dd/core/helpers/fs'; import type { BuildReport, GlobalContext, Logger, Output, PluginOptions } from '@dd/core/types'; import path from 'path'; -import { SYMBOL_TABLE_PATH_ENV } from '../constants'; +import { SYMBOL_TABLE_MAX_PART_BYTES, SYMBOL_TABLE_PATH_ENV } from '../constants'; import type { LiveDebuggerOptionsWithDefaults } from '../types'; import { createBuildModeTracker } from './build-mode'; @@ -17,6 +17,7 @@ import type { SymbolTablePayload } from './serialize'; import { serializeSymbolTable } from './serialize'; import { buildSymbolTable } from './table'; import type { ChunkFunction, ChunkFunctions, SymbolTable } from './types'; +import { uploadSymbolTable } from './upload'; const LOG_PREFIX = 'Live Debugger symbols'; const JS_OUTPUT_TYPES = new Set(['js', 'mjs', 'cjs']); @@ -135,9 +136,46 @@ export const writeSymbolTable = async ( log.info(`${LOG_PREFIX}: wrote ${payload.scopes.length} file(s) to "${absoluteDumpPath}".`); }; +type UploadSettings = { + apiKey: string; + site: string; + service: string; + version: string; +}; + +type UploadSetup = { settings: UploadSettings } | { skipReason: string }; + +// What the upload needs from the configuration. Git data is only known later in the build. +export const getUploadSetup = ( + pluginOptions: LiveDebuggerOptionsWithDefaults, + context: GlobalContext, +): UploadSetup => { + const { apiKey, site } = context.auth; + const { service, version, enableGit } = pluginOptions; + + if (!apiKey) { + return { skipReason: 'no API key is configured' }; + } + if (!service) { + return { + skipReason: 'no service is configured. Set "metadata.service"', + }; + } + if (!version) { + return { skipReason: 'no version is configured. Set "metadata.version"' }; + } + if (!enableGit) { + return { skipReason: 'git is disabled with "enableGit: false"' }; + } + + return { settings: { apiKey, site, service, version } }; +}; + /** - * Hooks that generate the symbol table after one-off builds. They're only added when something - * consumes the table, so other builds don't do any extra work. The table never fails the build. + * Hooks that generate the symbol table after one-off builds and upload it to SymDB. Live Debugger + * needs the table to create probes, so the upload is on by default. When the table can't be + * uploaded, it's only generated for the opt-in debug dump, so other builds don't do any extra work. + * The table never fails the build. */ export const getSymbolTableHooks = ( pluginOptions: LiveDebuggerOptionsWithDefaults, @@ -145,31 +183,83 @@ export const getSymbolTableHooks = ( log: Logger, ): Partial => { const dumpPath = process.env[SYMBOL_TABLE_PATH_ENV]; - if (!dumpPath) { - return {}; - } - + const uploadSetup = getUploadSetup(pluginOptions, context); const buildMode = createBuildModeTracker(); + const upload = async ( + payload: SymbolTablePayload, + functionCount: number, + settings: UploadSettings, + ) => { + // The git plugin collects git data while the build runs. + await context.gitReady; + const repositoryData = context.git; + if (!repositoryData) { + log.warn( + `Not uploading ${LOG_PREFIX}: no git repository data, for instance because the repository has no remote.`, + ); + return; + } + + const timer = log.time('upload symbol table'); + try { + await uploadSymbolTable( + payload, + { + ...settings, + repositoryUrl: repositoryData.remote, + commitSha: repositoryData.hash, + bundlerName: context.bundler.name, + pluginVersion: context.version, + }, + SYMBOL_TABLE_MAX_PART_BYTES, + log, + ); + log.info( + `${LOG_PREFIX}: uploaded ${functionCount} function(s) from ${payload.scopes.length} file(s).`, + ); + } catch (error) { + log.warn(`Could not upload ${LOG_PREFIX}: ${error}`); + } finally { + timer.end(); + } + }; + const handleSymbolTable = async (report: BuildReport) => { // Wait for the injection plugin to finish rewriting esbuild's output. await context.artifactsReady; + const timer = log.time('generate symbol table'); + let table: SymbolTable | undefined; + let payload: SymbolTablePayload | undefined; try { - const table = await generateSymbolTable({ + table = await generateSymbolTable({ outputs: report.outputs ?? [], buildRoot: context.buildRoot, log, }); - if (table) { - const payload = serializeSymbolTable(table, pluginOptions.version); - await writeSymbolTable(payload, dumpPath, context.bundler.outDir, log); - } + payload = table && serializeSymbolTable(table, pluginOptions.version); } catch (error) { log.warn(`Could not collect ${LOG_PREFIX}: ${error}`); } finally { timer.end(); } + + if (!table || !payload) { + return; + } + + if (dumpPath) { + try { + await writeSymbolTable(payload, dumpPath, context.bundler.outDir, log); + } catch (error) { + log.warn(`Could not write ${LOG_PREFIX}: ${error}`); + } + } + + if ('settings' in uploadSetup) { + await upload(payload, table.entries.length, uploadSetup.settings); + } }; return { @@ -179,8 +269,16 @@ export const getSymbolTableHooks = ( return; } + if ('skipReason' in uploadSetup) { + log.warn(`Not uploading ${LOG_PREFIX}: ${uploadSetup.skipReason}.`); + if (!dumpPath) { + return; + } + } + + // Let the bundler finish while the output or the git data is still in progress. const handling = handleSymbolTable(report); - if (context.artifactsPending) { + if (context.artifactsPending || context.gitPending) { context.queue(handling); return; } diff --git a/packages/plugins/live-debugger/src/symbol-table/upload.test.ts b/packages/plugins/live-debugger/src/symbol-table/upload.test.ts new file mode 100644 index 000000000..0405233a7 --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/upload.test.ts @@ -0,0 +1,191 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { doRequest } from '@dd/core/helpers/request'; +import { getMockLogger } from '@dd/tests/_jest/helpers/mocks'; +import { gunzipSync } from 'zlib'; + +import type { SymbolTablePayload, SymDbScope } from './serialize'; +import type { SymbolTableUploadContext } from './upload'; +import { createUploadParts, getIntakeUrl, uploadSymbolTable } from './upload'; + +jest.mock('@dd/core/helpers/request', () => { + const original = jest.requireActual('@dd/core/helpers/request'); + return { + ...original, + doRequest: jest.fn(), + }; +}); + +const doRequestMock = jest.mocked(doRequest); + +const CONTEXT: SymbolTableUploadContext = { + apiKey: 'api-key', + site: 'datadoghq.com', + service: 'shop', + version: '1.2.3', + repositoryUrl: 'https://github.com/DataDog/shop', + commitSha: 'abc123', + bundlerName: 'webpack', + pluginVersion: '3.3.3', +}; + +const MAX_PART_BYTES = 50 * 1024 * 1024; + +const getModuleScope = (index: number): SymDbScope => ({ + scope_type: 'MODULE', + name: `src/file-${index}.js`, + source_file: `src/file-${index}.js`, + scopes: [ + { + scope_type: 'FUNCTION', + name: `function${index}`, + source_file: `src/file-${index}.js`, + start_line: 1, + end_line: 3, + language_specifics: { function_id: `src/file-${index}.js;function${index}` }, + }, + ], +}); + +const getPayload = (moduleCount: number): SymbolTablePayload => ({ + language: 'browser', + version: CONTEXT.version, + chunk_debug_ids: ['aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'], + scopes: Array.from({ length: moduleCount }, (_, index) => getModuleScope(index)), +}); + +const readJson = (file: Buffer) => { + const content = gunzipSync(file).toString('utf-8'); + return JSON.parse(content); +}; + +const readFormPart = async (form: FormData, name: string) => { + const part = form.get(name); + if (!part || typeof part === 'string') { + throw new Error(`Missing the "${name}" part.`); + } + const buffer = await part.arrayBuffer(); + return { part, content: Buffer.from(buffer) }; +}; + +describe('Symbol table upload', () => { + beforeEach(() => { + doRequestMock.mockReset(); + }); + + test('Should post one part with the event and the gzipped symbol table.', async () => { + const payload = getPayload(2); + + const partCount = await uploadSymbolTable( + payload, + CONTEXT, + MAX_PART_BYTES, + getMockLogger(), + ); + + expect(partCount).toBe(1); + expect(doRequestMock).toHaveBeenCalledTimes(1); + const [requestOptions] = doRequestMock.mock.calls[0]; + expect(requestOptions).toEqual( + expect.objectContaining({ + url: getIntakeUrl(CONTEXT), + method: 'POST', + auth: { apiKey: 'api-key' }, + }), + ); + + const data = await requestOptions.getData?.(); + if (!(data?.data instanceof FormData)) { + throw new Error('Expected a multipart body.'); + } + expect(data.headers).toEqual( + expect.objectContaining({ 'DD-EVP-ORIGIN': 'webpack-build-plugin_live-debugger' }), + ); + + const event = await readFormPart(data.data, 'event'); + const file = await readFormPart(data.data, 'file'); + const eventJson = JSON.parse(event.content.toString('utf-8')); + const fileJson = readJson(file.content); + + expect(event.part.name).toBe('event.json'); + expect(file.part.name).toBe('file.gz'); + expect(eventJson).toEqual({ + ddsource: 'dd_debugger', + service: 'shop', + version: '1.2.3', + language: 'browser', + runtimeId: eventJson.uploadId, + type: 'symdb', + uploadId: expect.any(String), + batchNum: 1, + final: true, + attachmentSize: file.content.length, + }); + expect(fileJson).toEqual({ + service: 'shop', + version: '1.2.3', + language: 'browser', + upload_id: eventJson.uploadId, + batch_num: 1, + final: true, + chunk_debug_ids: payload.chunk_debug_ids, + scopes: payload.scopes, + }); + }); + + test('Should use a new upload ID for every upload.', async () => { + const payload = getPayload(1); + const uploadIds: string[] = []; + + for (let index = 0; index < 2; index++) { + // eslint-disable-next-line no-await-in-loop + await uploadSymbolTable(payload, CONTEXT, MAX_PART_BYTES, getMockLogger()); + const [requestOptions] = doRequestMock.mock.calls[index]; + // eslint-disable-next-line no-await-in-loop + const data = await requestOptions.getData?.(); + if (!(data?.data instanceof FormData)) { + throw new Error('Expected a multipart body.'); + } + // eslint-disable-next-line no-await-in-loop + const event = await readFormPart(data.data, 'event'); + const eventJson = JSON.parse(event.content.toString('utf-8')); + uploadIds.push(eventJson.uploadId); + } + + expect(uploadIds[0]).not.toBe(uploadIds[1]); + }); + + test('Should tag the upload with its git repository and commit.', () => { + const url = new URL(getIntakeUrl(CONTEXT)); + + expect(url.origin).toBe('https://debugger-intake.datadoghq.com'); + expect(url.pathname).toBe('/api/v2/debugger'); + expect(url.searchParams.get('ddtags')).toBe( + 'git.repository_url:https://github.com/DataDog/shop,git.commit.sha:abc123', + ); + }); + + test('Should split a table that is too big by file, in numbered parts.', async () => { + const payload = getPayload(5); + const singlePart = await createUploadParts(payload, CONTEXT, 'upload-id', MAX_PART_BYTES); + // Too small for the whole table, but big enough for one file. + const maxBytes = singlePart[0].file.length - 1; + + const parts = await createUploadParts(payload, CONTEXT, 'upload-id', maxBytes); + const files = parts.map((part) => readJson(part.file)); + const partScopes = files.flatMap((file) => file.scopes); + + expect(parts.length).toBeGreaterThan(1); + expect(parts.map((part) => part.batchNum)).toEqual( + Array.from({ length: parts.length }, (_, index) => index + 1), + ); + expect(parts.map((part) => part.final)).toEqual( + Array.from({ length: parts.length }, (_, index) => index === parts.length - 1), + ); + expect(files.map((file) => file.batch_num)).toEqual(parts.map((part) => part.batchNum)); + expect(files.every((file) => file.upload_id === 'upload-id')).toBe(true); + expect(partScopes).toEqual(payload.scopes); + }); +}); diff --git a/packages/plugins/live-debugger/src/symbol-table/upload.ts b/packages/plugins/live-debugger/src/symbol-table/upload.ts new file mode 100644 index 000000000..02e32e1d8 --- /dev/null +++ b/packages/plugins/live-debugger/src/symbol-table/upload.ts @@ -0,0 +1,219 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { doRequest, getOriginHeaders } from '@dd/core/helpers/request'; +import type { Logger } from '@dd/core/types'; +import { randomUUID } from 'crypto'; +import { promisify } from 'util'; +import { gzip } from 'zlib'; + +import { SYMBOL_TABLE_INTAKE_PATH, SYMBOL_TABLE_INTAKE_SUBDOMAIN } from '../constants'; + +import type { SymbolTablePayload, SymDbScope } from './serialize'; + +const gzipAsync = promisify(gzip); + +export type SymbolTableUploadContext = { + apiKey: string; + site: string; + service: string; + version: string; + repositoryUrl: string; + commitSha: string; + bundlerName: string; + pluginVersion: string; +}; + +export type UploadPart = { + batchNum: number; + final: boolean; + // The gzipped file part. + file: Buffer; +}; + +type FileContentArgs = { + payload: SymbolTablePayload; + context: SymbolTableUploadContext; + uploadId: string; + batchNum: number; + final: boolean; + scopes: SymDbScope[]; +}; + +// The upload's file part: the symbol table, with the fields of each request. +const getFileContent = ({ + payload, + context, + uploadId, + batchNum, + final, + scopes, +}: FileContentArgs): string => { + const file = { + service: context.service, + version: context.version, + language: payload.language, + upload_id: uploadId, + batch_num: batchNum, + final, + ...(payload.chunk_debug_ids ? { chunk_debug_ids: payload.chunk_debug_ids } : {}), + scopes, + }; + return JSON.stringify(file); +}; + +const compress = async (content: string): Promise => gzipAsync(content); + +// Split the files into groups whose compressed size fits the limit. A single file that doesn't fit +// still gets its own group, and the intake rejects it. +const splitScopes = async ( + scopes: SymDbScope[], + getCompressedSize: (scopes: SymDbScope[]) => Promise, + maxBytes: number, +): Promise => { + const compressedSize = await getCompressedSize(scopes); + if (compressedSize <= maxBytes || scopes.length <= 1) { + return [scopes]; + } + + const middle = Math.ceil(scopes.length / 2); + const firstHalf = scopes.slice(0, middle); + const secondHalf = scopes.slice(middle); + const firstGroups = await splitScopes(firstHalf, getCompressedSize, maxBytes); + const secondGroups = await splitScopes(secondHalf, getCompressedSize, maxBytes); + return [...firstGroups, ...secondGroups]; +}; + +/** + * Split the symbol table by file into gzipped file parts of at most `maxBytes`. Most tables fit in + * one part, which is then compressed only once. + */ +export const createUploadParts = async ( + payload: SymbolTablePayload, + context: SymbolTableUploadContext, + uploadId: string, + maxBytes: number, +): Promise => { + const singleContent = getFileContent({ + payload, + context, + uploadId, + batchNum: 1, + final: true, + scopes: payload.scopes, + }); + const singleFile = await compress(singleContent); + if (singleFile.length <= maxBytes) { + return [{ batchNum: 1, final: true, file: singleFile }]; + } + + const getCompressedSize = async (scopes: SymDbScope[]) => { + const content = getFileContent({ + payload, + context, + uploadId, + batchNum: 1, + final: false, + scopes, + }); + const file = await compress(content); + return file.length; + }; + const groups = await splitScopes(payload.scopes, getCompressedSize, maxBytes); + + const parts: UploadPart[] = []; + for (const [index, scopes] of groups.entries()) { + const batchNum = index + 1; + const final = batchNum === groups.length; + const content = getFileContent({ payload, context, uploadId, batchNum, final, scopes }); + // One part at a time, to keep memory low. + // eslint-disable-next-line no-await-in-loop + const file = await compress(content); + parts.push({ batchNum, final, file }); + } + + return parts; +}; + +export const getIntakeUrl = (context: SymbolTableUploadContext): string => { + const url = new URL( + `https://${SYMBOL_TABLE_INTAKE_SUBDOMAIN}.${context.site}/${SYMBOL_TABLE_INTAKE_PATH}`, + ); + const tags = [ + `git.repository_url:${context.repositoryUrl}`, + `git.commit.sha:${context.commitSha}`, + ]; + url.searchParams.set('ddtags', tags.join(',')); + return url.toString(); +}; + +// The request of one part: an `event` part describing it, and the gzipped `file` part. +export const getPartData = ( + part: UploadPart, + context: SymbolTableUploadContext, + payload: SymbolTablePayload, + uploadId: string, +) => { + const event = { + ddsource: 'dd_debugger', + service: context.service, + version: context.version, + language: payload.language, + runtimeId: uploadId, + type: 'symdb', + uploadId, + batchNum: part.batchNum, + final: part.final, + attachmentSize: part.file.length, + }; + const eventContent = JSON.stringify(event); + const eventBlob = new Blob([eventContent], { type: 'application/json' }); + const fileBlob = new Blob([part.file], { type: 'application/gzip' }); + + const form = new FormData(); + form.append('event', eventBlob, 'event.json'); + form.append('file', fileBlob, 'file.gz'); + + const headers = getOriginHeaders({ + bundler: context.bundlerName, + plugin: 'live-debugger', + version: context.pluginVersion, + }); + + return { data: form, headers }; +}; + +/** + * Upload the symbol table to SymDB, one part at a time, and return the number of parts. Each build + * uses a new upload ID, which SymDB also uses as the runtime ID, so a new upload for the same + * service and version replaces the previous one. + */ +export const uploadSymbolTable = async ( + payload: SymbolTablePayload, + context: SymbolTableUploadContext, + maxBytes: number, + log: Logger, +): Promise => { + const uploadId = randomUUID(); + const parts = await createUploadParts(payload, context, uploadId, maxBytes); + const url = getIntakeUrl(context); + + for (const part of parts) { + // In order, so the parts of an upload arrive in sequence. + // eslint-disable-next-line no-await-in-loop + await doRequest({ + url, + method: 'POST', + auth: { apiKey: context.apiKey }, + getData: () => getPartData(part, context, payload, uploadId), + onRetry: (error, attempt) => { + log.debug( + `Retrying the upload of symbol table part ${part.batchNum} (attempt ${attempt}): ${error.message}`, + ); + }, + }); + } + + return parts.length; +}; diff --git a/packages/plugins/live-debugger/src/symbolTable.integration.test.ts b/packages/plugins/live-debugger/src/symbolTable.integration.test.ts index d1eb48702..85ae012e9 100644 --- a/packages/plugins/live-debugger/src/symbolTable.integration.test.ts +++ b/packages/plugins/live-debugger/src/symbolTable.integration.test.ts @@ -10,8 +10,10 @@ import { getOutDir, getTempWorkingDir } from '@dd/tests/_jest/helpers/env'; import { defaultPluginOptions } from '@dd/tests/_jest/helpers/mocks'; import { BUNDLERS, getBundlerConfig } from '@dd/tests/_jest/helpers/runBundlers'; import { buildWithRspack, buildWithWebpack, configRspack, configWebpack } from '@dd/tools/bundlers'; +import { execFileSync } from 'child_process'; import fsp from 'fs/promises'; import { glob } from 'glob'; +import nock from 'nock'; import path from 'path'; import { SYMBOL_TABLE_PATH_ENV } from './constants'; @@ -123,6 +125,33 @@ const createFixture = async (workingDir: string) => { await mkdir(gitDir); }; +// Turn the fixture into a git repository with a remote, for the upload. +const REPOSITORY_URL = 'https://github.com/DataDog/symbol-table-fixture'; +const initGitRepository = (workingDir: string) => { + const commands = [ + ['init', '-q'], + ['remote', 'add', 'origin', REPOSITORY_URL], + ['add', '-A'], + [ + '-c', + 'user.name=fixture', + '-c', + 'user.email=fixture@example.com', + 'commit', + '-q', + '-m', + 'init', + ], + ]; + for (const args of commands) { + execFileSync('git', args, { cwd: workingDir }); + } + const commitSha = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: workingDir }); + // Git configuration can rewrite remote URLs (`insteadOf`), so read it back like the git plugin. + const remoteUrl = execFileSync('git', ['remote', 'get-url', 'origin'], { cwd: workingDir }); + return { commitSha: commitSha.toString().trim(), remoteUrl: remoteUrl.toString().trim() }; +}; + // Build with the symbol table dump enabled, and return the dump. const buildWithSymbolTable = async (dumpPath: string, build: () => Promise) => { process.env[SYMBOL_TABLE_PATH_ENV] = dumpPath; @@ -404,4 +433,75 @@ describe('Live Debugger symbol table', () => { expect(triple?.endLine).toBeLessThanOrEqual(10); }); }); + describe('upload', () => { + type UploadRequest = { path: string; apiKey: string | string[] | undefined }; + const uploads: Partial> = {}; + const buildErrors: string[] = []; + let uploadDir: string; + let repository: { commitSha: string; remoteUrl: string }; + + beforeAll(async () => { + const seed = `${Math.abs(jest.getSeed())}.symbol-table-upload`; + uploadDir = getTempWorkingDir(seed); + await createFixture(uploadDir); + repository = initGitRepository(uploadDir); + const buildRoot = path.resolve(uploadDir, APP_DIR); + + let requests: UploadRequest[] = []; + nock('https://debugger-intake.datadoghq.com') + .persist() + .post('/api/v2/debugger') + .query(true) + .reply(function recordUpload() { + requests.push({ path: this.req.path, apiKey: this.req.headers['dd-api-key'] }); + return [202, '{}']; + }); + + // Run the builds sequentially to ease the resources usage. + for (const { name: bundlerName, run } of BUNDLERS) { + const isXpack = XPACK_BUNDLERS.includes(bundlerName); + const entry = { main: isXpack ? './src/main-xpack.js' : './src/main.js' }; + const plugins = isXpack ? [sourceMapLoaderPlugin] : []; + const config = getBundlerConfig( + bundlerName, + buildRoot, + { + ...PLUGIN_OVERRIDES, + enableGit: true, + metadata: { version: '1.0.0', service: 'fixture-service' }, + }, + { entry, plugins }, + ); + // eslint-disable-next-line no-await-in-loop + const { errors } = await run(buildRoot, config); + buildErrors.push(...errors); + uploads[bundlerName] = requests; + requests = []; + } + }, 200000); + + afterAll(async () => { + nock.cleanAll(); + if (!process.env.NO_CLEANUP) { + await rm(uploadDir); + } + }); + + test('Should build without errors.', () => { + expect(buildErrors).toEqual([]); + }); + + test.each(BUNDLERS)('Should upload the table once with $name.', ({ name: bundlerName }) => { + const requests = uploads[bundlerName] ?? []; + const [request] = requests; + + expect(requests).toHaveLength(1); + expect(request.apiKey).toBe(defaultPluginOptions.auth.apiKey); + + const url = new URL(request.path, 'https://debugger-intake.datadoghq.com'); + expect(url.searchParams.get('ddtags')).toBe( + `git.repository_url:${repository.remoteUrl},git.commit.sha:${repository.commitSha}`, + ); + }); + }); }); diff --git a/packages/plugins/live-debugger/src/types.ts b/packages/plugins/live-debugger/src/types.ts index fd93181cc..3d4b58dcd 100644 --- a/packages/plugins/live-debugger/src/types.ts +++ b/packages/plugins/live-debugger/src/types.ts @@ -35,6 +35,10 @@ export type LiveDebuggerOptions = { export type LiveDebuggerOptionsWithDefaults = { version: string | undefined; + // From the top-level `metadata.service`. + service: string | undefined; + // Whether git data is collected (see the top-level `enableGit` option). + enableGit: boolean; // Whether output chunks get a debug ID (see `rum.sourceCodeContext.debugId`). debugId: boolean; include: (string | RegExp)[]; diff --git a/packages/plugins/live-debugger/src/validate.test.ts b/packages/plugins/live-debugger/src/validate.test.ts index 955a22a71..edce3f576 100644 --- a/packages/plugins/live-debugger/src/validate.test.ts +++ b/packages/plugins/live-debugger/src/validate.test.ts @@ -34,6 +34,8 @@ describe('validateOptions', () => { input: makeConfig(undefined), expected: { version: undefined, + service: undefined, + enableGit: false, debugId: false, include: [], exclude: expect.arrayContaining([/\/node_modules\//]), @@ -49,6 +51,8 @@ describe('validateOptions', () => { input: makeConfig({}), expected: { version: undefined, + service: undefined, + enableGit: false, debugId: false, include: [], exclude: expect.arrayContaining([/\/node_modules\//]), @@ -64,6 +68,8 @@ describe('validateOptions', () => { input: makeConfig({}, { version: '1.0.0' }), expected: { version: '1.0.0', + service: undefined, + enableGit: false, debugId: false, include: [], exclude: expect.arrayContaining([/\/node_modules\//]), @@ -114,6 +120,52 @@ describe('validateOptions', () => { expect(validateOptions(input, mockLogger)).toEqual(expected); }); + const serviceCases = [ + { + description: 'use metadata.service', + input: makeConfig({}, { service: 'cart' }), + expected: 'cart', + }, + { + description: 'not use rum.sourceCodeContext.service', + input: { + ...makeConfig({}), + rum: { sourceCodeContext: { service: 'checkout' } }, + } satisfies Options, + expected: undefined, + }, + ]; + + test.each(serviceCases)('should $description', ({ input, expected }) => { + expect(validateOptions(input, mockLogger).service).toBe(expected); + }); + + const gitCases = [ + { + description: 'collect git data with an API key', + input: { ...makeConfig({}), auth: { apiKey: 'api-key' } } satisfies Options, + expected: true, + }, + { + description: 'not collect git data without an API key', + input: makeConfig({}), + expected: false, + }, + { + description: 'not collect git data when git is disabled', + input: { + ...makeConfig({}), + auth: { apiKey: 'api-key' }, + enableGit: false, + } satisfies Options, + expected: false, + }, + ]; + + test.each(gitCases)('should $description', ({ input, expected }) => { + expect(validateOptions(input, mockLogger).enableGit).toBe(expected); + }); + it('should apply all default exclude patterns', () => { const result = validateOptions(makeConfig({}), mockLogger); expect(result.exclude).toHaveLength(9); diff --git a/packages/plugins/live-debugger/src/validate.ts b/packages/plugins/live-debugger/src/validate.ts index ce064a309..584135153 100644 --- a/packages/plugins/live-debugger/src/validate.ts +++ b/packages/plugins/live-debugger/src/validate.ts @@ -3,6 +3,7 @@ // Copyright 2019-Present Datadog, Inc. import { areDebugIdsEnabled } from '@dd/core/helpers/debugId'; +import { shouldGetGitInfo } from '@dd/core/helpers/plugins'; import type { Logger, Options } from '@dd/core/types'; import chalk from 'chalk'; @@ -126,6 +127,8 @@ export const validateOptions = (config: Options, log: Logger): LiveDebuggerOptio // Build the final configuration with defaults return { version: metadataVersion, + service: config.metadata?.service || undefined, + enableGit: shouldGetGitInfo(config), debugId: areDebugIdsEnabled(config, log), include: pluginConfig.include ?? [], exclude: pluginConfig.exclude || [ diff --git a/packages/plugins/rum/README.md b/packages/plugins/rum/README.md index 17e2241d7..9b614a9e8 100644 --- a/packages/plugins/rum/README.md +++ b/packages/plugins/rum/README.md @@ -42,7 +42,7 @@ rum?: { } | { debugId?: false; - service: string; + service?: string; version?: string; }; } @@ -130,6 +130,8 @@ To upload the corresponding source maps directly during the build, also set `sou The existing `rum.sourceCodeContext.debugId` configuration remains supported. Service and version can be provided alongside `debugId`; they identify and filter the application or micro-frontend, while the debug ID independently associates stack frames with source maps. +When `rum.sourceCodeContext.service` or `rum.sourceCodeContext.version` isn't set, the plugin falls back to the shared top-level [`metadata.service`](/README.md#metadataservice) and [`metadata.version`](/README.md#metadataversion). Without `debugId`, a service is required in one of the two places. + ### rum.sourceCodeContext.debugId > default: `false` diff --git a/packages/plugins/rum/src/types.ts b/packages/plugins/rum/src/types.ts index 7eaa37abc..f4d9e6126 100644 --- a/packages/plugins/rum/src/types.ts +++ b/packages/plugins/rum/src/types.ts @@ -15,7 +15,9 @@ type DebugIdSourceCodeContextOptions = { type ServiceVersionSourceCodeContextOptions = { debugId?: false; - service: string; + // Optional: when omitted, the validator falls back to the shared + // top-level `metadata.service`. At least one of the two must be set. + service?: string; version?: string; }; diff --git a/packages/plugins/rum/src/validate.test.ts b/packages/plugins/rum/src/validate.test.ts index ba1021413..b836758ba 100644 --- a/packages/plugins/rum/src/validate.test.ts +++ b/packages/plugins/rum/src/validate.test.ts @@ -120,6 +120,39 @@ describe('sourceCodeContext validation', () => { ); }); + test('should fall back to metadata.service when sourceCodeContext.service is unset', () => { + const pluginOptions = { + ...defaultPluginOptions, + metadata: { service: 'checkout', version: '1.2.3' }, + rum: { sourceCodeContext: {} }, + }; + const result = validateSourceCodeContextOptions(pluginOptions); + expect(result.errors).toHaveLength(0); + expect(result.config).toEqual({ service: 'checkout', version: '1.2.3' }); + }); + + test('should fall back to metadata.service alongside a debug ID', () => { + const pluginOptions = { + ...defaultPluginOptions, + metadata: { service: 'checkout', version: '1.2.3' }, + rum: { sourceCodeContext: { debugId: true } as const }, + }; + const result = validateSourceCodeContextOptions(pluginOptions); + expect(result.errors).toHaveLength(0); + expect(result.config).toEqual({ debugId: true, service: 'checkout', version: '1.2.3' }); + }); + + test('should prefer an explicit sourceCodeContext.service over metadata.service', () => { + const pluginOptions = { + ...defaultPluginOptions, + metadata: { service: 'checkout' }, + rum: { sourceCodeContext: { service: 'cart' } }, + }; + const result = validateSourceCodeContextOptions(pluginOptions); + expect(result.errors).toHaveLength(0); + expect(result.config).toEqual({ service: 'cart', version: undefined }); + }); + test('should fall back to metadata.version when sourceCodeContext.version is unset', () => { const pluginOptions = { ...defaultPluginOptions, diff --git a/packages/plugins/rum/src/validate.ts b/packages/plugins/rum/src/validate.ts index 06595356f..4ced59aaf 100644 --- a/packages/plugins/rum/src/validate.ts +++ b/packages/plugins/rum/src/validate.ts @@ -167,17 +167,23 @@ export const validateSourceCodeContextOptions = ( } const cfg: SourceCodeContextOptions = validatedOptions.sourceCodeContext; + // Resolve `service`: prefer the plugin-specific option, then fall back to + // the shared top-level `metadata.service`. + const service = cfg.service || options.metadata?.service; if (cfg.debugId === true) { toReturn.config = { ...cfg, - version: cfg.version || (cfg.service ? options.metadata?.version : undefined), + ...(service ? { service } : {}), + version: cfg.version || (service ? options.metadata?.version : undefined), }; return toReturn; } - if (!cfg.service || typeof cfg.service !== 'string') { - toReturn.errors.push(`Missing ${red('"rum.sourceCodeContext.service"')}.`); + if (!service || typeof service !== 'string') { + toReturn.errors.push( + `Missing ${red('"rum.sourceCodeContext.service"')} (set it directly or via ${red('"metadata.service"')}).`, + ); } if (toReturn.errors.length === 0) { @@ -185,6 +191,7 @@ export const validateSourceCodeContextOptions = ( // the shared top-level `metadata.version`. toReturn.config = { ...cfg, + service, version: cfg.version || options.metadata?.version, }; } diff --git a/packages/tests/src/_jest/helpers/mocks.ts b/packages/tests/src/_jest/helpers/mocks.ts index 6a51a4f57..768c77af2 100644 --- a/packages/tests/src/_jest/helpers/mocks.ts +++ b/packages/tests/src/_jest/helpers/mocks.ts @@ -309,12 +309,16 @@ export const getContextMock = (overrides: Partial = {}): GlobalCo buildRoot: '/cwd/path', env: 'test', getLogger: jest.fn(() => getMockLogger()), + gitPending: false, + gitReady: Promise.resolve(), asyncHook: jest.fn(), addMetric: jest.fn(), hook: jest.fn(), inject: jest.fn(), markArtifactsPending: jest.fn(), markArtifactsReady: jest.fn(), + markGitPending: jest.fn(), + markGitReady: jest.fn(), pluginNames: [], sendLog: jest.fn(), plugins: [], diff --git a/packages/tools/src/commands/integrity/readme.ts b/packages/tools/src/commands/integrity/readme.ts index cdb85a17f..49b2c83f6 100644 --- a/packages/tools/src/commands/integrity/readme.ts +++ b/packages/tools/src/commands/integrity/readme.ts @@ -363,6 +363,7 @@ export const updateReadmes = async (plugins: Workspace[], bundlers: Workspace[]) logLevel?: 'debug' | 'info' | 'warn' | 'error' | 'none', metadata?: { name?: string; + service?: string; version?: string; }; sourcemaps?: