diff --git a/.changeset/patterns-max-body-chars.md b/.changeset/patterns-max-body-chars.md new file mode 100644 index 000000000..38cf774c5 --- /dev/null +++ b/.changeset/patterns-max-body-chars.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/api': patch +--- + +feat(mcp): `maxBodyChars` on `clickstack_event_patterns` and +`clickstack_emerging_signals` mines only the start of each body, so long stack +traces and dumps are cut in the query and their patterns can be drilled into diff --git a/packages/api/src/mcp/__tests__/emergingSignalsTool.int.test.ts b/packages/api/src/mcp/__tests__/emergingSignalsTool.int.test.ts index 59ccb7301..fb7274446 100644 --- a/packages/api/src/mcp/__tests__/emergingSignalsTool.int.test.ts +++ b/packages/api/src/mcp/__tests__/emergingSignalsTool.int.test.ts @@ -218,6 +218,24 @@ describe('MCP Emerging Signals Tool', () => { expect(checkoutEmerging).toBeUndefined(); }); + it('mines only the first maxBodyChars characters of each body', async () => { + const result = await callTool(client, 'clickstack_emerging_signals', { + sourceId: logSource._id.toString(), + currentStartTime: new Date(now - 5 * 60_000).toISOString(), + currentEndTime: new Date(now).toISOString(), + baselineStartTime: new Date(now - 10 * 60_000).toISOString(), + baselineEndTime: new Date(now - 5 * 60_000).toISOString(), + sampleSize: 2000, + maxBodyChars: 12, + }); + + expect(result.isError).toBeFalsy(); + const output = JSON.parse(getFirstText(result)); + expect(output.summary.bodyColumn).toBe('leftUTF8(Body, 12)'); + const emerging: EmergingPattern[] = output.emerging; + expect(emerging.map(p => p.sample)).toEqual(['shadow-eval ']); + }); + it('reports nothing novel when both windows share the same patterns', async () => { // Two non-overlapping halves of the baseline-only region (now-10min .. // now-5min). Both contain only the steady-state checkout template and diff --git a/packages/api/src/mcp/__tests__/queryTool.int.test.ts b/packages/api/src/mcp/__tests__/queryTool.int.test.ts index 0e613fd5b..0169fe6c1 100644 --- a/packages/api/src/mcp/__tests__/queryTool.int.test.ts +++ b/packages/api/src/mcp/__tests__/queryTool.int.test.ts @@ -1301,6 +1301,36 @@ describe('MCP Query Tools', () => { expect(p.drilldownUnavailable).toContain('longer than 4000'); }); + it('mines the start of long bodies with maxBodyChars and drills by that prefix', async () => { + await bulkInsertLogs( + [101, 202, 303].map(id => ({ + Body: `Error: job ${id} failed ${'at frame '.repeat(1000)}${id}`, + ServiceName: 'trace', + SeverityText: 'ERROR', + Timestamp: base, + })), + ); + const output = await mine({ + where: 'ServiceName:trace', + maxBodyChars: 27, + }); + expect(output.summary.bodyColumn).toBe('leftUTF8(Body, 27)'); + expect(output.patterns).toHaveLength(1); + const [p] = output.patterns; + expect(p.pattern).toBe('Error: job <*> failed at fr'); + for (const sample of p.samples) expect(sample).toHaveLength(27); + const rows = await drill(output, p); + expect(rows).toHaveLength(3); + for (const r of rows) expect(r.Body.length).toBeGreaterThan(9000); + + const result = await callTool(client, 'clickstack_event_patterns', { + sourceId: logSource._id.toString(), + ...window, + maxBodyChars: 2.5, + }); + expect(result.isError).toBe(true); + }); + it('separates sampled counts from estimates', async () => { const output = await mine({ where: 'ServiceName:svc-a', diff --git a/packages/api/src/mcp/tools/query/emergingSignals.ts b/packages/api/src/mcp/tools/query/emergingSignals.ts index be00d9637..f0687028f 100644 --- a/packages/api/src/mcp/tools/query/emergingSignals.ts +++ b/packages/api/src/mcp/tools/query/emergingSignals.ts @@ -75,6 +75,17 @@ const emergingSignalsSchema = z.object({ 'if omitted (Body for logs, SpanName for traces). ' + 'Example: "Body", "SpanName", "SpanAttributes[\'http.url\']".', ), + maxBodyChars: z + .number() + .int() + .min(1) + .max(1_000_000) + .optional() + .describe( + 'Mine only the first maxBodyChars characters of each value, e.g. 1000 for long ' + + 'stack traces or request dumps. Long values are cut in the query, so less data ' + + 'is read, and each sample is the cut value.', + ), currentStartTime: z .string() .describe( @@ -204,6 +215,7 @@ export function registerEmergingSignals({ where: input.where, whereLanguage: input.whereLanguage, bodyExpression: input.bodyExpression, + maxBodyChars: input.maxBodyChars, sampleSize: input.sampleSize, trendBuckets: 0, }; diff --git a/packages/api/src/mcp/tools/query/eventPatterns.ts b/packages/api/src/mcp/tools/query/eventPatterns.ts index 6e09c127c..eb3bba768 100644 --- a/packages/api/src/mcp/tools/query/eventPatterns.ts +++ b/packages/api/src/mcp/tools/query/eventPatterns.ts @@ -43,7 +43,20 @@ const eventPatternsSchema = z.object({ 'Examples: "Body", "SpanName", "LogAttributes[\'exception.type\']", ' + "\"JSONExtractString(Body, 'error', 'message')\". Other function calls are rejected. " + "Rows where the value is '' (missing key, invalid JSON) are mined as an empty " + - 'pattern; add where " != \'\'" (whereLanguage:"sql") to exclude them.', + 'pattern; add where " != \'\'" (whereLanguage:"sql") to exclude them. ' + + 'To mine the start of long values, use maxBodyChars.', + ), + maxBodyChars: z + .number() + .int() + .min(1) + .max(1_000_000) + .optional() + .describe( + 'Mine only the first maxBodyChars characters of each value, e.g. 1000 for long ' + + 'stack traces or request dumps. Long values are cut in the query, so less data ' + + 'is read; samples are the cut values, and whereSnippet matches events by that ' + + 'prefix (summary.bodyColumn shows the leftUTF8 expression used).', ), topN: z .number() @@ -124,6 +137,7 @@ export function registerEventPatterns({ where: input.where, whereLanguage: input.whereLanguage, bodyExpression: input.bodyExpression, + maxBodyChars: input.maxBodyChars, sampleSize: input.sampleSize, topN: input.topN, trendBuckets: input.trendBuckets, diff --git a/packages/api/src/mcp/tools/query/runEventPatterns.ts b/packages/api/src/mcp/tools/query/runEventPatterns.ts index a6d6fbb50..ba8edb76b 100644 --- a/packages/api/src/mcp/tools/query/runEventPatterns.ts +++ b/packages/api/src/mcp/tools/query/runEventPatterns.ts @@ -69,6 +69,8 @@ export async function mineWindowPatterns( where?: string; whereLanguage?: 'lucene' | 'sql'; bodyExpression?: string; + /** Mine only the first maxBodyChars characters of each body. */ + maxBodyChars?: number; sampleSize?: number; trendBuckets?: number; abortSignal?: AbortSignal; @@ -152,6 +154,12 @@ export async function mineWindowPatterns( ), }; } + // Truncate in the query, so long bodies are not transferred, and in the + // drill-down predicate, so it matches the prefix the template came from. + // The tool schemas bound maxBodyChars to an integer from 1 to 1,000,000. + if (options?.maxBodyChars != null) { + bodyColumn = `leftUTF8(${bodyColumn}, ${options.maxBodyChars})`; + } const filters = getSourceTableFilters(source); const tableFilterSql = andSql( @@ -392,6 +400,7 @@ export async function runEventPatterns( where?: string; whereLanguage?: 'lucene' | 'sql'; bodyExpression?: string; + maxBodyChars?: number; sampleSize?: number; topN?: number; trendBuckets?: number; @@ -416,6 +425,7 @@ export async function runEventPatterns( where: options?.where, whereLanguage: options?.whereLanguage, bodyExpression: options?.bodyExpression, + maxBodyChars: options?.maxBodyChars, sampleSize: options?.sampleSize, trendBuckets, abortSignal: options?.abortSignal,