From 96d57d5b50987157f8beb07bbd6e1f4b0ae977b9 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:10:31 -0500 Subject: [PATCH 01/10] perf: read only median-sized parts when discovering Map keys from a text index --- .changeset/map-key-index-median-parts.md | 12 +++++ packages/api/openapi.json | 14 ++++++ .../src/mcp/tools/sources/describeSource.ts | 4 ++ packages/api/src/mcp/tools/sources/schemas.ts | 11 +++++ packages/api/src/models/source.ts | 2 + .../routers/api/__tests__/sources.int.test.ts | 37 +++++++++++++++ .../src/routers/external-api/v2/sources.ts | 12 +++++ .../Sources/SourceForm/LogTableModelForm.tsx | 2 + .../SourceForm/MapKeyIndexMaxPartsFormRow.tsx | 36 +++++++++++++++ .../SourceForm/TraceTableModelForm.tsx | 2 + .../src/__tests__/metadata.test.ts | 40 ++++++++++++++++ packages/common-utils/src/core/metadata.ts | 46 ++++++++++++++++--- packages/common-utils/src/types.ts | 11 +++++ 13 files changed, 223 insertions(+), 6 deletions(-) create mode 100644 .changeset/map-key-index-median-parts.md create mode 100644 packages/app/src/components/Sources/SourceForm/MapKeyIndexMaxPartsFormRow.tsx diff --git a/.changeset/map-key-index-median-parts.md b/.changeset/map-key-index-median-parts.md new file mode 100644 index 0000000000..de0c160850 --- /dev/null +++ b/.changeset/map-key-index-median-parts.md @@ -0,0 +1,12 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/api': patch +'@hyperdx/app': patch +--- + +perf: read only 20 median-sized parts when discovering Map keys from a text index + +Map key discovery read the text index dictionary of every part in the time +range, which is slow on tables with many parts. It now reads the 20 parts +closest to the median size. Log and trace sources can change the count with +the new "Map key index parts" setting (`mapKeyIndexMaxParts`). diff --git a/packages/api/openapi.json b/packages/api/openapi.json index ba618f6178..624c22afa9 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -5655,6 +5655,13 @@ "nullable": true, "example": "auto" }, + "mapKeyIndexMaxParts": { + "type": "integer", + "minimum": 1, + "description": "How many data parts Map attribute key discovery reads from a text index, picked around the median part size. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20.", + "nullable": true, + "example": 20 + }, "highlightedTraceAttributeExpressions": { "type": "array", "description": "Expressions defining trace-level attributes which are displayed in the trace view for the selected trace.", @@ -5915,6 +5922,13 @@ "nullable": true, "example": "auto" }, + "mapKeyIndexMaxParts": { + "type": "integer", + "minimum": 1, + "description": "How many data parts Map attribute key discovery reads from a text index, picked around the median part size. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20.", + "nullable": true, + "example": 20 + }, "highlightedTraceAttributeExpressions": { "type": "array", "description": "Expressions defining trace-level attributes which are displayed in the trace view for the selected trace.", diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts index 881a75ed80..49d09879bb 100644 --- a/packages/api/src/mcp/tools/sources/describeSource.ts +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -306,6 +306,10 @@ async function describeSourceSchema( maxKeys: 50, connectionId, metadataMVs, + mapKeyIndexMaxParts: + 'mapKeyIndexMaxParts' in source + ? source.mapKeyIndexMaxParts + : undefined, dateRange, timestampValueExpression, signal, diff --git a/packages/api/src/mcp/tools/sources/schemas.ts b/packages/api/src/mcp/tools/sources/schemas.ts index c095cb6a15..236eaad947 100644 --- a/packages/api/src/mcp/tools/sources/schemas.ts +++ b/packages/api/src/mcp/tools/sources/schemas.ts @@ -475,6 +475,17 @@ const mcpSaveSourceSchema = z.object({ 'Log and trace sources. Rollup tables used for fast field discovery and ' + 'value autocomplete in filters and search suggestions.', ), + mapKeyIndexMaxParts: z + .number() + .int() + .min(1) + .optional() + .describe( + 'Log and trace sources. How many data parts Map attribute key discovery ' + + 'reads from a text index, picked around the median part size. Raise it ' + + 'if attribute keys are missing from filters; lower it if field loading ' + + 'is slow. Omit for the default of 20.', + ), }); export type McpSaveSourceInput = z.infer; diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index 42d8e44b7a..53d90f0fb9 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -190,6 +190,7 @@ export const LogSource = Source.discriminator( type: MetadataMaterializedViewsMongoSchema, default: undefined, }, + mapKeyIndexMaxParts: Number, orderByExpression: String, }), ); @@ -242,6 +243,7 @@ export const TraceSource = Source.discriminator( type: MetadataMaterializedViewsMongoSchema, default: undefined, }, + mapKeyIndexMaxParts: Number, orderByExpression: String, }), ); diff --git a/packages/api/src/routers/api/__tests__/sources.int.test.ts b/packages/api/src/routers/api/__tests__/sources.int.test.ts index ea83a0874d..da05d29e2b 100644 --- a/packages/api/src/routers/api/__tests__/sources.int.test.ts +++ b/packages/api/src/routers/api/__tests__/sources.int.test.ts @@ -1672,4 +1672,41 @@ describe('sources router', () => { expect(sources).toHaveLength(0); }); }); + + describe('mapKeyIndexMaxParts field', () => { + it('POST / - persists mapKeyIndexMaxParts and PUT /:id updates it', async () => { + const { agent } = await getLoggedInAgent(server); + + const created = await agent + .post('/sources') + .send({ ...MOCK_SOURCE, mapKeyIndexMaxParts: 50 }) + .expect(200); + expect(created.body.mapKeyIndexMaxParts).toBe(50); + + await agent + .put(`/sources/${created.body.id}`) + .send({ ...MOCK_SOURCE, id: created.body.id, mapKeyIndexMaxParts: 5 }) + .expect(200); + + const updated = await Source.findById(created.body.id).lean(); + if (updated?.kind !== SourceKind.Log) { + throw new Error(`expected Log source, got ${updated?.kind}`); + } + expect(updated.mapKeyIndexMaxParts).toBe(5); + }); + + it.each([0, 2.5, -1])( + 'POST / - rejects mapKeyIndexMaxParts %s', + async mapKeyIndexMaxParts => { + const { agent } = await getLoggedInAgent(server); + + await agent + .post('/sources') + .send({ ...MOCK_SOURCE, mapKeyIndexMaxParts }) + .expect(400); + + expect(await Source.find({}).lean()).toHaveLength(0); + }, + ); + }); }); diff --git a/packages/api/src/routers/external-api/v2/sources.ts b/packages/api/src/routers/external-api/v2/sources.ts index 8da4fd0b4e..5301f87814 100644 --- a/packages/api/src/routers/external-api/v2/sources.ts +++ b/packages/api/src/routers/external-api/v2/sources.ts @@ -417,6 +417,12 @@ function formatExternalSource(source: SourceDocument) { * description: Controls whether lucene rendering uses ClickHouse text indices via hasAllTokens() against the implicit column. "auto" detects a covering index at query time, "enabled" forces text index usage, "disabled" forces a LIKE/hasToken fallback. * nullable: true * example: auto + * mapKeyIndexMaxParts: + * type: integer + * minimum: 1 + * description: How many data parts Map attribute key discovery reads from a text index, picked around the median part size. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20. + * nullable: true + * example: 20 * highlightedTraceAttributeExpressions: * type: array * description: Expressions defining trace-level attributes which are displayed in the trace view for the selected trace. @@ -623,6 +629,12 @@ function formatExternalSource(source: SourceDocument) { * description: Controls whether lucene rendering uses ClickHouse text indices via hasAllTokens() against the implicit column. "auto" detects a covering index at query time, "enabled" forces text index usage, "disabled" forces a LIKE/hasToken fallback. * nullable: true * example: auto + * mapKeyIndexMaxParts: + * type: integer + * minimum: 1 + * description: How many data parts Map attribute key discovery reads from a text index, picked around the median part size. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20. + * nullable: true + * example: 20 * highlightedTraceAttributeExpressions: * type: array * description: Expressions defining trace-level attributes which are displayed in the trace view for the selected trace. diff --git a/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx b/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx index 58c2826ba7..e9f6cf0a93 100644 --- a/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx +++ b/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx @@ -21,6 +21,7 @@ import { import { ExpressionFormRow } from './ExpressionFormRow'; import { FormRow } from './FormRow'; import { HighlightedAttributeExpressionsFormRow } from './HighlightedAttributes'; +import { MapKeyIndexMaxPartsFormRow } from './MapKeyIndexMaxPartsFormRow'; import { MaterializedViewsFormSection, MetadataMaterializedViewsFormSection, @@ -303,6 +304,7 @@ export function LogTableModelForm(props: TableModelProps) { /> + ; +}) { + return ( + + ( + field.onChange(v === '' ? undefined : v)} + min={1} + allowDecimal={false} + allowNegative={false} + placeholder={String(DEFAULT_MAP_KEY_INDEX_MAX_PARTS)} + error={fieldState.error?.message} + /> + )} + /> + + ); +} diff --git a/packages/app/src/components/Sources/SourceForm/TraceTableModelForm.tsx b/packages/app/src/components/Sources/SourceForm/TraceTableModelForm.tsx index 42ba611479..6386e58980 100644 --- a/packages/app/src/components/Sources/SourceForm/TraceTableModelForm.tsx +++ b/packages/app/src/components/Sources/SourceForm/TraceTableModelForm.tsx @@ -17,6 +17,7 @@ import { import { ExpressionFormRow } from './ExpressionFormRow'; import { FormRow } from './FormRow'; import { HighlightedAttributeExpressionsFormRow } from './HighlightedAttributes'; +import { MapKeyIndexMaxPartsFormRow } from './MapKeyIndexMaxPartsFormRow'; import { MaterializedViewsFormSection, MetadataMaterializedViewsFormSection, @@ -397,6 +398,7 @@ export function TraceTableModelForm(props: TableModelProps) { /> + { expect(mockClickhouseClient.query).toHaveBeenCalledTimes(1); }, ); + + it.each(textIndexLookups)( + 'reads only the parts nearest the median size on the %s text index path', + async (_, textIndexLookup) => { + const md = new Metadata(mockClickhouseClient, new MetadataCache()); + jest.spyOn(md, 'getServerVersion').mockResolvedValue([26, 3, 0, 0]); + jest + .spyOn(md, 'getMapColumnTextIndexes') + .mockResolvedValue(textIndexLookup); + (mockClickhouseClient.query as jest.Mock).mockResolvedValue({ + json: () => Promise.resolve({ data: [{ key: 'http.method' }] }), + }); + const args = { + databaseName: 'default', + tableName: 'otel_logs', + column: 'LogAttributes', + connectionId: 'test_connection', + dateRange: [ + new Date('2024-01-01T00:00:00Z'), + new Date('2024-01-01T01:00:00Z'), + ] as [Date, Date], + timestampValueExpression: 'Timestamp', + }; + + await md.getMapKeys(args); + await md.getMapKeys({ ...args, mapKeyIndexMaxParts: 5 }); + + // A different limit is a different result, not a cache hit. + expect(mockClickhouseClient.query).toHaveBeenCalledTimes(2); + const calls = (mockClickhouseClient.query as jest.Mock).mock.calls; + const partsLimits = calls.map(([opts]) => { + const match = opts.query.match( + /row_number\(\) OVER \(ORDER BY rows, name\)[\s\S]+ORDER BY abs\(rn - \(cnt \+ 1\) \/ 2\), name\s+LIMIT \{(\w+):Int32\}/, + ); + expect(match).not.toBeNull(); + return opts.query_params[match[1]]; + }); + expect(partsLimits).toEqual([20, 5]); + }, + ); }); describe('getMapValues', () => { diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 6eb9ac386e..170c6d078c 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -65,6 +65,9 @@ import { // Between 1e6 - 5e6 is a good range. export const DEFAULT_METADATA_MAX_ROWS_TO_READ = 3e6; const DEFAULT_MAX_KEYS = 1000; +// shortcut: keys found only in the smallest or largest parts of the range can +// be missed; raise it per source (mapKeyIndexMaxParts) if keys go missing. +export const DEFAULT_MAP_KEY_INDEX_MAX_PARTS = 20; // Cap keys per dispatched query: each key is another operation for the db to fetch, and simply fetching all keys at once can be too much for the db to handle. export const GET_ALL_KEY_VALUES_CHUNK_SIZE = 100; @@ -896,21 +899,37 @@ export class Metadata { tableName, dateRange, timestampValueExpression, + maxParts, }: { databaseName: string; tableName: string; dateRange?: [Date, Date]; timestampValueExpression?: string; + /** Keep only the `maxParts` overlapping parts closest to the median row count. */ + maxParts?: number; }): Promise { if (!dateRange || !timestampValueExpression) return chSql`1`; const startTime = chSql`fromUnixTimestamp64Milli(${{ Int64: dateRange[0].getTime() }})`; const endTime = chSql`fromUnixTimestamp64Milli(${{ Int64: dateRange[1].getTime() }})`; - return chSql`part_name IN ( - SELECT name + const overlappingParts = chSql` FROM system.parts WHERE database = ${{ String: databaseName }} AND table = ${{ String: tableName }} AND active=1 - AND ((min_time >= ${startTime} AND min_time <= ${endTime}) OR (max_time <= ${endTime} AND max_time >= ${startTime}) OR (min_time <= ${startTime} AND max_time >= ${endTime})) + AND ((min_time >= ${startTime} AND min_time <= ${endTime}) OR (max_time <= ${endTime} AND max_time >= ${startTime}) OR (min_time <= ${startTime} AND max_time >= ${endTime}))`; + if (maxParts == null) { + return chSql`part_name IN (SELECT name ${overlappingParts})`; + } + // Rank by size and pick from the middle out: tiny unmerged parts hold few + // keys and the largest merged parts have the biggest dictionaries. + return chSql`part_name IN ( + SELECT name FROM ( + SELECT name, + row_number() OVER (ORDER BY rows, name) AS rn, + count() OVER () AS cnt + ${overlappingParts} + ) + ORDER BY abs(rn - (cnt + 1) / 2), name + LIMIT ${{ Int32: maxParts }} )`; } @@ -1075,6 +1094,7 @@ export class Metadata { dateRange: callerDateRange, timestampValueExpression, conditionSql, + mapKeyIndexMaxParts = DEFAULT_MAP_KEY_INDEX_MAX_PARTS, signal, }: { databaseName: string; @@ -1087,6 +1107,8 @@ export class Metadata { dateRange?: [Date, Date]; timestampValueExpression?: string; conditionSql?: string; + /** Parts the text index paths read; see `partsOverlapFilter`. */ + mapKeyIndexMaxParts?: number; signal?: AbortSignal; }) { inlineNonNegativeInt(maxKeys, 'maxKeys'); @@ -1110,10 +1132,10 @@ export class Metadata { ? `${alignedDateRange[0].getTime()}.${alignedDateRange[1].getTime()}.${timestampValueExpression}` : ''; const unscopedCacheKey = metricName - ? `${connectionId}.${databaseName}.${tableName}.${column}.${metricName}.${dateRangeCacheSuffix}.${maxKeys}.keys` + ? `${connectionId}.${databaseName}.${tableName}.${column}.${metricName}.${dateRangeCacheSuffix}.${maxKeys}.${mapKeyIndexMaxParts}.keys` : alignedDateRange - ? `${connectionId}.${databaseName}.${tableName}.${column}.${alignedCacheSuffix}.${maxKeys}.keys` - : `${connectionId}.${databaseName}.${tableName}.${column}.${dateRangeCacheSuffix}.${maxKeys}.keys`; + ? `${connectionId}.${databaseName}.${tableName}.${column}.${alignedCacheSuffix}.${maxKeys}.${mapKeyIndexMaxParts}.keys` + : `${connectionId}.${databaseName}.${tableName}.${column}.${dateRangeCacheSuffix}.${maxKeys}.${mapKeyIndexMaxParts}.keys`; const cacheKey = conditionSql ? `${unscopedCacheKey}.${objectHash(conditionSql)}.condition` : unscopedCacheKey; @@ -1137,6 +1159,7 @@ export class Metadata { alignedDateRange, timestampValueExpression, conditionSql, + mapKeyIndexMaxParts, cacheKey, signal, }), @@ -1156,6 +1179,7 @@ export class Metadata { alignedDateRange, timestampValueExpression, conditionSql, + mapKeyIndexMaxParts, cacheKey, signal, }: { @@ -1170,6 +1194,7 @@ export class Metadata { alignedDateRange?: [Date, Date]; timestampValueExpression?: string; conditionSql?: string; + mapKeyIndexMaxParts: number; cacheKey: string; signal?: AbortSignal; }): Promise { @@ -1205,6 +1230,7 @@ export class Metadata { tableName, dateRange, timestampValueExpression, + maxParts: mapKeyIndexMaxParts, }); const index = textIndexInfo.key.indexName; // `cardinality` is the token's row count in the part, not a distinct count. @@ -1250,6 +1276,7 @@ export class Metadata { tableName, dateRange, timestampValueExpression, + maxParts: mapKeyIndexMaxParts, }); const index = textIndexInfo.kv.indexName; const separator = textIndexInfo.kv.separator; @@ -1965,6 +1992,7 @@ export class Metadata { connectionId, metricName, metadataMVs, + mapKeyIndexMaxParts, dateRange, timestampValueExpression, condition, @@ -2038,6 +2066,7 @@ export class Metadata { connectionId, metricName, metadataMVs, + mapKeyIndexMaxParts, dateRange, timestampValueExpression, conditionSql, @@ -3468,6 +3497,7 @@ export type TableConnection = { connectionId: string; metricName?: string; metadataMVs?: MetadataMaterializedViews; + mapKeyIndexMaxParts?: number; // Bounds Map key discovery; without it Map keys are skipped. #3037 timestampValueExpression?: string; }; @@ -3511,6 +3541,10 @@ export function tcFromSource(source?: TSource): TableConnection { source && (isLogSource(source) || isTraceSource(source)) ? source.metadataMaterializedViews : undefined, + mapKeyIndexMaxParts: + source && (isLogSource(source) || isTraceSource(source)) + ? source.mapKeyIndexMaxParts + : undefined, timestampValueExpression: source?.timestampValueExpression, }; } diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 23e58fab77..081d198141 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -2798,6 +2798,15 @@ export type MetadataMaterializedViews = z.infer< typeof MetadataMaterializedViewsSchema >; +/** + * How many parts Map key discovery reads from a text index. Unset uses + * DEFAULT_MAP_KEY_INDEX_MAX_PARTS. A cleared form NumberInput yields `''`. + */ +const MapKeyIndexMaxPartsSchema = z.preprocess( + v => (v === '' || v === null ? undefined : v), + z.number().int().min(1).optional(), +); + // Log source form schema export const LogSourceSchema = BaseSourceSchema.extend({ kind: z.literal(SourceKind.Log), @@ -2855,6 +2864,7 @@ export const LogSourceSchema = BaseSourceSchema.extend({ HighlightedAttributeExpressionsSchema.optional(), materializedViews: z.array(MaterializedViewConfigurationSchema).optional(), metadataMaterializedViews: MetadataMaterializedViewsSchema.optional(), + mapKeyIndexMaxParts: MapKeyIndexMaxPartsSchema, orderByExpression: z.string().optional(), useTextIndexForImplicitColumn: UseTextIndexSchema.optional(), }); @@ -2902,6 +2912,7 @@ export const TraceSourceSchema = BaseSourceSchema.extend({ HighlightedAttributeExpressionsSchema.optional(), materializedViews: z.array(MaterializedViewConfigurationSchema).optional(), metadataMaterializedViews: MetadataMaterializedViewsSchema.optional(), + mapKeyIndexMaxParts: MapKeyIndexMaxPartsSchema, orderByExpression: z.string().optional(), useTextIndexForImplicitColumn: UseTextIndexSchema.optional(), }); From bf19d5af79cabd9d9c27afedf2c946b267b6c570 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:25:47 -0500 Subject: [PATCH 02/10] feat: choose median, smallest, or largest parts for Map key discovery --- .changeset/map-key-index-median-parts.md | 4 +- packages/api/openapi.json | 26 +++++++++- .../src/mcp/tools/sources/describeSource.ts | 4 ++ packages/api/src/mcp/tools/sources/schemas.ts | 11 +++- packages/api/src/models/source.ts | 9 ++++ .../routers/api/__tests__/sources.int.test.ts | 38 ++++++++++++++ .../src/routers/external-api/v2/sources.ts | 16 +++++- .../Sources/SourceForm/LogTableModelForm.tsx | 6 ++- .../SourceForm/MapKeyIndexMaxPartsFormRow.tsx | 26 +++++++++- .../SourceForm/TraceTableModelForm.tsx | 6 ++- .../Sources/SourceForm/constants.ts | 7 +++ .../src/__tests__/metadata.test.ts | 51 ++++++++++++++++++- packages/common-utils/src/core/metadata.ts | 48 +++++++++++++---- packages/common-utils/src/types.ts | 13 +++++ 14 files changed, 245 insertions(+), 20 deletions(-) diff --git a/.changeset/map-key-index-median-parts.md b/.changeset/map-key-index-median-parts.md index de0c160850..1dbf073f47 100644 --- a/.changeset/map-key-index-median-parts.md +++ b/.changeset/map-key-index-median-parts.md @@ -9,4 +9,6 @@ perf: read only 20 median-sized parts when discovering Map keys from a text inde Map key discovery read the text index dictionary of every part in the time range, which is slow on tables with many parts. It now reads the 20 parts closest to the median size. Log and trace sources can change the count with -the new "Map key index parts" setting (`mapKeyIndexMaxParts`). +the new "Map key index parts" setting (`mapKeyIndexMaxParts`), and which parts +are read with "Map key index part size" (`mapKeyIndexPartSelection`: median, +smallest, or largest). diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 624c22afa9..8ec58632e9 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -5658,10 +5658,21 @@ "mapKeyIndexMaxParts": { "type": "integer", "minimum": 1, - "description": "How many data parts Map attribute key discovery reads from a text index, picked around the median part size. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20.", + "description": "How many data parts Map attribute key discovery reads from a text index, chosen by mapKeyIndexPartSelection. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20.", "nullable": true, "example": 20 }, + "mapKeyIndexPartSelection": { + "type": "string", + "enum": [ + "median", + "smallest", + "largest" + ], + "description": "Which data parts, by row count, Map attribute key discovery reads from a text index. \"median\" (the default) balances coverage and cost, \"smallest\" reads the newest unmerged parts, \"largest\" reads the most merged data.", + "nullable": true, + "example": "median" + }, "highlightedTraceAttributeExpressions": { "type": "array", "description": "Expressions defining trace-level attributes which are displayed in the trace view for the selected trace.", @@ -5925,10 +5936,21 @@ "mapKeyIndexMaxParts": { "type": "integer", "minimum": 1, - "description": "How many data parts Map attribute key discovery reads from a text index, picked around the median part size. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20.", + "description": "How many data parts Map attribute key discovery reads from a text index, chosen by mapKeyIndexPartSelection. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20.", "nullable": true, "example": 20 }, + "mapKeyIndexPartSelection": { + "type": "string", + "enum": [ + "median", + "smallest", + "largest" + ], + "description": "Which data parts, by row count, Map attribute key discovery reads from a text index. \"median\" (the default) balances coverage and cost, \"smallest\" reads the newest unmerged parts, \"largest\" reads the most merged data.", + "nullable": true, + "example": "median" + }, "highlightedTraceAttributeExpressions": { "type": "array", "description": "Expressions defining trace-level attributes which are displayed in the trace view for the selected trace.", diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts index 49d09879bb..58f6233075 100644 --- a/packages/api/src/mcp/tools/sources/describeSource.ts +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -310,6 +310,10 @@ async function describeSourceSchema( 'mapKeyIndexMaxParts' in source ? source.mapKeyIndexMaxParts : undefined, + mapKeyIndexPartSelection: + 'mapKeyIndexPartSelection' in source + ? source.mapKeyIndexPartSelection + : undefined, dateRange, timestampValueExpression, signal, diff --git a/packages/api/src/mcp/tools/sources/schemas.ts b/packages/api/src/mcp/tools/sources/schemas.ts index 236eaad947..2e8857fdc5 100644 --- a/packages/api/src/mcp/tools/sources/schemas.ts +++ b/packages/api/src/mcp/tools/sources/schemas.ts @@ -482,10 +482,19 @@ const mcpSaveSourceSchema = z.object({ .optional() .describe( 'Log and trace sources. How many data parts Map attribute key discovery ' + - 'reads from a text index, picked around the median part size. Raise it ' + + 'reads from a text index, chosen by mapKeyIndexPartSelection. Raise it ' + 'if attribute keys are missing from filters; lower it if field loading ' + 'is slow. Omit for the default of 20.', ), + mapKeyIndexPartSelection: z + .enum(['median', 'smallest', 'largest']) + .optional() + .describe( + 'Log and trace sources. Which data parts, by row count, Map attribute ' + + 'key discovery reads from a text index: "median" (used when unset) ' + + 'balances coverage and cost, "smallest" reads the newest unmerged ' + + 'parts, "largest" reads the most merged data.', + ), }); export type McpSaveSourceInput = z.infer; diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index 53d90f0fb9..2954e224a4 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -1,6 +1,7 @@ import { BaseSourceSchema, LogSourceSchema, + MapKeyIndexPartSelection, MetricsDataType, MetricSourceSchema, PROMQL_SOURCE_BACKENDS, @@ -191,6 +192,10 @@ export const LogSource = Source.discriminator( default: undefined, }, mapKeyIndexMaxParts: Number, + mapKeyIndexPartSelection: { + type: String, + enum: Object.values(MapKeyIndexPartSelection), + }, orderByExpression: String, }), ); @@ -244,6 +249,10 @@ export const TraceSource = Source.discriminator( default: undefined, }, mapKeyIndexMaxParts: Number, + mapKeyIndexPartSelection: { + type: String, + enum: Object.values(MapKeyIndexPartSelection), + }, orderByExpression: String, }), ); diff --git a/packages/api/src/routers/api/__tests__/sources.int.test.ts b/packages/api/src/routers/api/__tests__/sources.int.test.ts index da05d29e2b..9b4ca14c43 100644 --- a/packages/api/src/routers/api/__tests__/sources.int.test.ts +++ b/packages/api/src/routers/api/__tests__/sources.int.test.ts @@ -1709,4 +1709,42 @@ describe('sources router', () => { }, ); }); + + describe('mapKeyIndexPartSelection field', () => { + it('POST / - persists mapKeyIndexPartSelection and PUT /:id updates it', async () => { + const { agent } = await getLoggedInAgent(server); + + const created = await agent + .post('/sources') + .send({ ...MOCK_SOURCE, mapKeyIndexPartSelection: 'largest' }) + .expect(200); + expect(created.body.mapKeyIndexPartSelection).toBe('largest'); + + await agent + .put(`/sources/${created.body.id}`) + .send({ + ...MOCK_SOURCE, + id: created.body.id, + mapKeyIndexPartSelection: 'smallest', + }) + .expect(200); + + const updated = await Source.findById(created.body.id).lean(); + if (updated?.kind !== SourceKind.Log) { + throw new Error(`expected Log source, got ${updated?.kind}`); + } + expect(updated.mapKeyIndexPartSelection).toBe('smallest'); + }); + + it('POST / - rejects an unknown mapKeyIndexPartSelection', async () => { + const { agent } = await getLoggedInAgent(server); + + await agent + .post('/sources') + .send({ ...MOCK_SOURCE, mapKeyIndexPartSelection: 'random' }) + .expect(400); + + expect(await Source.find({}).lean()).toHaveLength(0); + }); + }); }); diff --git a/packages/api/src/routers/external-api/v2/sources.ts b/packages/api/src/routers/external-api/v2/sources.ts index 5301f87814..7248b5bc8e 100644 --- a/packages/api/src/routers/external-api/v2/sources.ts +++ b/packages/api/src/routers/external-api/v2/sources.ts @@ -420,9 +420,15 @@ function formatExternalSource(source: SourceDocument) { * mapKeyIndexMaxParts: * type: integer * minimum: 1 - * description: How many data parts Map attribute key discovery reads from a text index, picked around the median part size. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20. + * description: How many data parts Map attribute key discovery reads from a text index, chosen by mapKeyIndexPartSelection. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20. * nullable: true * example: 20 + * mapKeyIndexPartSelection: + * type: string + * enum: [median, smallest, largest] + * description: Which data parts, by row count, Map attribute key discovery reads from a text index. "median" (the default) balances coverage and cost, "smallest" reads the newest unmerged parts, "largest" reads the most merged data. + * nullable: true + * example: median * highlightedTraceAttributeExpressions: * type: array * description: Expressions defining trace-level attributes which are displayed in the trace view for the selected trace. @@ -632,9 +638,15 @@ function formatExternalSource(source: SourceDocument) { * mapKeyIndexMaxParts: * type: integer * minimum: 1 - * description: How many data parts Map attribute key discovery reads from a text index, picked around the median part size. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20. + * description: How many data parts Map attribute key discovery reads from a text index, chosen by mapKeyIndexPartSelection. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20. * nullable: true * example: 20 + * mapKeyIndexPartSelection: + * type: string + * enum: [median, smallest, largest] + * description: Which data parts, by row count, Map attribute key discovery reads from a text index. "median" (the default) balances coverage and cost, "smallest" reads the newest unmerged parts, "largest" reads the most merged data. + * nullable: true + * example: median * highlightedTraceAttributeExpressions: * type: array * description: Expressions defining trace-level attributes which are displayed in the trace view for the selected trace. diff --git a/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx b/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx index e9f6cf0a93..c2b8ffe449 100644 --- a/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx +++ b/packages/app/src/components/Sources/SourceForm/LogTableModelForm.tsx @@ -21,7 +21,10 @@ import { import { ExpressionFormRow } from './ExpressionFormRow'; import { FormRow } from './FormRow'; import { HighlightedAttributeExpressionsFormRow } from './HighlightedAttributes'; -import { MapKeyIndexMaxPartsFormRow } from './MapKeyIndexMaxPartsFormRow'; +import { + MapKeyIndexMaxPartsFormRow, + MapKeyIndexPartSelectionFormRow, +} from './MapKeyIndexMaxPartsFormRow'; import { MaterializedViewsFormSection, MetadataMaterializedViewsFormSection, @@ -305,6 +308,7 @@ export function LogTableModelForm(props: TableModelProps) { + ); } + +export function MapKeyIndexPartSelectionFormRow({ + control, +}: { + control: Control; +}) { + return ( + + + + ); +} diff --git a/packages/app/src/components/Sources/SourceForm/TraceTableModelForm.tsx b/packages/app/src/components/Sources/SourceForm/TraceTableModelForm.tsx index 6386e58980..d17f20a402 100644 --- a/packages/app/src/components/Sources/SourceForm/TraceTableModelForm.tsx +++ b/packages/app/src/components/Sources/SourceForm/TraceTableModelForm.tsx @@ -17,7 +17,10 @@ import { import { ExpressionFormRow } from './ExpressionFormRow'; import { FormRow } from './FormRow'; import { HighlightedAttributeExpressionsFormRow } from './HighlightedAttributes'; -import { MapKeyIndexMaxPartsFormRow } from './MapKeyIndexMaxPartsFormRow'; +import { + MapKeyIndexMaxPartsFormRow, + MapKeyIndexPartSelectionFormRow, +} from './MapKeyIndexMaxPartsFormRow'; import { MaterializedViewsFormSection, MetadataMaterializedViewsFormSection, @@ -399,6 +402,7 @@ export function TraceTableModelForm(props: TableModelProps) { + { expect(partsLimits).toEqual([20, 5]); }, ); + + it.each(textIndexLookups)( + 'orders the parts by the configured part selection on the %s text index path', + async (_, textIndexLookup) => { + const md = new Metadata(mockClickhouseClient, new MetadataCache()); + jest.spyOn(md, 'getServerVersion').mockResolvedValue([26, 3, 0, 0]); + jest + .spyOn(md, 'getMapColumnTextIndexes') + .mockResolvedValue(textIndexLookup); + (mockClickhouseClient.query as jest.Mock).mockResolvedValue({ + json: () => Promise.resolve({ data: [{ key: 'http.method' }] }), + }); + const args = { + databaseName: 'default', + tableName: 'otel_logs', + column: 'LogAttributes', + connectionId: 'test_connection', + dateRange: [ + new Date('2024-01-01T00:00:00Z'), + new Date('2024-01-01T01:00:00Z'), + ] as [Date, Date], + timestampValueExpression: 'Timestamp', + }; + + for (const mapKeyIndexPartSelection of [ + MapKeyIndexPartSelection.Smallest, + MapKeyIndexPartSelection.Largest, + MapKeyIndexPartSelection.Median, + ]) { + await md.getMapKeys({ ...args, mapKeyIndexPartSelection }); + } + + // Each selection is a different result, not a cache hit. + const orders = (mockClickhouseClient.query as jest.Mock).mock.calls.map( + ([opts]) => + opts.query.match(/\)\s+ORDER BY (.+)\s+LIMIT \{\w+:Int32\}/)?.[1], + ); + expect(orders).toEqual([ + 'rn', + 'rn DESC', + 'abs(rn - (cnt + 1) / 2), name', + ]); + }, + ); }); describe('getMapValues', () => { diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 170c6d078c..8226f810a6 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -38,7 +38,12 @@ import type { MetadataMaterializedViews, TSource, } from '@/types'; -import { isLogSource, isTraceSource, SourceKind } from '@/types'; +import { + isLogSource, + isTraceSource, + MapKeyIndexPartSelection, + SourceKind, +} from '@/types'; import { ClickHouseVersion, @@ -65,8 +70,9 @@ import { // Between 1e6 - 5e6 is a good range. export const DEFAULT_METADATA_MAX_ROWS_TO_READ = 3e6; const DEFAULT_MAX_KEYS = 1000; -// shortcut: keys found only in the smallest or largest parts of the range can -// be missed; raise it per source (mapKeyIndexMaxParts) if keys go missing. +// shortcut: keys found only in parts outside the selection can be missed; raise +// it per source (mapKeyIndexMaxParts) or pick other parts +// (mapKeyIndexPartSelection) if keys go missing. export const DEFAULT_MAP_KEY_INDEX_MAX_PARTS = 20; // Cap keys per dispatched query: each key is another operation for the db to fetch, and simply fetching all keys at once can be too much for the db to handle. @@ -900,13 +906,16 @@ export class Metadata { dateRange, timestampValueExpression, maxParts, + partSelection = MapKeyIndexPartSelection.Median, }: { databaseName: string; tableName: string; dateRange?: [Date, Date]; timestampValueExpression?: string; - /** Keep only the `maxParts` overlapping parts closest to the median row count. */ + /** Keep only `maxParts` overlapping parts, chosen by `partSelection`. */ maxParts?: number; + /** Which parts by row count; only applies with `maxParts`. */ + partSelection?: MapKeyIndexPartSelection; }): Promise { if (!dateRange || !timestampValueExpression) return chSql`1`; const startTime = chSql`fromUnixTimestamp64Milli(${{ Int64: dateRange[0].getTime() }})`; @@ -919,8 +928,13 @@ export class Metadata { if (maxParts == null) { return chSql`part_name IN (SELECT name ${overlappingParts})`; } - // Rank by size and pick from the middle out: tiny unmerged parts hold few - // keys and the largest merged parts have the biggest dictionaries. + // Rank by size. Median picks from the middle out: tiny unmerged parts + // hold few keys and the largest merged parts have the biggest dictionaries. + const order = { + [MapKeyIndexPartSelection.Median]: 'abs(rn - (cnt + 1) / 2), name', + [MapKeyIndexPartSelection.Smallest]: 'rn', + [MapKeyIndexPartSelection.Largest]: 'rn DESC', + }[partSelection]; return chSql`part_name IN ( SELECT name FROM ( SELECT name, @@ -928,7 +942,7 @@ export class Metadata { count() OVER () AS cnt ${overlappingParts} ) - ORDER BY abs(rn - (cnt + 1) / 2), name + ORDER BY ${{ UNSAFE_RAW_SQL: order }} LIMIT ${{ Int32: maxParts }} )`; } @@ -1095,6 +1109,7 @@ export class Metadata { timestampValueExpression, conditionSql, mapKeyIndexMaxParts = DEFAULT_MAP_KEY_INDEX_MAX_PARTS, + mapKeyIndexPartSelection = MapKeyIndexPartSelection.Median, signal, }: { databaseName: string; @@ -1109,6 +1124,7 @@ export class Metadata { conditionSql?: string; /** Parts the text index paths read; see `partsOverlapFilter`. */ mapKeyIndexMaxParts?: number; + mapKeyIndexPartSelection?: MapKeyIndexPartSelection; signal?: AbortSignal; }) { inlineNonNegativeInt(maxKeys, 'maxKeys'); @@ -1132,10 +1148,10 @@ export class Metadata { ? `${alignedDateRange[0].getTime()}.${alignedDateRange[1].getTime()}.${timestampValueExpression}` : ''; const unscopedCacheKey = metricName - ? `${connectionId}.${databaseName}.${tableName}.${column}.${metricName}.${dateRangeCacheSuffix}.${maxKeys}.${mapKeyIndexMaxParts}.keys` + ? `${connectionId}.${databaseName}.${tableName}.${column}.${metricName}.${dateRangeCacheSuffix}.${maxKeys}.${mapKeyIndexMaxParts}.${mapKeyIndexPartSelection}.keys` : alignedDateRange - ? `${connectionId}.${databaseName}.${tableName}.${column}.${alignedCacheSuffix}.${maxKeys}.${mapKeyIndexMaxParts}.keys` - : `${connectionId}.${databaseName}.${tableName}.${column}.${dateRangeCacheSuffix}.${maxKeys}.${mapKeyIndexMaxParts}.keys`; + ? `${connectionId}.${databaseName}.${tableName}.${column}.${alignedCacheSuffix}.${maxKeys}.${mapKeyIndexMaxParts}.${mapKeyIndexPartSelection}.keys` + : `${connectionId}.${databaseName}.${tableName}.${column}.${dateRangeCacheSuffix}.${maxKeys}.${mapKeyIndexMaxParts}.${mapKeyIndexPartSelection}.keys`; const cacheKey = conditionSql ? `${unscopedCacheKey}.${objectHash(conditionSql)}.condition` : unscopedCacheKey; @@ -1160,6 +1176,7 @@ export class Metadata { timestampValueExpression, conditionSql, mapKeyIndexMaxParts, + mapKeyIndexPartSelection, cacheKey, signal, }), @@ -1180,6 +1197,7 @@ export class Metadata { timestampValueExpression, conditionSql, mapKeyIndexMaxParts, + mapKeyIndexPartSelection, cacheKey, signal, }: { @@ -1195,6 +1213,7 @@ export class Metadata { timestampValueExpression?: string; conditionSql?: string; mapKeyIndexMaxParts: number; + mapKeyIndexPartSelection: MapKeyIndexPartSelection; cacheKey: string; signal?: AbortSignal; }): Promise { @@ -1231,6 +1250,7 @@ export class Metadata { dateRange, timestampValueExpression, maxParts: mapKeyIndexMaxParts, + partSelection: mapKeyIndexPartSelection, }); const index = textIndexInfo.key.indexName; // `cardinality` is the token's row count in the part, not a distinct count. @@ -1277,6 +1297,7 @@ export class Metadata { dateRange, timestampValueExpression, maxParts: mapKeyIndexMaxParts, + partSelection: mapKeyIndexPartSelection, }); const index = textIndexInfo.kv.indexName; const separator = textIndexInfo.kv.separator; @@ -1993,6 +2014,7 @@ export class Metadata { metricName, metadataMVs, mapKeyIndexMaxParts, + mapKeyIndexPartSelection, dateRange, timestampValueExpression, condition, @@ -2067,6 +2089,7 @@ export class Metadata { metricName, metadataMVs, mapKeyIndexMaxParts, + mapKeyIndexPartSelection, dateRange, timestampValueExpression, conditionSql, @@ -3498,6 +3521,7 @@ export type TableConnection = { metricName?: string; metadataMVs?: MetadataMaterializedViews; mapKeyIndexMaxParts?: number; + mapKeyIndexPartSelection?: MapKeyIndexPartSelection; // Bounds Map key discovery; without it Map keys are skipped. #3037 timestampValueExpression?: string; }; @@ -3545,6 +3569,10 @@ export function tcFromSource(source?: TSource): TableConnection { source && (isLogSource(source) || isTraceSource(source)) ? source.mapKeyIndexMaxParts : undefined, + mapKeyIndexPartSelection: + source && (isLogSource(source) || isTraceSource(source)) + ? source.mapKeyIndexPartSelection + : undefined, timestampValueExpression: source?.timestampValueExpression, }; } diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 081d198141..4210af9dd5 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -28,6 +28,17 @@ export enum UseTextIndex { export const UseTextIndexSchema = z.nativeEnum(UseTextIndex); +/** Which data parts, by row count, text index key and value lookups read. */ +export enum MapKeyIndexPartSelection { + Median = 'median', + Smallest = 'smallest', + Largest = 'largest', +} + +export const MapKeyIndexPartSelectionSchema = z.nativeEnum( + MapKeyIndexPartSelection, +); + // -------------------------- // UI // -------------------------- @@ -2865,6 +2876,7 @@ export const LogSourceSchema = BaseSourceSchema.extend({ materializedViews: z.array(MaterializedViewConfigurationSchema).optional(), metadataMaterializedViews: MetadataMaterializedViewsSchema.optional(), mapKeyIndexMaxParts: MapKeyIndexMaxPartsSchema, + mapKeyIndexPartSelection: MapKeyIndexPartSelectionSchema.optional(), orderByExpression: z.string().optional(), useTextIndexForImplicitColumn: UseTextIndexSchema.optional(), }); @@ -2913,6 +2925,7 @@ export const TraceSourceSchema = BaseSourceSchema.extend({ materializedViews: z.array(MaterializedViewConfigurationSchema).optional(), metadataMaterializedViews: MetadataMaterializedViewsSchema.optional(), mapKeyIndexMaxParts: MapKeyIndexMaxPartsSchema, + mapKeyIndexPartSelection: MapKeyIndexPartSelectionSchema.optional(), orderByExpression: z.string().optional(), useTextIndexForImplicitColumn: UseTextIndexSchema.optional(), }); From 94375c0d77603e406f99677d05c1f44d31dee92f Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:05:16 -0500 Subject: [PATCH 03/10] perf: read only median-sized parts in text index value lookups --- packages/api/openapi.json | 8 +-- .../src/mcp/tools/sources/describeSource.ts | 20 ++++--- packages/api/src/mcp/tools/sources/schemas.ts | 12 ++-- .../src/routers/external-api/v2/sources.ts | 8 +-- .../components/DBSearchPageFilters/hooks.ts | 2 + .../SourceForm/MapKeyIndexMaxPartsFormRow.tsx | 4 +- packages/app/src/hooks/useMetadata.tsx | 48 +++++++++++----- .../src/__tests__/metadata.test.ts | 56 +++++++++++++++++++ packages/common-utils/src/core/metadata.ts | 31 ++++++++-- packages/common-utils/src/types.ts | 2 +- 10 files changed, 146 insertions(+), 45 deletions(-) diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 8ec58632e9..25df134c2e 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -5658,7 +5658,7 @@ "mapKeyIndexMaxParts": { "type": "integer", "minimum": 1, - "description": "How many data parts Map attribute key discovery reads from a text index, chosen by mapKeyIndexPartSelection. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20.", + "description": "How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20.", "nullable": true, "example": 20 }, @@ -5669,7 +5669,7 @@ "smallest", "largest" ], - "description": "Which data parts, by row count, Map attribute key discovery reads from a text index. \"median\" (the default) balances coverage and cost, \"smallest\" reads the newest unmerged parts, \"largest\" reads the most merged data.", + "description": "Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. \"median\" (the default) balances coverage and cost, \"smallest\" reads the newest unmerged parts, \"largest\" reads the most merged data.", "nullable": true, "example": "median" }, @@ -5936,7 +5936,7 @@ "mapKeyIndexMaxParts": { "type": "integer", "minimum": 1, - "description": "How many data parts Map attribute key discovery reads from a text index, chosen by mapKeyIndexPartSelection. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20.", + "description": "How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20.", "nullable": true, "example": 20 }, @@ -5947,7 +5947,7 @@ "smallest", "largest" ], - "description": "Which data parts, by row count, Map attribute key discovery reads from a text index. \"median\" (the default) balances coverage and cost, \"smallest\" reads the newest unmerged parts, \"largest\" reads the most merged data.", + "description": "Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. \"median\" (the default) balances coverage and cost, \"smallest\" reads the newest unmerged parts, \"largest\" reads the most merged data.", "nullable": true, "example": "median" }, diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts index 58f6233075..4811a036c7 100644 --- a/packages/api/src/mcp/tools/sources/describeSource.ts +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -228,6 +228,12 @@ async function describeSourceSchema( 'metadataMaterializedViews' in source ? source.metadataMaterializedViews : undefined; + const mapKeyIndexMaxParts = + 'mapKeyIndexMaxParts' in source ? source.mapKeyIndexMaxParts : undefined; + const mapKeyIndexPartSelection = + 'mapKeyIndexPartSelection' in source + ? source.mapKeyIndexPartSelection + : undefined; const now = new Date(); const dateRange: [Date, Date] = [ @@ -306,14 +312,8 @@ async function describeSourceSchema( maxKeys: 50, connectionId, metadataMVs, - mapKeyIndexMaxParts: - 'mapKeyIndexMaxParts' in source - ? source.mapKeyIndexMaxParts - : undefined, - mapKeyIndexPartSelection: - 'mapKeyIndexPartSelection' in source - ? source.mapKeyIndexPartSelection - : undefined, + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, dateRange, timestampValueExpression, signal, @@ -360,6 +360,8 @@ async function describeSourceSchema( maxValuesPerKey: MAX_LC_VALUES, connectionId, metadataMVs, + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, dateRange, timestampValueExpression, signal, @@ -407,6 +409,8 @@ async function describeSourceSchema( maxValuesPerKey: MAX_MAP_KEY_VALUES, connectionId, metadataMVs, + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, dateRange, timestampValueExpression, signal, diff --git a/packages/api/src/mcp/tools/sources/schemas.ts b/packages/api/src/mcp/tools/sources/schemas.ts index 2e8857fdc5..e7a9ccad91 100644 --- a/packages/api/src/mcp/tools/sources/schemas.ts +++ b/packages/api/src/mcp/tools/sources/schemas.ts @@ -481,17 +481,17 @@ const mcpSaveSourceSchema = z.object({ .min(1) .optional() .describe( - 'Log and trace sources. How many data parts Map attribute key discovery ' + - 'reads from a text index, chosen by mapKeyIndexPartSelection. Raise it ' + - 'if attribute keys are missing from filters; lower it if field loading ' + - 'is slow. Omit for the default of 20.', + 'Log and trace sources. How many data parts are read from text indexes ' + + 'to discover attribute keys and filter values, chosen by ' + + 'mapKeyIndexPartSelection. Raise it if keys or values are missing from ' + + 'filters; lower it if they load slowly. Omit for the default of 20.', ), mapKeyIndexPartSelection: z .enum(['median', 'smallest', 'largest']) .optional() .describe( - 'Log and trace sources. Which data parts, by row count, Map attribute ' + - 'key discovery reads from a text index: "median" (used when unset) ' + + 'Log and trace sources. Which data parts, by row count, are read from ' + + 'text indexes for attribute keys and filter values: "median" (used when unset) ' + 'balances coverage and cost, "smallest" reads the newest unmerged ' + 'parts, "largest" reads the most merged data.', ), diff --git a/packages/api/src/routers/external-api/v2/sources.ts b/packages/api/src/routers/external-api/v2/sources.ts index 7248b5bc8e..c00d4776ca 100644 --- a/packages/api/src/routers/external-api/v2/sources.ts +++ b/packages/api/src/routers/external-api/v2/sources.ts @@ -420,13 +420,13 @@ function formatExternalSource(source: SourceDocument) { * mapKeyIndexMaxParts: * type: integer * minimum: 1 - * description: How many data parts Map attribute key discovery reads from a text index, chosen by mapKeyIndexPartSelection. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20. + * description: How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20. * nullable: true * example: 20 * mapKeyIndexPartSelection: * type: string * enum: [median, smallest, largest] - * description: Which data parts, by row count, Map attribute key discovery reads from a text index. "median" (the default) balances coverage and cost, "smallest" reads the newest unmerged parts, "largest" reads the most merged data. + * description: Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. "median" (the default) balances coverage and cost, "smallest" reads the newest unmerged parts, "largest" reads the most merged data. * nullable: true * example: median * highlightedTraceAttributeExpressions: @@ -638,13 +638,13 @@ function formatExternalSource(source: SourceDocument) { * mapKeyIndexMaxParts: * type: integer * minimum: 1 - * description: How many data parts Map attribute key discovery reads from a text index, chosen by mapKeyIndexPartSelection. Raise it if attribute keys are missing from filters; lower it if field loading is slow. Defaults to 20. + * description: How many data parts are read from text indexes to discover attribute keys and filter values, chosen by mapKeyIndexPartSelection. Raise it if keys or values are missing from filters; lower it if they load slowly. Defaults to 20. * nullable: true * example: 20 * mapKeyIndexPartSelection: * type: string * enum: [median, smallest, largest] - * description: Which data parts, by row count, Map attribute key discovery reads from a text index. "median" (the default) balances coverage and cost, "smallest" reads the newest unmerged parts, "largest" reads the most merged data. + * description: Which data parts, by row count, are read from text indexes to discover attribute keys and filter values. "median" (the default) balances coverage and cost, "smallest" reads the newest unmerged parts, "largest" reads the most merged data. * nullable: true * example: median * highlightedTraceAttributeExpressions: diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.ts b/packages/app/src/components/DBSearchPageFilters/hooks.ts index c3f65d8b76..28bbb604a4 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.ts +++ b/packages/app/src/components/DBSearchPageFilters/hooks.ts @@ -271,6 +271,8 @@ function useFacets({ tableName: tableConnection.tableName, connectionId: tableConnection.connectionId, metadataMVs: tableConnection.metadataMVs, + mapKeyIndexMaxParts: tableConnection.mapKeyIndexMaxParts, + mapKeyIndexPartSelection: tableConnection.mapKeyIndexPartSelection, keyExpressions: [sqlKey], maxValuesPerKey: SINGLE_KEY_VALUES_LIMIT, dateRange, diff --git a/packages/app/src/components/Sources/SourceForm/MapKeyIndexMaxPartsFormRow.tsx b/packages/app/src/components/Sources/SourceForm/MapKeyIndexMaxPartsFormRow.tsx index aa1c52ca2e..1c5a10f591 100644 --- a/packages/app/src/components/Sources/SourceForm/MapKeyIndexMaxPartsFormRow.tsx +++ b/packages/app/src/components/Sources/SourceForm/MapKeyIndexMaxPartsFormRow.tsx @@ -16,7 +16,7 @@ export function MapKeyIndexMaxPartsFormRow({ return ( s.id === chartConfig.source) : undefined; + const isLogOrTrace = + !!source && (isLogSource(source) || isTraceSource(source)); const metadataMVs = usableMVsOverride ?? - (source && (isLogSource(source) || isTraceSource(source)) - ? source.metadataMaterializedViews - : undefined); - return { chartConfig, source, metadataMVs }; + (isLogOrTrace ? source.metadataMaterializedViews : undefined); + return { + chartConfig, + source, + metadataMVs, + mapKeyIndexMaxParts: isLogOrTrace + ? source.mapKeyIndexMaxParts + : undefined, + mapKeyIndexPartSelection: isLogOrTrace + ? source.mapKeyIndexPartSelection + : undefined, + }; }); if (mode === 'exact') { @@ -606,18 +616,26 @@ export function useMultipleGetKeyValues( } // mode === 'all' - const streams = lookups.map(({ chartConfig, metadataMVs }) => - metadata.streamAllKeyValues({ - databaseName: chartConfig.from.databaseName, - tableName: chartConfig.from.tableName, - keyExpressions: limitedKeys, - maxValuesPerKey: limit, - connectionId: chartConfig.connection, + const streams = lookups.map( + ({ + chartConfig, metadataMVs, - dateRange: chartConfig.dateRange, - timestampValueExpression: chartConfig.timestampValueExpression, - signal, - }), + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, + }) => + metadata.streamAllKeyValues({ + databaseName: chartConfig.from.databaseName, + tableName: chartConfig.from.tableName, + keyExpressions: limitedKeys, + maxValuesPerKey: limit, + connectionId: chartConfig.connection, + metadataMVs, + mapKeyIndexMaxParts, + mapKeyIndexPartSelection, + dateRange: chartConfig.dateRange, + timestampValueExpression: chartConfig.timestampValueExpression, + signal, + }), ); for await (const keyValues of mergeStreams(streams)) { yield keyValues.map(({ key, value }) => ({ diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index 4551448b2c..f3c8f49678 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1403,6 +1403,62 @@ describe('Metadata', () => { expect(sql).toContain('GROUP BY column, key'); }); + it.each([ + ['map KV', "LogAttributes['requestId']"], + ['native', 'TraceId'], + ])( + 'reads only the parts nearest the median size from the %s text index', + async (_, keyExpression) => { + setupDefaultLogsSchema(); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [keyExpression], + }); + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [keyExpression], + mapKeyIndexMaxParts: 5, + }); + + const partsLimits = (mockClickhouseClient.query as jest.Mock).mock.calls + .filter(([opts]) => opts.query.includes('mergeTreeTextIndex(')) + .map(([opts]) => { + const match = opts.query.match( + /ORDER BY abs\(rn - \(cnt \+ 1\) \/ 2\), name\s+LIMIT \{(\w+):Int32\}/, + ); + expect(match).not.toBeNull(); + return opts.query_params[match[1]]; + }); + expect(partsLimits).toEqual([20, 5]); + }, + ); + + it.each([ + ['map KV', "LogAttributes['requestId']"], + ['native', 'TraceId'], + ])( + 'orders the %s text index parts by the configured part selection', + async (_, keyExpression) => { + setupDefaultLogsSchema(); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [keyExpression], + mapKeyIndexPartSelection: MapKeyIndexPartSelection.Largest, + }); + + const textIndexCall = ( + mockClickhouseClient.query as jest.Mock + ).mock.calls.find(([opts]) => + opts.query.includes('mergeTreeTextIndex('), + ); + expect(textIndexCall[0].query).toMatch( + /\)\s+ORDER BY rn DESC\s+LIMIT \{\w+:Int32\}/, + ); + }, + ); + it('routes ServiceName and SeverityText through getMetadataMVKeyValues (KV rollup MV path)', async () => { setupDefaultLogsSchema(); diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 8226f810a6..75a9c2f42c 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -70,9 +70,9 @@ import { // Between 1e6 - 5e6 is a good range. export const DEFAULT_METADATA_MAX_ROWS_TO_READ = 3e6; const DEFAULT_MAX_KEYS = 1000; -// shortcut: keys found only in parts outside the selection can be missed; raise -// it per source (mapKeyIndexMaxParts) or pick other parts -// (mapKeyIndexPartSelection) if keys go missing. +// shortcut: keys and values found only in parts outside the selection can be +// missed; raise it per source (mapKeyIndexMaxParts) or pick other parts +// (mapKeyIndexPartSelection) if they go missing. export const DEFAULT_MAP_KEY_INDEX_MAX_PARTS = 20; // Cap keys per dispatched query: each key is another operation for the db to fetch, and simply fetching all keys at once can be too much for the db to handle. @@ -112,6 +112,9 @@ type AllKeyValuesQuery = { maxValuesPerKey?: number; connectionId: string; metadataMVs?: MetadataMaterializedViews; + /** Parts the text index paths read; see `partsOverlapFilter`. */ + mapKeyIndexMaxParts?: number; + mapKeyIndexPartSelection?: MapKeyIndexPartSelection; dateRange: [Date, Date]; timestampValueExpression: string; signal?: AbortSignal; @@ -1744,17 +1747,21 @@ export class Metadata { tableName, connectionId, queryOptions, + maxParts, + partSelection, dateRange, timestampValueExpression, signal, }: TableConnection & { queryOptions: TextIndexMapColumnQueryOptions; + maxParts: number; + partSelection: MapKeyIndexPartSelection; dateRange: [Date, Date]; timestampValueExpression: string; signal?: AbortSignal; }): Promise { const queryOptionsHash = objectHash(queryOptions); - const cacheKey = `${databaseName}.${tableName}.${connectionId}.${dateRange[0].toString()}.${dateRange[1].toString()}.${queryOptionsHash}.${timestampValueExpression}.getMapTextIndexKeyValues`; + const cacheKey = `${databaseName}.${tableName}.${connectionId}.${dateRange[0].toString()}.${dateRange[1].toString()}.${queryOptionsHash}.${maxParts}.${partSelection}.${timestampValueExpression}.getMapTextIndexKeyValues`; return this.cache.getOrFetch(cacheKey, async () => { try { const sqlBranches: Array = []; @@ -1777,6 +1784,8 @@ export class Metadata { tableName, dateRange, timestampValueExpression, + maxParts, + partSelection, }); const valueSql = chSql`substring(token, position(token, ${{ String: info.separator }}) + ${{ Int32: info.separator.length }})`; const sql = chSql` @@ -1836,17 +1845,21 @@ export class Metadata { tableName, connectionId, queryOptions, + maxParts, + partSelection, dateRange, timestampValueExpression, signal, }: TableConnection & { queryOptions: TextIndexColumnQueryOptions; + maxParts: number; + partSelection: MapKeyIndexPartSelection; dateRange: [Date, Date]; timestampValueExpression: string; signal?: AbortSignal; }): Promise { const queryOptionsHash = objectHash(queryOptions); - const cacheKey = `${databaseName}.${tableName}.${connectionId}.${dateRange[0].toString()}.${dateRange[1].toString()}.${queryOptionsHash}.${timestampValueExpression}.getTextIndexKeyValues`; + const cacheKey = `${databaseName}.${tableName}.${connectionId}.${dateRange[0].toString()}.${dateRange[1].toString()}.${queryOptionsHash}.${maxParts}.${partSelection}.${timestampValueExpression}.getTextIndexKeyValues`; return this.cache.getOrFetch(cacheKey, async () => { try { const sqlBranches: Array = []; @@ -1856,6 +1869,8 @@ export class Metadata { tableName, dateRange, timestampValueExpression, + maxParts, + partSelection, }); const sql = chSql` SELECT * FROM ( @@ -2920,6 +2935,8 @@ export class Metadata { maxValuesPerKey, connectionId, metadataMVs, + mapKeyIndexMaxParts = DEFAULT_MAP_KEY_INDEX_MAX_PARTS, + mapKeyIndexPartSelection = MapKeyIndexPartSelection.Median, dateRange, timestampValueExpression, signal, @@ -3039,6 +3056,8 @@ export class Metadata { tableName, connectionId, queryOptions: mapTextIndexQueryOptions, + maxParts: mapKeyIndexMaxParts, + partSelection: mapKeyIndexPartSelection, dateRange, timestampValueExpression, signal, @@ -3052,6 +3071,8 @@ export class Metadata { tableName, connectionId, queryOptions: nativeTextIndexQueryOptions, + maxParts: mapKeyIndexMaxParts, + partSelection: mapKeyIndexPartSelection, dateRange, timestampValueExpression, signal, diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 4210af9dd5..73e0236caf 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -2810,7 +2810,7 @@ export type MetadataMaterializedViews = z.infer< >; /** - * How many parts Map key discovery reads from a text index. Unset uses + * How many parts key and value discovery reads from a text index. Unset uses * DEFAULT_MAP_KEY_INDEX_MAX_PARTS. A cleared form NumberInput yields `''`. */ const MapKeyIndexMaxPartsSchema = z.preprocess( From d13a79418919d7ba589753d147787e0d1fa46763 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:57:12 -0500 Subject: [PATCH 04/10] fix: let "load more" read every part in the time range --- .changeset/map-key-index-median-parts.md | 15 ++++++----- .../DBSearchPageFilters/hooks.test.tsx | 3 +++ .../components/DBSearchPageFilters/hooks.ts | 4 +-- .../src/__tests__/metadata.test.ts | 26 +++++++++++++++++++ packages/common-utils/src/core/metadata.ts | 17 +++++++----- 5 files changed, 50 insertions(+), 15 deletions(-) diff --git a/.changeset/map-key-index-median-parts.md b/.changeset/map-key-index-median-parts.md index 1dbf073f47..62e41e1ad7 100644 --- a/.changeset/map-key-index-median-parts.md +++ b/.changeset/map-key-index-median-parts.md @@ -4,11 +4,12 @@ '@hyperdx/app': patch --- -perf: read only 20 median-sized parts when discovering Map keys from a text index +perf: read only 20 median-sized parts for text index key and value lookups -Map key discovery read the text index dictionary of every part in the time -range, which is slow on tables with many parts. It now reads the 20 parts -closest to the median size. Log and trace sources can change the count with -the new "Map key index parts" setting (`mapKeyIndexMaxParts`), and which parts -are read with "Map key index part size" (`mapKeyIndexPartSelection`: median, -smallest, or largest). +Map key discovery and "all values" lookups on text indexes read the index +dictionary of every part in the time range, which is slow on tables with many +parts. They now read the 20 parts closest to the median size. Log and trace +sources can change the count with the new "Map key index parts" setting +(`mapKeyIndexMaxParts`), and which parts are read with "Map key index part size" +(`mapKeyIndexPartSelection`: median, smallest, or largest). "Load more" on a +single filter key still reads every part in the time range. diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx b/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx index 62330b0f53..d55cf10f72 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx +++ b/packages/app/src/components/DBSearchPageFilters/hooks.test.tsx @@ -866,6 +866,9 @@ describe('useFetchFacets', () => { }); expect(getAllKeyValues).toHaveBeenCalledTimes(1); + expect(getAllKeyValues).toHaveBeenCalledWith( + expect.objectContaining({ mapKeyIndexMaxParts: null }), + ); expect(getKeyValuesWithMVs).not.toHaveBeenCalled(); expect(result.current.data.keyValues).toEqual([ { key: 'ServiceName', value: ['api', 'web'] }, diff --git a/packages/app/src/components/DBSearchPageFilters/hooks.ts b/packages/app/src/components/DBSearchPageFilters/hooks.ts index 28bbb604a4..dc2007eee9 100644 --- a/packages/app/src/components/DBSearchPageFilters/hooks.ts +++ b/packages/app/src/components/DBSearchPageFilters/hooks.ts @@ -271,8 +271,8 @@ function useFacets({ tableName: tableConnection.tableName, connectionId: tableConnection.connectionId, metadataMVs: tableConnection.metadataMVs, - mapKeyIndexMaxParts: tableConnection.mapKeyIndexMaxParts, - mapKeyIndexPartSelection: tableConnection.mapKeyIndexPartSelection, + // "Load more" wants every value in the time range, not a sample of parts. + mapKeyIndexMaxParts: null, keyExpressions: [sqlKey], maxValuesPerKey: SINGLE_KEY_VALUES_LIMIT, dateRange, diff --git a/packages/common-utils/src/__tests__/metadata.test.ts b/packages/common-utils/src/__tests__/metadata.test.ts index f3c8f49678..8ea6193746 100644 --- a/packages/common-utils/src/__tests__/metadata.test.ts +++ b/packages/common-utils/src/__tests__/metadata.test.ts @@ -1434,6 +1434,32 @@ describe('Metadata', () => { }, ); + it.each([ + ['map KV', "LogAttributes['requestId']"], + ['native', 'TraceId'], + ])( + 'reads every part in the time range from the %s text index when the part limit is null', + async (_, keyExpression) => { + setupDefaultLogsSchema(); + + await metadata.getAllKeyValues({ + ...baseArgs, + keyExpressions: [keyExpression], + mapKeyIndexMaxParts: null, + }); + + const textIndexCall = ( + mockClickhouseClient.query as jest.Mock + ).mock.calls.find(([opts]) => + opts.query.includes('mergeTreeTextIndex('), + ); + expect(textIndexCall[0].query).toMatch( + /part_name IN \(SELECT name\s+FROM system\.parts/, + ); + expect(textIndexCall[0].query).not.toContain('row_number()'); + }, + ); + it.each([ ['map KV', "LogAttributes['requestId']"], ['native', 'TraceId'], diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 75a9c2f42c..6409a4b7d2 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -112,8 +112,11 @@ type AllKeyValuesQuery = { maxValuesPerKey?: number; connectionId: string; metadataMVs?: MetadataMaterializedViews; - /** Parts the text index paths read; see `partsOverlapFilter`. */ - mapKeyIndexMaxParts?: number; + /** + * Parts the text index paths read; see `partsOverlapFilter`. Unset uses the + * default; null reads every part in the time range. + */ + mapKeyIndexMaxParts?: number | null; mapKeyIndexPartSelection?: MapKeyIndexPartSelection; dateRange: [Date, Date]; timestampValueExpression: string; @@ -1754,7 +1757,8 @@ export class Metadata { signal, }: TableConnection & { queryOptions: TextIndexMapColumnQueryOptions; - maxParts: number; + /** null reads every part in the time range. */ + maxParts: number | null; partSelection: MapKeyIndexPartSelection; dateRange: [Date, Date]; timestampValueExpression: string; @@ -1784,7 +1788,7 @@ export class Metadata { tableName, dateRange, timestampValueExpression, - maxParts, + maxParts: maxParts ?? undefined, partSelection, }); const valueSql = chSql`substring(token, position(token, ${{ String: info.separator }}) + ${{ Int32: info.separator.length }})`; @@ -1852,7 +1856,8 @@ export class Metadata { signal, }: TableConnection & { queryOptions: TextIndexColumnQueryOptions; - maxParts: number; + /** null reads every part in the time range. */ + maxParts: number | null; partSelection: MapKeyIndexPartSelection; dateRange: [Date, Date]; timestampValueExpression: string; @@ -1869,7 +1874,7 @@ export class Metadata { tableName, dateRange, timestampValueExpression, - maxParts, + maxParts: maxParts ?? undefined, partSelection, }); const sql = chSql` From c66706a2f9ca797f40f86c7ad6ec61ee312bfa55 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:15:26 -0500 Subject: [PATCH 05/10] fix: refetch filter values when a source's part settings change --- .../src/hooks/__tests__/useMetadata.test.tsx | 28 +++++++++++++++ packages/app/src/hooks/useMetadata.tsx | 35 +++++++++++-------- 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/packages/app/src/hooks/__tests__/useMetadata.test.tsx b/packages/app/src/hooks/__tests__/useMetadata.test.tsx index 58fd549a08..9f3ed73a7f 100644 --- a/packages/app/src/hooks/__tests__/useMetadata.test.tsx +++ b/packages/app/src/hooks/__tests__/useMetadata.test.tsx @@ -454,6 +454,34 @@ describe('useGetKeyValues', () => { ).toEqual([{ kvRollupTable: 'logs_kv' }, { kvRollupTable: 'traces_kv' }]); }); + it("refetches 'all' mode values when the source's part settings change", async () => { + const defaultSources = jest.mocked(useSources)(); + const withParts = (mapKeyIndexMaxParts: number) => + jest.mocked(useSources).mockReturnValue({ + data: [{ id: 'logs', kind: 'log', mapKeyIndexMaxParts }], + isLoading: false, + } as any); + const streamAllKeyValues = mockLookups.all([], []); + + try { + withParts(20); + const { result, rerender } = renderValues({ + mode: 'all', + chartConfigs: [logsConfig], + }); + await waitForSettled(result); + + withParts(5); + rerender(); + await waitFor(() => expect(streamAllKeyValues).toHaveBeenCalledTimes(2)); + expect( + streamAllKeyValues.mock.calls.map(([args]) => args.mapKeyIndexMaxParts), + ).toEqual([20, 5]); + } finally { + jest.mocked(useSources).mockReturnValue(defaultSources); + } + }); + it("passes limit to 'all' mode lookups as maxValuesPerKey", async () => { const streamAllKeyValues = mockLookups.all([]); diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index 75766fbe16..ddda65d147 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -530,12 +530,27 @@ export function useMultipleGetKeyValues( ? { retry, structuralSharing, refetchOnWindowFocus: true } : {}; + // Resolved here, not in the stream, so editing a source's part settings + // changes the query key instead of serving cached facets. + const partSettings = chartConfigsArr.map(chartConfig => { + const source = chartConfig.source + ? sources?.find(s => s.id === chartConfig.source) + : undefined; + return source && (isLogSource(source) || isTraceSource(source)) + ? { + mapKeyIndexMaxParts: source.mapKeyIndexMaxParts, + mapKeyIndexPartSelection: source.mapKeyIndexPartSelection, + } + : {}; + }); + const query = useStreamingQuery({ queryKey: [ 'useMetadata.useGetKeyValues', mode, metadataMVsOverride, ...chartConfigsArr.map(cc => ({ ...cc })), + partSettings, ...keys, // Serialized: react-query hashes keys with JSON.stringify, which would // flatten every distinct Set selection to `{}`. @@ -573,26 +588,16 @@ export function useMultipleGetKeyValues( }, ); - const lookups = chartConfigsArr.map(chartConfig => { + const lookups = chartConfigsArr.map((chartConfig, i) => { const source = chartConfig.source ? sources?.find(s => s.id === chartConfig.source) : undefined; - const isLogOrTrace = - !!source && (isLogSource(source) || isTraceSource(source)); const metadataMVs = usableMVsOverride ?? - (isLogOrTrace ? source.metadataMaterializedViews : undefined); - return { - chartConfig, - source, - metadataMVs, - mapKeyIndexMaxParts: isLogOrTrace - ? source.mapKeyIndexMaxParts - : undefined, - mapKeyIndexPartSelection: isLogOrTrace - ? source.mapKeyIndexPartSelection - : undefined, - }; + (source && (isLogSource(source) || isTraceSource(source)) + ? source.metadataMaterializedViews + : undefined); + return { chartConfig, source, metadataMVs, ...partSettings[i] }; }); if (mode === 'exact') { From 9085488dd3789d3af9cc8528d1cb4215038f7afa Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:23:07 -0500 Subject: [PATCH 06/10] refactor: read source part settings with tcFromSource --- packages/app/src/hooks/useMetadata.tsx | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/packages/app/src/hooks/useMetadata.tsx b/packages/app/src/hooks/useMetadata.tsx index ddda65d147..94a9fd3026 100644 --- a/packages/app/src/hooks/useMetadata.tsx +++ b/packages/app/src/hooks/useMetadata.tsx @@ -13,6 +13,7 @@ import { parseKeyPath, TableConnection, TableMetadata, + tcFromSource, } from '@hyperdx/common-utils/dist/core/metadata'; import { FilterState, @@ -533,15 +534,12 @@ export function useMultipleGetKeyValues( // Resolved here, not in the stream, so editing a source's part settings // changes the query key instead of serving cached facets. const partSettings = chartConfigsArr.map(chartConfig => { - const source = chartConfig.source - ? sources?.find(s => s.id === chartConfig.source) - : undefined; - return source && (isLogSource(source) || isTraceSource(source)) - ? { - mapKeyIndexMaxParts: source.mapKeyIndexMaxParts, - mapKeyIndexPartSelection: source.mapKeyIndexPartSelection, - } - : {}; + const { mapKeyIndexMaxParts, mapKeyIndexPartSelection } = tcFromSource( + chartConfig.source + ? sources?.find(s => s.id === chartConfig.source) + : undefined, + ); + return { mapKeyIndexMaxParts, mapKeyIndexPartSelection }; }); const query = useStreamingQuery({ From b795c0178ceee5e22ef5fa3c2f3df40cc673d785 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:32:22 -0500 Subject: [PATCH 07/10] test: pass source part settings to 'all' mode value lookups --- .../src/hooks/__tests__/useMetadata.test.tsx | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/packages/app/src/hooks/__tests__/useMetadata.test.tsx b/packages/app/src/hooks/__tests__/useMetadata.test.tsx index 9f3ed73a7f..2957db0819 100644 --- a/packages/app/src/hooks/__tests__/useMetadata.test.tsx +++ b/packages/app/src/hooks/__tests__/useMetadata.test.tsx @@ -454,6 +454,56 @@ describe('useGetKeyValues', () => { ).toEqual([{ kvRollupTable: 'logs_kv' }, { kvRollupTable: 'traces_kv' }]); }); + it("passes log and trace sources' part settings to 'all' mode lookups, and no others", async () => { + const defaultSources = jest.mocked(useSources)(); + jest.mocked(useSources).mockReturnValue({ + data: [ + { + id: 'logs', + kind: 'log', + mapKeyIndexMaxParts: 7, + mapKeyIndexPartSelection: 'largest', + }, + // Not a log or trace source, so its fields don't apply. + { + id: 'metrics', + kind: 'metric', + mapKeyIndexMaxParts: 9, + mapKeyIndexPartSelection: 'smallest', + }, + ], + isLoading: false, + } as any); + const streamAllKeyValues = mockLookups.all([], []); + + try { + const { result } = renderValues({ + mode: 'all', + chartConfigs: [ + logsConfig, + createMockChartConfig({ + source: 'metrics', + from: { databaseName: 'db', tableName: 'metrics' }, + }), + ], + }); + await waitForSettled(result); + + expect( + streamAllKeyValues.mock.calls.map(([args]) => [ + args.tableName, + args.mapKeyIndexMaxParts, + args.mapKeyIndexPartSelection, + ]), + ).toEqual([ + ['logs', 7, 'largest'], + ['metrics', undefined, undefined], + ]); + } finally { + jest.mocked(useSources).mockReturnValue(defaultSources); + } + }); + it("refetches 'all' mode values when the source's part settings change", async () => { const defaultSources = jest.mocked(useSources)(); const withParts = (mapKeyIndexMaxParts: number) => From caca3ff739daafdf2208bb8e75a75b39ab0113b3 Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:39:08 -0500 Subject: [PATCH 08/10] refactor: read describe_source settings with tcFromSource --- .../mcp/__tests__/describeSourceTimeout.test.ts | 2 ++ .../api/src/mcp/tools/sources/describeSource.ts | 17 ++++++----------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/packages/api/src/mcp/__tests__/describeSourceTimeout.test.ts b/packages/api/src/mcp/__tests__/describeSourceTimeout.test.ts index 75d4d99a62..f06bde491c 100644 --- a/packages/api/src/mcp/__tests__/describeSourceTimeout.test.ts +++ b/packages/api/src/mcp/__tests__/describeSourceTimeout.test.ts @@ -12,6 +12,8 @@ jest.mock('@/controllers/connection', () => ({ jest.mock('@/clickhouse', () => ({ ClickhouseClient: jest.fn() })); jest.mock('@hyperdx/common-utils/dist/core/metadata', () => ({ getMetadata: () => mockMetadata, + tcFromSource: jest.requireActual('@hyperdx/common-utils/dist/core/metadata') + .tcFromSource, })); jest.mock('@/controllers/promql', () => ({ getPromqlLabelCatalog: jest.fn(), diff --git a/packages/api/src/mcp/tools/sources/describeSource.ts b/packages/api/src/mcp/tools/sources/describeSource.ts index 4811a036c7..c096429d32 100644 --- a/packages/api/src/mcp/tools/sources/describeSource.ts +++ b/packages/api/src/mcp/tools/sources/describeSource.ts @@ -3,7 +3,10 @@ import { filterColumnMetaByType, JSDataType, } from '@hyperdx/common-utils/dist/clickhouse'; -import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { + getMetadata, + tcFromSource, +} from '@hyperdx/common-utils/dist/core/metadata'; import { type MetricTable, SourceKind } from '@hyperdx/common-utils/dist/types'; import { trace } from '@opentelemetry/api'; import SqlString from 'sqlstring'; @@ -224,16 +227,8 @@ async function describeSourceSchema( // Shared by stages 2–4 so map-key discovery can use rollup tables // instead of falling back to expensive main-table scans. - const metadataMVs = - 'metadataMaterializedViews' in source - ? source.metadataMaterializedViews - : undefined; - const mapKeyIndexMaxParts = - 'mapKeyIndexMaxParts' in source ? source.mapKeyIndexMaxParts : undefined; - const mapKeyIndexPartSelection = - 'mapKeyIndexPartSelection' in source - ? source.mapKeyIndexPartSelection - : undefined; + const { metadataMVs, mapKeyIndexMaxParts, mapKeyIndexPartSelection } = + tcFromSource(source); const now = new Date(); const dateRange: [Date, Date] = [ From 10c5608e07f213ec66f8d2490ca3e63970f9542a Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:35:05 -0500 Subject: [PATCH 09/10] fix: pick the part order with a switch --- packages/common-utils/src/core/metadata.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/common-utils/src/core/metadata.ts b/packages/common-utils/src/core/metadata.ts index 6409a4b7d2..81a93eedad 100644 --- a/packages/common-utils/src/core/metadata.ts +++ b/packages/common-utils/src/core/metadata.ts @@ -936,11 +936,17 @@ export class Metadata { } // Rank by size. Median picks from the middle out: tiny unmerged parts // hold few keys and the largest merged parts have the biggest dictionaries. - const order = { - [MapKeyIndexPartSelection.Median]: 'abs(rn - (cnt + 1) / 2), name', - [MapKeyIndexPartSelection.Smallest]: 'rn', - [MapKeyIndexPartSelection.Largest]: 'rn DESC', - }[partSelection]; + let order: string; + switch (partSelection) { + case MapKeyIndexPartSelection.Smallest: + order = 'rn'; + break; + case MapKeyIndexPartSelection.Largest: + order = 'rn DESC'; + break; + default: + order = 'abs(rn - (cnt + 1) / 2), name'; + } return chSql`part_name IN ( SELECT name FROM ( SELECT name, From f02de10219085d3f1af727acf391a09c623a620d Mon Sep 17 00:00:00 2001 From: Aaron Knudtson <87577305+knudtty@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:01:33 -0500 Subject: [PATCH 10/10] test: mock sources without as any --- .../src/hooks/__tests__/useMetadata.test.tsx | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/packages/app/src/hooks/__tests__/useMetadata.test.tsx b/packages/app/src/hooks/__tests__/useMetadata.test.tsx index 2957db0819..031caa81f6 100644 --- a/packages/app/src/hooks/__tests__/useMetadata.test.tsx +++ b/packages/app/src/hooks/__tests__/useMetadata.test.tsx @@ -454,26 +454,30 @@ describe('useGetKeyValues', () => { ).toEqual([{ kvRollupTable: 'logs_kv' }, { kvRollupTable: 'traces_kv' }]); }); - it("passes log and trace sources' part settings to 'all' mode lookups, and no others", async () => { - const defaultSources = jest.mocked(useSources)(); + // Sources here only carry the fields the hook reads. + const mockSources = (data: Record[]) => jest.mocked(useSources).mockReturnValue({ - data: [ - { - id: 'logs', - kind: 'log', - mapKeyIndexMaxParts: 7, - mapKeyIndexPartSelection: 'largest', - }, - // Not a log or trace source, so its fields don't apply. - { - id: 'metrics', - kind: 'metric', - mapKeyIndexMaxParts: 9, - mapKeyIndexPartSelection: 'smallest', - }, - ], + data, isLoading: false, - } as any); + } as unknown as ReturnType); + + it("passes log and trace sources' part settings to 'all' mode lookups, and no others", async () => { + const defaultSources = jest.mocked(useSources)(); + mockSources([ + { + id: 'logs', + kind: 'log', + mapKeyIndexMaxParts: 7, + mapKeyIndexPartSelection: 'largest', + }, + // Not a log or trace source, so its fields don't apply. + { + id: 'metrics', + kind: 'metric', + mapKeyIndexMaxParts: 9, + mapKeyIndexPartSelection: 'smallest', + }, + ]); const streamAllKeyValues = mockLookups.all([], []); try { @@ -507,10 +511,7 @@ describe('useGetKeyValues', () => { it("refetches 'all' mode values when the source's part settings change", async () => { const defaultSources = jest.mocked(useSources)(); const withParts = (mapKeyIndexMaxParts: number) => - jest.mocked(useSources).mockReturnValue({ - data: [{ id: 'logs', kind: 'log', mapKeyIndexMaxParts }], - isLoading: false, - } as any); + mockSources([{ id: 'logs', kind: 'log', mapKeyIndexMaxParts }]); const streamAllKeyValues = mockLookups.all([], []); try {