From 465b9c52319f4f54cfd67de5c1daa54632014b40 Mon Sep 17 00:00:00 2001 From: Alex Fedotyev <61838744+alex-fedotyev@users.noreply.github.com> Date: Fri, 20 Nov 2026 05:55:23 +0000 Subject: [PATCH] fix(common-utils): merge DDSketch and GK quantile states with their accuracy A materialized view column built with quantileDD(accuracy, level) or quantileGK(accuracy, level) was merged as quantileDDMerge(level), which ClickHouse reads as an accuracy, so p95 came back as a different quantile with no error. GK merges failed the EXPLAIN check, so those views were skipped. The optimizer now reads the sketch's accuracy from the column type and passes it to the renderer, which emits quantileDDMerge(0.01, 0.95). Sketches whose state doesn't depend on the level (t-digest, reservoir, BFloat16, exact) render as before. Co-Authored-By: Claude Opus 5.5 --- .changeset/mv-quantile-sketch-accuracy.md | 7 + ...terializedViewQuantileSketches.int.test.ts | 216 ++++++++++++++++++ .../src/__tests__/renderChartConfig.test.ts | 52 +++++ .../__tests__/materializedViews.test.ts | 105 +++++++++ .../src/core/materializedViews.ts | 37 ++- .../src/core/renderChartConfig.ts | 14 +- packages/common-utils/src/types.ts | 5 + 7 files changed, 429 insertions(+), 7 deletions(-) create mode 100644 .changeset/mv-quantile-sketch-accuracy.md create mode 100644 packages/common-utils/src/__tests__/materializedViewQuantileSketches.int.test.ts diff --git a/.changeset/mv-quantile-sketch-accuracy.md b/.changeset/mv-quantile-sketch-accuracy.md new file mode 100644 index 0000000000..25bd74a2df --- /dev/null +++ b/.changeset/mv-quantile-sketch-accuracy.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/app': patch +'@hyperdx/api': patch +--- + +fix: read percentiles correctly from materialized views that store DDSketch (`quantileDD`) or GK (`quantileGK`) states. The merge now passes the sketch's accuracy ahead of the level, so charts and alerts no longer return a different percentile without an error, and GK views are no longer skipped. diff --git a/packages/common-utils/src/__tests__/materializedViewQuantileSketches.int.test.ts b/packages/common-utils/src/__tests__/materializedViewQuantileSketches.int.test.ts new file mode 100644 index 0000000000..cce58fbaf3 --- /dev/null +++ b/packages/common-utils/src/__tests__/materializedViewQuantileSketches.int.test.ts @@ -0,0 +1,216 @@ +import type { ClickHouseClient } from '@clickhouse/client'; +import { createClient } from '@clickhouse/client'; + +import { parameterizedQueryToSql } from '@/clickhouse'; +import { ClickhouseClient as HdxClickhouseClient } from '@/clickhouse/node'; +import { tryOptimizeConfigWithMaterializedViewWithExplanations } from '@/core/materializedViews'; +import { Metadata, MetadataCache } from '@/core/metadata'; +import { renderChartConfig } from '@/core/renderChartConfig'; +import { + BuilderChartConfigWithOptDateRange, + DisplayType, + SourceKind, + TTraceSource, +} from '@/types'; + +// Percentiles read from a materialized view have to match the raw table +// whichever quantile sketch the view stores. Sketches built with an accuracy +// (DDSketch, GK) only merge correctly when the merge is given that accuracy +// ahead of the level. +describe('materialized view quantile sketches (integration)', () => { + let client: ClickHouseClient; + let hdxClient: HdxClickhouseClient; + let metadata: Metadata; + + const DB = 'default'; + const SPANS_TABLE = 'test_mv_sketch_spans'; + const SKETCHES = [ + { name: 'quantileDD', params: '0.01, 0.5', tolerance: 0.01 }, + { name: 'quantileGK', params: '1000, 0.5', tolerance: 0.01 }, + { name: 'quantileTDigest', params: '0.5', tolerance: 0.02 }, + { name: 'quantileBFloat16', params: '0.5', tolerance: 0.01 }, + ]; + const rollupTable = (sketch: string) => `test_mv_sketch_rollup_${sketch}`; + + const sourceFor = (sketch: string) => + ({ + id: 'test-traces', + kind: SourceKind.Trace, + name: 'Traces', + connection: 'test-connection', + from: { databaseName: DB, tableName: SPANS_TABLE }, + timestampValueExpression: 'Timestamp', + durationExpression: 'Duration', + materializedViews: [ + { + databaseName: DB, + tableName: rollupTable(sketch), + dimensionColumns: 'ServiceName', + minGranularity: '1 minute', + timestampColumn: 'Timestamp', + aggregatedColumns: [ + { aggFn: 'count', mvColumn: 'count', sourceColumn: '' }, + { + aggFn: 'quantile', + mvColumn: 'quantile__Duration', + sourceColumn: 'Duration', + }, + ], + }, + ], + }) as TTraceSource; + + // Five-minute buckets, so each bucket merges five one-minute sketch states. + const percentileConfig = ( + level: number, + ): BuilderChartConfigWithOptDateRange => ({ + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: DB, tableName: SPANS_TABLE }, + select: [ + { + alias: 'percentile', + aggFn: 'quantile', + level, + valueExpression: 'Duration', + aggCondition: '', + }, + ], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'Timestamp', + dateRange: [ + new Date('2025-06-01T00:00:00Z'), + new Date('2025-06-01T00:10:00Z'), + ], + granularity: '5 minute', + }); + + async function queryRows(sql: string) { + const result = await client.query({ query: sql, format: 'JSONEachRow' }); + return (await result.json()) as Record[]; + } + + beforeAll(async () => { + const host = process.env.CLICKHOUSE_HOST || 'http://localhost:8123'; + const username = process.env.CLICKHOUSE_USER || 'default'; + const password = process.env.CLICKHOUSE_PASSWORD || ''; + + client = createClient({ url: host, username, password }); + hdxClient = new HdxClickhouseClient({ host, username, password }); + + await client.command({ + query: ` + CREATE OR REPLACE TABLE ${DB}.${SPANS_TABLE} ( + Timestamp DateTime64(9), + ServiceName LowCardinality(String), + Duration UInt64 + ) + ENGINE = MergeTree() + ORDER BY (Timestamp) + `, + }); + // 100,000 spans over ten minutes. Durations from 1 microsecond to + // 100 milliseconds are permuted across the spans, so every minute holds + // a spread of the whole range. + await client.command({ + query: ` + INSERT INTO ${DB}.${SPANS_TABLE} + SELECT + toDateTime64('2025-06-01 00:00:00', 9, 'UTC') + toIntervalMillisecond(number * 6), + 'api', + ((number * 7919) % 100000 + 1) * 1000 + FROM numbers(100000) + `, + }); + + for (const { name, params } of SKETCHES) { + await client.command({ + query: ` + CREATE OR REPLACE TABLE ${DB}.${rollupTable(name)} ( + Timestamp DateTime, + ServiceName LowCardinality(String), + count SimpleAggregateFunction(sum, UInt64), + quantile__Duration AggregateFunction(${name}(${params}), UInt64) + ) + ENGINE = AggregatingMergeTree() + ORDER BY (Timestamp, ServiceName) + `, + }); + await client.command({ + query: ` + INSERT INTO ${DB}.${rollupTable(name)} + SELECT + toStartOfMinute(Timestamp) AS Timestamp, + ServiceName, + count() AS count, + ${name}State(${params})(Duration) AS quantile__Duration + FROM ${DB}.${SPANS_TABLE} + GROUP BY Timestamp, ServiceName + `, + }); + } + }); + + beforeEach(() => { + metadata = new Metadata(hdxClient, new MetadataCache()); + }); + + afterAll(async () => { + for (const { name } of SKETCHES) { + await client.command({ + query: `DROP TABLE IF EXISTS ${DB}.${rollupTable(name)}`, + }); + } + await client.command({ + query: `DROP TABLE IF EXISTS ${DB}.${SPANS_TABLE}`, + }); + await hdxClient.close(); + await client.close(); + }); + + it.each(SKETCHES)( + 'reads p95 and p99 from a $name rollup within its error of the raw table', + async ({ name, tolerance }) => { + for (const level of [0.95, 0.99]) { + const { optimizedConfig } = + await tryOptimizeConfigWithMaterializedViewWithExplanations( + percentileConfig(level), + metadata, + hdxClient, + undefined, + sourceFor(name), + ); + expect(optimizedConfig?.from.tableName).toBe(rollupTable(name)); + if (optimizedConfig == null) { + return; + } + + const rollupRows = await queryRows( + parameterizedQueryToSql( + await renderChartConfig(optimizedConfig, metadata, undefined), + ), + ); + const exactRows = await queryRows(` + SELECT + toStartOfInterval(toDateTime(Timestamp), INTERVAL 5 MINUTE) AS __hdx_time_bucket, + quantileExact(${level})(Duration) AS percentile + FROM ${DB}.${SPANS_TABLE} + GROUP BY __hdx_time_bucket + `); + const exactByBucket = new Map( + exactRows.map(row => [row.__hdx_time_bucket, Number(row.percentile)]), + ); + + expect(rollupRows).toHaveLength(2); + for (const row of rollupRows) { + const exact = exactByBucket.get(row.__hdx_time_bucket); + expect(exact).toBeDefined(); + const relativeError = + Math.abs(Number(row.percentile) - (exact ?? 0)) / (exact ?? 1); + expect(relativeError).toBeLessThanOrEqual(tolerance); + } + } + }, + ); +}); diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index e048142170..6e098084f5 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -3396,6 +3396,58 @@ describe('renderChartConfig', () => { expect(actual).toMatchSnapshot(); }); + it('should merge a quantile sketch with its accuracy ahead of the level', async () => { + // Built the way the materialized view optimizer builds it: a quantile + // select item with its aggFn swapped for the sketch's merge function. + const sketchMerge = ( + level: number, + condition?: { aggCondition: string; aggConditionLanguage: 'lucene' }, + ) => ({ + ...{ + aggFn: 'quantile' as const, + level, + valueExpression: 'quantile__Duration', + aggCondition: '', + ...condition, + }, + aggFn: 'quantileDDMerge', + sketchParams: [0.01], + }); + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Table, + connection: 'test-connection', + from: { + databaseName: 'default', + tableName: 'logs', + }, + select: [ + sketchMerge(0.95), + sketchMerge(0.99, { + aggCondition: 'severity:"ERROR"', + aggConditionLanguage: 'lucene', + }), + ], + where: '', + whereLanguage: 'sql', + groupBy: 'severity', + timestampValueExpression: 'timestamp', + dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], + }; + + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toContain( + 'quantileDDMerge(0.01, 0.95)(quantile__Duration)', + ); + expect(actual).toContain( + "quantileDDMergeIf(0.01, 0.99)(quantile__Duration, ((severity = 'ERROR')) AND", + ); + }); + it('should generate SQL for an histogram merge function', async () => { const config: ChartConfigWithOptDateRange = { displayType: DisplayType.Table, diff --git a/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts b/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts index f25ad35367..f44c8bffb5 100644 --- a/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts +++ b/packages/common-utils/src/clickhouse/__tests__/materializedViews.test.ts @@ -399,6 +399,111 @@ describe('materializedViews', () => { expect(result.errors).toBeUndefined(); }); + it.each([ + { + type: 'AggregateFunction(quantileDD(0.01, 0.5), UInt64)', + aggFn: 'quantileDDMerge', + sketchParams: [0.01], + }, + { + type: 'AggregateFunction(quantilesDD(0.001, 0.9, 0.99), UInt64)', + aggFn: 'quantileDDMerge', + sketchParams: [0.001], + }, + { + type: 'AggregateFunction(quantileGK(100, 0.5), UInt64)', + aggFn: 'quantileGKMerge', + sketchParams: [100], + }, + ])( + 'should carry the accuracy of a $type select into the merge', + async ({ type, aggFn, sketchParams }) => { + const sketchMetadata: Metadata = { + getColumn: jest.fn().mockImplementation(({ column }) => { + if (column === 'quantile__Duration') { + return { type } as unknown as ColumnMeta; + } + return undefined; + }), + } as unknown as Metadata; + + const chartConfig: ChartConfigWithOptDateRange = { + from: { + databaseName: 'default', + tableName: 'otel_spans', + }, + select: [ + { + valueExpression: 'Duration', + aggFn: 'quantile', + level: 0.95, + }, + ], + where: '', + connection: 'test-connection', + }; + + const result = await tryConvertConfigToMaterializedViewSelect( + chartConfig, + MV_CONFIG_METRIC_ROLLUP_1M, + sketchMetadata, + ); + + expect(result.optimizedConfig?.select).toEqual([ + { + valueExpression: 'quantile__Duration', + aggFn, + level: 0.95, + sketchParams, + }, + ]); + expect(result.errors).toBeUndefined(); + }, + ); + + it('should not carry sketch parameters for sketches merged by level alone', async () => { + const bfloat16Metadata: Metadata = { + getColumn: jest.fn().mockImplementation(({ column }) => { + if (column === 'quantile__Duration') { + return { + type: 'AggregateFunction(quantileBFloat16(0.5), UInt64)', + } as unknown as ColumnMeta; + } + return undefined; + }), + } as unknown as Metadata; + + const chartConfig: ChartConfigWithOptDateRange = { + from: { + databaseName: 'default', + tableName: 'otel_spans', + }, + select: [ + { + valueExpression: 'Duration', + aggFn: 'quantile', + level: 0.99, + }, + ], + where: '', + connection: 'test-connection', + }; + + const result = await tryConvertConfigToMaterializedViewSelect( + chartConfig, + MV_CONFIG_METRIC_ROLLUP_1M, + bfloat16Metadata, + ); + + expect(result.optimizedConfig?.select).toEqual([ + { + valueExpression: 'quantile__Duration', + aggFn: 'quantileBFloat16Merge', + level: 0.99, + }, + ]); + }); + it('should convert a histogram AggregateFunction select', async () => { const chartConfig: ChartConfigWithOptDateRange = { from: { diff --git a/packages/common-utils/src/core/materializedViews.ts b/packages/common-utils/src/core/materializedViews.ts index e7ba7d132d..1b1490f69c 100644 --- a/packages/common-utils/src/core/materializedViews.ts +++ b/packages/common-utils/src/core/materializedViews.ts @@ -221,12 +221,19 @@ async function isSimpleAggregateFunction( } } +// Quantile sketches whose state is built with an accuracy ahead of the level, +// e.g. quantileDD(relative_accuracy, level) and quantileGK(accuracy, level). +// Their -Merge has to be given the same accuracy: with the level alone, +// ClickHouse reads the level as the accuracy and returns a different quantile +// without raising an error. +const ACCURACY_PARAMETERIZED_QUANTILES = new Set(['quantileDD', 'quantileGK']); + // Variants of quantile (ex. quantileExact, quantileDD, etc.) async function getQuantileAggregateFunction( tableConnection: TableConnection, column: string, metadata: Metadata, -) { +): Promise<{ name: string; sketchParams?: number[] } | undefined> { try { const columnMeta = await metadata.getColumn({ ...tableConnection, @@ -238,16 +245,31 @@ async function getQuantileAggregateFunction( return undefined; } - // Use regex to extract the quantile function name inside AggregateFunction(...) + // Use regex to extract the quantile function name and its parameters inside AggregateFunction(...) // For example, AggregateFunction(quantile(0.95), Int64) --> quantile // AggregateFunction(quantileTDigest(0.95), Int64) --> quantileTDigest - // AggregateFunction(quantileDD(0.001, 0.95), Int64) --> quantileDD + // AggregateFunction(quantileDD(0.001, 0.95), Int64) --> quantileDD, accuracy 0.001 // The plural `quantiles*` variants return arrays, but a select item carries a // single `level`, so we normalize to the singular form to pull a scalar value. // AggregateFunction(quantiles(0.9, 0.95), UInt64) --> quantile // AggregateFunction(quantilesTDigest(0.9, 0.95), UInt64) --> quantileTDigest - const match = type.match(/^AggregateFunction\(\s*([^(, ]+)\s*\(/); - return match?.[1]?.replace(/^quantiles/, 'quantile'); + // AggregateFunction(quantilesDD(0.01, 0.9, 0.95), UInt64) --> quantileDD, accuracy 0.01 + const match = type.match(/^AggregateFunction\(\s*([^(, ]+)\s*\(([^)]*)\)/); + const name = match?.[1]?.replace(/^quantiles/, 'quantile'); + if (!name) { + return undefined; + } + if (!ACCURACY_PARAMETERIZED_QUANTILES.has(name)) { + return { name }; + } + + // The accuracy is the first parameter in both the singular and plural forms. + // Without it the merge can't match the state, so leave the select item + // unconverted and let the EXPLAIN check reject this view. + const accuracy = Number(match?.[2]?.split(',')[0]?.trim()); + return Number.isFinite(accuracy) && accuracy > 0 + ? { name, sketchParams: [accuracy] } + : undefined; } catch { return undefined; } @@ -439,6 +461,7 @@ async function convertSelectToMaterializedViewSelect( ); } + let sketchParams: number[] | undefined; if (isQuantileSelectItem(selectItem)) { const quantileAggregateFunction = await getQuantileAggregateFunction( mvTableConnection, @@ -446,7 +469,8 @@ async function convertSelectToMaterializedViewSelect( metadata, ); if (quantileAggregateFunction) { - aggFn = quantileAggregateFunction; + aggFn = quantileAggregateFunction.name; + sketchParams = quantileAggregateFunction.sketchParams; } } @@ -461,6 +485,7 @@ async function convertSelectToMaterializedViewSelect( ...selectItem, valueExpression: aggregatedColumnConfig.mvColumn, aggFn: aggMergeFn, + ...(sketchParams != null ? { sketchParams } : {}), }; } diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index 95c696b4d9..30691de7f5 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -590,6 +590,7 @@ const aggFnExpr = ({ fn, expr, level, + sketchParams, where, sampleWeightExpression, keepValueType = false, @@ -597,6 +598,8 @@ const aggFnExpr = ({ fn: AggregateFunction | AggregateFunctionWithCombinators; expr?: string; level?: number; + /** Parameters of a quantile sketch state that precede the level, for -Merge. */ + sketchParams?: number[]; where?: string; sampleWeightExpression?: string; /** Aggregate `expr` as-is instead of casting it to Float64. */ @@ -620,9 +623,17 @@ const aggFnExpr = ({ const shouldParameterizeWithLevel = level && (fn.startsWith('quantile') || fn.startsWith('histogram')); + // A sketch built with an accuracy (quantileDD, quantileGK) merges with + // that accuracy ahead of the level, e.g. quantileDDMerge(0.01, 0.95). + const mergeParams = [ + ...(fn.startsWith('quantile') ? (sketchParams ?? []) : []), + level, + ]; const renderedFnArgsWithQuantileLevel = shouldParameterizeWithLevel ? chSql`(${{ - UNSAFE_RAW_SQL: Number.isFinite(level) ? `${level}` : '0', + UNSAFE_RAW_SQL: mergeParams + .map(param => (Number.isFinite(param) ? `${param}` : '0')) + .join(', '), }})` : []; @@ -863,6 +874,7 @@ async function renderSelectList( expr: select.valueExpression, // @ts-expect-error (TS doesn't know that we've already checked for quantile) level: select.level, + sketchParams: select.sketchParams, where: whereClause.sql, sampleWeightExpression: chartConfig.sampleWeightExpression, }); diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 73e0236caf..7b681c4d36 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -481,6 +481,11 @@ export const DerivedColumnSchema = z.intersection( // `ColorConditionSchema` so no schema change was needed (the union's // string-match operators were always intended for this slice). colorRules: z.array(ColorConditionSchema).max(10).optional(), + // Parameters a materialized view's quantile sketch was built with, other + // than the level: the accuracy of quantileDD(0.01, 0.5) or + // quantileGK(100, 0.5). The materialized view optimizer sets it so the + // -Merge call matches the stored state; it is not user-configurable. + sketchParams: z.array(z.number()).optional(), }), ); export const SelectListSchema = z.array(DerivedColumnSchema).or(z.string());