diff --git a/.changeset/mv-min-date-inference.md b/.changeset/mv-min-date-inference.md new file mode 100644 index 000000000..6689e1869 --- /dev/null +++ b/.changeset/mv-min-date-inference.md @@ -0,0 +1,5 @@ +--- +'@hyperdx/app': patch +--- + +feat: when a materialized view's configuration is inferred in the source form, its minimum date now defaults to the first whole bucket after the view was created. Views only hold rows inserted after they exist, so ranges before that read the source table instead of undercounting. Clear the date if the view was backfilled. diff --git a/packages/app/src/utils/__tests__/materializedViews.test.ts b/packages/app/src/utils/__tests__/materializedViews.test.ts index d2b800330..e5435ce35 100644 --- a/packages/app/src/utils/__tests__/materializedViews.test.ts +++ b/packages/app/src/utils/__tests__/materializedViews.test.ts @@ -300,6 +300,49 @@ describe('inferMaterializedViewConfig', () => { }); }); + it.each([ + { + createdAt: '2026-11-20T04:12:18Z', + minDate: '2026-11-20T05:00:00.000Z', + }, + // Created on a bucket boundary: that bucket is already whole + { + createdAt: '2026-11-20T04:00:00Z', + minDate: '2026-11-20T04:00:00.000Z', + }, + ])( + 'should start a view created at $createdAt at its first whole bucket', + async ({ createdAt, minDate }) => { + const getTableMetadata = mockMetadata.getTableMetadata; + mockMetadata.getTableMetadata = jest.fn().mockImplementation(args => + args.tableName === 'test_mv' + ? Promise.resolve({ + ...mv.meta, + metadata_modification_time: createdAt, + }) + : getTableMetadata(args), + ); + + const actualConfig = await inferMaterializedViewConfig( + { + databaseName: 'test_db', + tableName: 'test_mv', + connectionId: 'test_connection', + }, + { + databaseName: 'test_db', + tableName: 'test_source_table', + connectionId: 'test_connection', + }, + mockMetadata, + ); + + // One-hour buckets, from the view's toStartOfHour + expect(actualConfig?.minGranularity).toBe('1 hour'); + expect(actualConfig?.minDate).toBe(minDate); + }, + ); + it('should infer materialized view configuration when given the name of a SummingMergeTree target table', async () => { const sourceTableConnection: TableConnection = { databaseName: 'test_db', diff --git a/packages/app/src/utils/materializedViews.ts b/packages/app/src/utils/materializedViews.ts index 5fb79d37d..07360575b 100644 --- a/packages/app/src/utils/materializedViews.ts +++ b/packages/app/src/utils/materializedViews.ts @@ -10,12 +10,14 @@ import { TableMetadata, } from '@hyperdx/common-utils/dist/core/metadata'; import { + getAlignedDateRange, Granularity, splitAndTrimWithBracket, } from '@hyperdx/common-utils/dist/core/utils'; import { InternalAggregateFunction, MaterializedViewConfiguration, + SQLIntervalSchema, } from '@hyperdx/common-utils/dist/types'; export const MV_AGGREGATE_FUNCTIONS = [ @@ -456,6 +458,8 @@ export async function inferMaterializedViewConfig( .map(col => col.name) .join(', '); + const minDate = inferMinDate(mvMetadata, minGranularity); + return { databaseName: mvTableMetadata.database, tableName: mvTableMetadata.name, @@ -463,5 +467,31 @@ export async function inferMaterializedViewConfig( minGranularity, timestampColumn, aggregatedColumns, + ...(minDate != null ? { minDate } : {}), }; } + +/** + * The earliest time a materialized view holds complete data for: the start of + * the first whole bucket after the view was created. A view only captures rows + * inserted after it exists, so treating it as complete for earlier ranges would + * undercount them. A backfilled view covers more; the date can be cleared in + * the source form then. + */ +function inferMinDate( + mvMetadata: TableMetadata | undefined, + minGranularity: string, +): string | undefined { + const createdAt = mvMetadata?.metadata_modification_time + ? new Date(mvMetadata.metadata_modification_time) + : undefined; + if (createdAt == null || Number.isNaN(createdAt.getTime())) { + return undefined; + } + + const granularity = SQLIntervalSchema.safeParse(minGranularity); + const firstWholeBucket = granularity.success + ? getAlignedDateRange([createdAt, createdAt], granularity.data)[1] + : createdAt; + return firstWholeBucket.toISOString(); +}