From 9f4bf96d23a0cab1d562fa514968d4ecf24e8c88 Mon Sep 17 00:00:00 2001 From: Karl Power Date: Wed, 18 Nov 2026 14:20:47 +0100 Subject: [PATCH 1/4] feat: let SQL query dashboard filters use an optional source --- .../sql-query-dashboard-filter-source.md | 9 + packages/api/openapi.json | 7 +- .../__tests__/dashboards/validation.test.ts | 16 +- .../api/src/mcp/prompts/dashboards/content.ts | 14 +- .../api/src/mcp/tools/dashboards/schemas.ts | 13 +- .../src/mcp/tools/dashboards/validation.ts | 10 +- .../api/__tests__/dashboard.int.test.ts | 124 ++++++++++ packages/api/src/routers/api/dashboards.ts | 10 +- .../__tests__/dashboards.int.test.ts | 78 ++++++- .../src/routers/external-api/v2/dashboards.ts | 13 +- .../external-api/v2/utils/dashboards.ts | 29 ++- .../src/utils/__tests__/filterSources.test.ts | 76 ++++++- packages/api/src/utils/externalApi.ts | 6 +- packages/api/src/utils/filterSources.ts | 66 +++++- packages/api/src/utils/zod.ts | 5 +- packages/app/src/DBDashboardImportPage.tsx | 55 ++++- packages/app/src/DBDashboardPage.tsx | 2 + packages/app/src/DashboardFilters.tsx | 5 + .../__tests__/DBDashboardImportPage.test.tsx | 141 +++++++++++- .../ChartEditor/RawSqlChartEditor.tsx | 63 +----- .../DashboardFilterEditForm.tsx | 6 + .../SqlQueryFilterEditForm.tsx | 212 ++++++++++++++---- .../__tests__/SqlQueryFilterEditForm.test.tsx | 156 ++++++++++++- .../__tests__/filterFormState.test.ts | 19 +- .../DashboardFiltersModal/filterFormState.ts | 8 +- .../DashboardFiltersModal/index.tsx | 5 + .../useSqlQueryFilterValues.test.tsx | 105 ++++++++- .../src/hooks/useDashboardFilterValues.tsx | 5 + .../app/src/hooks/useSqlQueryFilterValues.tsx | 100 ++++++++- .../__tests__/connectionSourceSync.test.ts} | 2 +- .../app/src/utils/connectionSourceSync.ts | 63 ++++++ .../src/__tests__/filters.test.ts | 87 ++++++- .../common-utils/src/__tests__/utils.test.ts | 59 +++++ .../src/core/renderChartConfig.ts | 47 +++- packages/common-utils/src/core/utils.ts | 5 + packages/common-utils/src/filters.ts | 108 +++++++-- packages/common-utils/src/types.ts | 5 + 37 files changed, 1527 insertions(+), 207 deletions(-) create mode 100644 .changeset/sql-query-dashboard-filter-source.md rename packages/app/src/{components/ChartEditor/__tests__/resolveConnectionSourceSync.test.ts => utils/__tests__/connectionSourceSync.test.ts} (97%) create mode 100644 packages/app/src/utils/connectionSourceSync.ts diff --git a/.changeset/sql-query-dashboard-filter-source.md b/.changeset/sql-query-dashboard-filter-source.md new file mode 100644 index 0000000000..5eef88e796 --- /dev/null +++ b/.changeset/sql-query-dashboard-filter-source.md @@ -0,0 +1,9 @@ +--- +'@hyperdx/common-utils': patch +'@hyperdx/app': patch +'@hyperdx/api': patch +--- + +feat: let SQL query dashboard filters use an optional source + +A SQL query filter can now pick a source on its connection. With one, its SQL can use `$__sourceTable` and `$__filters`, which applies the dashboard's filter selections for that source, the same way raw SQL tiles do. The external API and MCP read and write it as `sourceId`. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index d3d0e8dc9b..9a0f6ae4b4 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -4551,10 +4551,15 @@ "description": "ClickHouse connection the query runs on: a connection ID, or\n`$name` of a CONNECTION filter on the dashboard.\n", "example": "65f5e4a3b9e77c001a567890" }, + "sourceId": { + "type": "string", + "description": "Optional Log, Trace, Session, or Metric source on the same\nconnection. Lets `sqlTemplate` use `$__sourceTable` and\n`$__filters`. Requires a connection ID in `connectionId`, not\na connection variable.\n", + "example": "65f5e4a3b9e77c001a345678" + }, "sqlTemplate": { "type": "string", "minLength": 1, - "description": "Query whose first column populates the dropdown. May use the time\nmacros and other filters' variables, but not `$__sourceTable` or\n`$__filters`, since the filter has no source.\n", + "description": "Query whose first column populates the dropdown. May use the time\nmacros and other filters' variables. `$__sourceTable` and\n`$__filters` need a `sourceId`; `$__filters` expands to the\ndashboard's broadcast filter selections for that source.\n", "example": "SELECT DISTINCT env FROM default.envs ORDER BY env" }, "isBroadcastEnabled": { diff --git a/packages/api/src/mcp/__tests__/dashboards/validation.test.ts b/packages/api/src/mcp/__tests__/dashboards/validation.test.ts index 0e9034c096..31ab9695fd 100644 --- a/packages/api/src/mcp/__tests__/dashboards/validation.test.ts +++ b/packages/api/src/mcp/__tests__/dashboards/validation.test.ts @@ -937,9 +937,10 @@ describe('getFilterVariableWarnings', () => { }); describe('SQL query filters', () => { - const sqlFilter = (sqlTemplate: string) => ({ + const sqlFilter = (sqlTemplate: string, sourceId?: string) => ({ name: 'Environment', sqlTemplate, + sourceId, isVariableEnabled: true, variableName: 'env', }); @@ -969,10 +970,21 @@ describe('getFilterVariableWarnings', () => { sqlFilter('SELECT env FROM $__sourceTable'), ]); expect(warnings).toEqual([ - 'Filter "Environment": SQL query filters have no source, so $__sourceTable cannot be used', + 'Filter "Environment": SQL uses $__sourceTable but no source is selected', ]); }); + it('accepts source macros when the filter has a source', () => { + expect( + getFilterVariableWarnings([ + sqlFilter( + 'SELECT env FROM $__sourceTable WHERE $__filters', + '65f5e4a3b9e77c001a345678', + ), + ]), + ).toEqual([]); + }); + it.each([ ['a source macro', 'SELECT env FROM $__sourceTable(gauge'], ['a time macro', 'SELECT env FROM envs WHERE $__timeFilter(ts'], diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index 125b3c16e5..eabc46825e 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -200,7 +200,7 @@ Dashboards open with a 15-minute default window. There is no dashboard-level fie - Using $__filter or $__conditionalAll in a Lucene where (they are matched as literal text; switch to whereLanguage: "sql", or write ServiceName:$var). - Chaining a filter's where off a filter that is not variable-enabled. Only isVariableEnabled: true publishes the $variableName token; without it the reference resolves to nothing and the downstream dropdown is unscoped or empty. - Referencing a filter's OWN variable in its where. That narrows its dropdown to the values already picked, so the rest of the options vanish from a multi-select. Reference a sibling filter's variable instead. -- Mixing filter-type fields: expression/sourceId/where belong to QUERY_EXPRESSION filters, options to STATIC_LIST filters, connectionId/sqlTemplate to SQL_QUERY filters, sourceType/sources to ADHOC filters. +- Mixing filter-type fields: expression/sourceId/where belong to QUERY_EXPRESSION filters, options to STATIC_LIST filters, connectionId/sqlTemplate (and an optional sourceId) to SQL_QUERY filters, sourceType/sources to ADHOC filters. - Referencing an ADHOC filter's variableName in a tile. Ad hoc conditions reach tiles only by broadcast; the name is not a variable. - Expecting a STATIC_LIST filter to broadcast. It cannot; a tile must reference its $variableName (via $__filter with the column passed explicitly) or nothing changes when the user picks a value.`; } @@ -1213,10 +1213,11 @@ Example (static list): ] SQL query filter shape: - { type: "SQL_QUERY", name, connectionId, sqlTemplate, variableName? } + { type: "SQL_QUERY", name, connectionId, sourceId?, sqlTemplate, variableName? } - connectionId ClickHouse connection the query runs on (from clickstack_list_sources). - sqlTemplate SELECT whose FIRST column becomes the dropdown values, cast to string with nulls and duplicates dropped, in row order (add ORDER BY). May use the time macros and other filters' variables; cannot use $__sourceTable or $__filters, because the filter has no source. + connectionId ClickHouse connection the query runs on (from clickstack_list_sources), or a CONNECTION filter's "$variableName". + sourceId Optional log, trace, session, or metric source on that connection. Requires a connection ID, not a connection variable. Needed for $__sourceTable and $__filters; $__filters expands to the dashboard's broadcast filter selections for that source. + sqlTemplate SELECT whose FIRST column becomes the dropdown values, cast to string with nulls and duplicates dropped, in row order (add ORDER BY). May use the time macros and other filters' variables, plus $__sourceTable and $__filters when sourceId is set. Use it when the values live in a lookup table, dictionary, or join that no single source column covers; otherwise prefer QUERY_EXPRESSION. Like a static filter it is variable-only, so tiles pass the column explicitly: $__filter(, $). @@ -1225,6 +1226,11 @@ Example (SQL query, narrowed by another filter's variable): { type: "SQL_QUERY", name: "Environment", connectionId: "", sqlTemplate: "SELECT DISTINCT env FROM default.envs WHERE $__filter(cloud, $cloud) ORDER BY env", variableName: "env" } ] +Example (SQL query on a source, narrowed by the dashboard's broadcast filters): + filters: [ + { type: "SQL_QUERY", name: "Host", connectionId: "", sourceId: "", sqlTemplate: "SELECT DISTINCT ResourceAttributes['host.name'] FROM $__sourceTable WHERE $__timeFilter(TimestampTime) AND $__filters ORDER BY 1", variableName: "host" } + ] + Ad hoc filter shape: { type: "ADHOC", name, sourceType, sources, where?, whereLanguage?, appliesToSourceIds?, variableName? } diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index 2dbf181762..86bc522de5 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -1798,13 +1798,22 @@ const mcpSqlQueryFilterSchema = z 'ClickHouse connection the query runs on: a connection ID from ' + 'clickstack_list_sources, or `$name` of a CONNECTION filter on this dashboard.', ), + sourceId: objectIdSchema + .optional() + .describe( + 'Optional log, trace, session, or metric source on the same connection. ' + + 'Lets sqlTemplate use $__sourceTable and $__filters. Requires a ' + + 'connection ID in connectionId, not a connection variable. Get IDs ' + + 'from clickstack_list_sources.', + ), sqlTemplate: SqlQueryDashboardFilterSchema.shape.sqlTemplate.describe( 'ClickHouse SELECT whose FIRST column becomes the dropdown values (cast to ' + 'string; nulls and duplicates dropped; row order kept, so add ORDER BY). ' + 'May use the time macros ($__timeFilter(), $__fromTime, $__toTime) ' + "and other filters' variables, ideally as $__filter(, $). " + - 'Cannot use $__sourceTable or $__filters: the filter has no source, so name ' + - 'tables explicitly. Example: "SELECT DISTINCT env FROM default.envs ORDER BY env".', + '$__sourceTable and $__filters need a sourceId; $__filters expands to the ' + + "dashboard's broadcast filter selections for that source. Without a sourceId, " + + 'name tables explicitly. Example: "SELECT DISTINCT env FROM default.envs ORDER BY env".', ), }) .describe( diff --git a/packages/api/src/mcp/tools/dashboards/validation.ts b/packages/api/src/mcp/tools/dashboards/validation.ts index d323c86acc..65eb0b204e 100644 --- a/packages/api/src/mcp/tools/dashboards/validation.ts +++ b/packages/api/src/mcp/tools/dashboards/validation.ts @@ -166,7 +166,9 @@ export function getRawSqlTileMacroWarnings( */ export type FilterForVariableValidation = FilterForVariableDeclaration & Partial> & - Partial>; + Partial> & { + sourceId?: string; + }; /** * Non-blocking checks on the dashboard variables references in a filter's `where`. @@ -225,7 +227,11 @@ export function getFilterVariableWarnings( if (sqlTemplate) { // Also reports variable macro errors, which `result` already holds. - const detail = validateSqlQueryFilterTemplate(sqlTemplate, variables); + // Source tables aren't loaded here, so a source only allows its macros. + const detail = validateSqlQueryFilterTemplate(sqlTemplate, { + variables, + sourceTables: filter.sourceId ? 'unknown' : undefined, + }); if (detail && !issues.includes(detail)) issues.push(detail); } diff --git a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts index d6065229ca..c53edc4aa1 100644 --- a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts +++ b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts @@ -2181,6 +2181,130 @@ describe('dashboard router', () => { /does not name a connection variable/, ); }); + + const createSqlFilterSource = (kind: SourceKind, teamId = team._id) => + Source.create({ + kind, + name: `Test ${kind} source`, + team: teamId, + connection: connectionId, + from: { databaseName: 'test_db', tableName: `${kind}_table` }, + timestampValueExpression: 'timestamp', + defaultTableSelectExpression: 'body', + }); + + it('persists a SQL query filter with a source', async () => { + const source = await createSqlFilterSource(SourceKind.Log); + const filter = makeSqlFilter({ + source: source._id.toString(), + sqlTemplate: + 'SELECT DISTINCT ServiceName FROM $__sourceTable WHERE $__filters', + }); + + const created = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [filter] }) + .expect(200); + + expect(created.body.filters).toEqual([filter]); + }); + + it('rejects a source that does not exist', async () => { + const source = new Types.ObjectId().toString(); + + const response = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [makeSqlFilter({ source })] }) + .expect(400); + + expect(response.body.message).toContain(source); + }); + + it("rejects another team's source", async () => { + const source = await createSqlFilterSource( + SourceKind.Log, + new Types.ObjectId(), + ); + + await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + filters: [makeSqlFilter({ source: source._id.toString() })], + }) + .expect(400); + }); + + it('rejects a source on another connection', async () => { + const source = await createSqlFilterSource(SourceKind.Log); + const otherConnection = await Connection.create({ + team: team._id, + name: 'Other Connection', + host: 'http://localhost:8123', + username: 'default', + }); + + const response = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + filters: [ + makeSqlFilter({ + source: source._id.toString(), + connection: otherConnection._id.toString(), + }), + ], + }) + .expect(400); + + expect(response.body.message).toContain("not on the filter's connection"); + }); + + it('rejects a PromQL source', async () => { + const source = await createSqlFilterSource(SourceKind.Promql); + + const response = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + filters: [makeSqlFilter({ source: source._id.toString() })], + }) + .expect(400); + + expect(response.body.message).toContain(source._id.toString()); + }); + + it('rejects a source with a connection variable', async () => { + const source = await createSqlFilterSource(SourceKind.Log); + + const response = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + filters: [ + connectionFilter, + makeSqlFilter({ + connection: '$conn', + source: source._id.toString(), + }), + ], + }) + .expect(400); + + expect(response.body.message).toContain('not variables'); + }); + + it('rejects a source variable', async () => { + const response = await agent + .post('/dashboards') + .send({ + ...MOCK_DASHBOARD, + filters: [makeSqlFilter({ source: '$logs' })], + }) + .expect(400); + + expect(response.body.message).toContain('not variables'); + }); }); describe('ad hoc filters', () => { diff --git a/packages/api/src/routers/api/dashboards.ts b/packages/api/src/routers/api/dashboards.ts index ac54d3e74a..27bf2f7d52 100644 --- a/packages/api/src/routers/api/dashboards.ts +++ b/packages/api/src/routers/api/dashboards.ts @@ -40,6 +40,7 @@ import type { ObjectId } from '@/models'; import { getChangedFilterSourceRefs, getFilterSourceRefError, + getSqlQueryFilterSourceVariableError, } from '@/utils/filterSources'; import { objectIdSchema } from '@/utils/zod'; @@ -67,9 +68,10 @@ const addFilterIssues = ( }; /** - * Rejects PROMETHEUS_LABEL and ADHOC filters that reference a source that is - * missing or of the wrong kind. Resolving these filters needs a live source of - * the right kind, unlike other filter types. References the stored filter with + * Rejects PROMETHEUS_LABEL, ADHOC and SQL_QUERY filters that reference a source + * that is missing or of the wrong kind, and SQL_QUERY filters whose source is + * on another connection or that pair a source with a variable. Resolving these filters needs a live source of the + * right kind, unlike other filter types. References the stored filter with * the same ID already made are skipped, so a since-deleted source doesn't block * unrelated saves. Returns an error message, or null when there is nothing to * reject. @@ -79,6 +81,8 @@ async function validateFilterSources( filters: DashboardFilter[] = [], existingFilters: DashboardFilter[] = [], ): Promise { + const variableError = getSqlQueryFilterSourceVariableError(filters); + if (variableError != null) return variableError; const refs = getChangedFilterSourceRefs(filters, existingFilters).filter( // Variable references are checked by validateDashboardVariableReferences ref => parseVariableReference(ref.id) == null, diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts index 88faec017f..374f5c3247 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts @@ -6362,10 +6362,86 @@ describe('External API v2 Dashboards - new format', () => { ['broadcast enabled', { isBroadcastEnabled: true }], ['not variable-enabled', { isVariableEnabled: false }], ['no SQL', { sqlTemplate: '' }], - ['a source', { sourceId: new ObjectId().toString() }], + [ + 'a source that does not exist', + { sourceId: new ObjectId().toString() }, + ], ])('rejects a SQL query filter with %s', async (_label, overrides) => { await expectFilters([sqlFilterInput(overrides)], 400); }); + + it('accepts a SQL query filter with a source and round-trips it through GET', async () => { + const sqlTemplate = + 'SELECT DISTINCT ServiceName FROM $__sourceTable WHERE $__filters'; + const response = await sendFilters([ + sqlFilterInput({ sourceId: traceSource._id.toString(), sqlTemplate }), + ]); + expect(response.status).toBe(200); + + const [filter] = response.body.data.filters; + expect(filter).toMatchObject({ + type: 'SQL_QUERY', + connectionId: connection._id.toString(), + sourceId: traceSource._id.toString(), + sqlTemplate, + }); + expect(filter).not.toHaveProperty('source'); + + const getResponse = await authRequest( + 'get', + `${BASE_URL}/${response.body.data.id}`, + ).expect(200); + expect(getResponse.body.data.filters).toEqual( + response.body.data.filters, + ); + }); + + it('rejects a PromQL source', async () => { + const response = await sendFilters([ + sqlFilterInput({ sourceId: promqlSource._id.toString() }), + ]); + + expect(response.status).toBe(400); + expect(response.body.message).toContain(promqlSource._id.toString()); + }); + + it('rejects a source on a different connection', async () => { + const otherConnection = await Connection.create({ + team: team._id, + name: 'Other', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + + const response = await sendFilters([ + sqlFilterInput({ + connectionId: otherConnection._id.toString(), + sourceId: traceSource._id.toString(), + }), + ]); + + expect(response.status).toBe(400); + expect(response.body.message).toContain(traceSource._id.toString()); + }); + + it('rejects a source with a connection variable', async () => { + const response = await sendFilters([ + { + id: new ObjectId().toString(), + type: 'CONNECTION', + name: 'Connection', + variableName: 'conn', + }, + sqlFilterInput({ + connectionId: '$conn', + sourceId: traceSource._id.toString(), + }), + ]); + + expect(response.status).toBe(400); + expect(response.body.message).toContain('not variables'); + }); }); describe('static-list filters', () => { diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index 511775a035..3cf80c645e 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -2909,13 +2909,22 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * ClickHouse connection the query runs on: a connection ID, or * `$name` of a CONNECTION filter on the dashboard. * example: "65f5e4a3b9e77c001a567890" + * sourceId: + * type: string + * description: | + * Optional Log, Trace, Session, or Metric source on the same + * connection. Lets `sqlTemplate` use `$__sourceTable` and + * `$__filters`. Requires a connection ID in `connectionId`, not + * a connection variable. + * example: "65f5e4a3b9e77c001a345678" * sqlTemplate: * type: string * minLength: 1 * description: | * Query whose first column populates the dropdown. May use the time - * macros and other filters' variables, but not `$__sourceTable` or - * `$__filters`, since the filter has no source. + * macros and other filters' variables. `$__sourceTable` and + * `$__filters` need a `sourceId`; `$__filters` expands to the + * dashboard's broadcast filter selections for that source. * example: "SELECT DISTINCT env FROM default.envs ORDER BY env" * isBroadcastEnabled: * type: boolean diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index fbde606d52..54f1301cab 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -71,6 +71,7 @@ import { getChangedFilterSourceRefs, getFilterSourceRefError, getInvalidSourceIds, + getSqlQueryFilterSourceVariableError, } from '@/utils/filterSources'; import logger from '@/utils/logger'; import { @@ -1325,8 +1326,8 @@ function getMissingSources( } for (const filter of filters ?? []) { - // PROMETHEUS_LABEL and ADHOC sources are checked with the stored filter's - // references skipped; see `validateDashboardTiles`. + // PROMETHEUS_LABEL, ADHOC and SQL_QUERY sources are checked with the stored + // filter's references skipped; see `validateDashboardTiles`. if (isQueryExpressionFilter(filter)) sourceIds.add(filter.sourceId); } @@ -1907,17 +1908,21 @@ export async function validateDashboardTiles( return `PromQL tiles require a PromQL source. The following source IDs are not PromQL sources: ${nonPromqlTileSourceIds.join(', ')}`; } - // PROMETHEUS_LABEL and ADHOC filters need live sources of the right kind. As - // in the internal API, references the stored filter already made are not - // re-checked, so a since-deleted source doesn't block unrelated saves. + // PROMETHEUS_LABEL, ADHOC and SQL_QUERY filters need live sources of the + // right kind. As in the internal API, references the stored filter already + // made are not re-checked, so a since-deleted source doesn't block unrelated + // saves. + // Create bodies carry no filter IDs; an empty one matches no stored filter. + const internalFilters = (filters ?? []).map(filter => + translateExternalFilterToFilter({ id: '', ...filter }), + ); + const sqlQueryFilterVariableError = + getSqlQueryFilterSourceVariableError(internalFilters); + if (sqlQueryFilterVariableError != null) return sqlQueryFilterVariableError; const filterSourceError = getFilterSourceRefError( - getChangedFilterSourceRefs( - // Create bodies carry no filter IDs; an empty one matches no stored filter. - (filters ?? []).map(filter => - translateExternalFilterToFilter({ id: '', ...filter }), - ), - existingFilters, - ).filter(ref => parseVariableReference(ref.id) == null), + getChangedFilterSourceRefs(internalFilters, existingFilters).filter( + ref => parseVariableReference(ref.id) == null, + ), sources, ); if (filterSourceError != null) return filterSourceError; diff --git a/packages/api/src/utils/__tests__/filterSources.test.ts b/packages/api/src/utils/__tests__/filterSources.test.ts index 10d8d2a4b4..739bf68575 100644 --- a/packages/api/src/utils/__tests__/filterSources.test.ts +++ b/packages/api/src/utils/__tests__/filterSources.test.ts @@ -3,6 +3,7 @@ import { DashboardFilter, SourceKind } from '@hyperdx/common-utils/dist/types'; import { getChangedFilterSourceRefs, getFilterSourceRefError, + getSqlQueryFilterSourceVariableError, } from '@/utils/filterSources'; const adhoc = ( @@ -28,7 +29,27 @@ const label: DashboardFilter = { isVariableEnabled: true, }; +const sqlQuery = ( + overrides: Partial> = {}, +): DashboardFilter => ({ + id: 's1', + type: 'SQL_QUERY', + name: 'Hosts', + connection: 'conn1', + source: 'logs', + sqlTemplate: 'SELECT 1', + isBroadcastEnabled: false, + isVariableEnabled: true, + ...overrides, +}); + const sql = [SourceKind.Log, SourceKind.Trace, SourceKind.Session]; +const sqlQueryKinds = [ + SourceKind.Log, + SourceKind.Trace, + SourceKind.Session, + SourceKind.Metric, +]; describe('getChangedFilterSourceRefs', () => { it('returns every reference without stored filters', () => { @@ -59,6 +80,21 @@ describe('getChangedFilterSourceRefs', () => { ).toEqual([{ id: 'logs', kinds: [SourceKind.Promql] }]); }); + it("returns a SQL query filter's source with its connection", () => { + expect(getChangedFilterSourceRefs([sqlQuery()])).toEqual([ + { id: 'logs', kinds: sqlQueryKinds, connection: 'conn1' }, + ]); + }); + + it('re-checks a SQL query source whose connection changed', () => { + expect( + getChangedFilterSourceRefs( + [sqlQuery({ connection: 'conn2' })], + [sqlQuery()], + ), + ).toEqual([{ id: 'logs', kinds: sqlQueryKinds, connection: 'conn2' }]); + }); + it('checks a filter without an ID, or with a new one', () => { const { id: _id, ...withoutId } = adhoc(); expect( @@ -75,8 +111,8 @@ describe('getChangedFilterSourceRefs', () => { describe('getFilterSourceRefError', () => { const sources = [ - { _id: 'logs', kind: SourceKind.Log }, - { _id: 'prom', kind: SourceKind.Promql }, + { _id: 'logs', kind: SourceKind.Log, connection: 'conn1' }, + { _id: 'prom', kind: SourceKind.Promql, connection: 'conn1' }, ]; it('accepts references to existing sources of the right kind', () => { @@ -105,4 +141,40 @@ describe('getFilterSourceRefError', () => { 'The following filter source IDs do not exist or are not a kind of source the filter supports: prom, gone', ); }); + + it('names sources not on the connection the filter requires', () => { + expect( + getFilterSourceRefError( + [ + { id: 'logs', kinds: sql, connection: 'conn1' }, + { id: 'logs', kinds: sql, connection: 'conn2' }, + ], + sources, + ), + ).toBe( + "The following filter source IDs are not on the filter's connection: logs", + ); + }); +}); + +describe('getSqlQueryFilterSourceVariableError', () => { + it('accepts a source ID on a connection ID, or no source', () => { + expect( + getSqlQueryFilterSourceVariableError([ + sqlQuery(), + sqlQuery({ connection: '$conn', source: undefined }), + ]), + ).toBeNull(); + }); + + it('names filters with a source variable or a connection variable', () => { + expect( + getSqlQueryFilterSourceVariableError([ + sqlQuery({ name: 'A', source: '$logs' }), + sqlQuery({ name: 'B', connection: '$conn' }), + ]), + ).toBe( + 'SQL_QUERY filters with a source need a source ID and a connection ID, not variables. The following filters use a variable: A, B', + ); + }); }); diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index 164e59d7f3..c445cb5746 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -265,8 +265,9 @@ export function translateFilterToExternalFilter( case 'SQL_QUERY': return { - ...omit(filter, 'connection', ...omittedKeys), + ...omit(filter, 'connection', 'source', ...omittedKeys), connectionId: filter.connection.toString(), + ...(filter.source ? { sourceId: filter.source.toString() } : {}), }; case 'DATA_SOURCE': @@ -312,8 +313,9 @@ export function translateExternalFilterToFilter( case 'SQL_QUERY': return { - ...omit(filter, 'connectionId'), + ...omit(filter, 'connectionId', 'sourceId'), connection: filter.connectionId, + ...(filter.sourceId ? { source: filter.sourceId } : {}), }; case 'QUERY_EXPRESSION': diff --git a/packages/api/src/utils/filterSources.ts b/packages/api/src/utils/filterSources.ts index 2cc2df23fa..ce613ff096 100644 --- a/packages/api/src/utils/filterSources.ts +++ b/packages/api/src/utils/filterSources.ts @@ -2,7 +2,10 @@ import { getAdhocFilterSourceKinds, isAdhocFilter, isPrometheusLabelFilter, + isSqlQueryFilter, + SQL_QUERY_FILTER_SOURCE_KINDS, } from '@hyperdx/common-utils/dist/filters'; +import { parseVariableReference } from '@hyperdx/common-utils/dist/referenceVariables'; import { DashboardFilter, SourceKind } from '@hyperdx/common-utils/dist/types'; /** A filter as stored or as sent on create, when it has no ID yet. */ @@ -11,7 +14,12 @@ type FilterWithOptionalId = T extends unknown : never; /** A source a reference points at, and the kinds it must be. */ -export type SourceRef = { id: string; kinds: SourceKind[] }; +export type SourceRef = { + id: string; + kinds: SourceKind[]; + /** The connection the source must be on, when the filter has its own. */ + connection?: string; +}; /** The sources a filter reads from, each with the kinds it must be. */ function getFilterSourceRefs(filter: FilterWithOptionalId): SourceRef[] { @@ -24,6 +32,15 @@ function getFilterSourceRefs(filter: FilterWithOptionalId): SourceRef[] { id => ({ id, kinds }), ); } + if (isSqlQueryFilter(filter) && filter.source) { + return [ + { + id: filter.source, + kinds: SQL_QUERY_FILTER_SOURCE_KINDS, + connection: filter.connection, + }, + ]; + } return []; } @@ -35,7 +52,8 @@ export function getChangedFilterSourceRefs( filters: FilterWithOptionalId[] = [], existingFilters: DashboardFilter[] = [], ): SourceRef[] { - const refKey = ({ id, kinds }: SourceRef) => `${id}:${kinds.join(',')}`; + const refKey = ({ id, kinds, connection }: SourceRef) => + `${id}:${kinds.join(',')}:${connection ?? ''}`; const existingRefsById = new Map( existingFilters.map(filter => [ filter.id, @@ -67,14 +85,50 @@ export function getInvalidSourceIds( } /** - * An error naming the referenced sources that are missing or of the wrong - * kind, or null when there are none. + * An error naming the referenced sources that are missing, of the wrong kind, + * or not on the connection the filter requires, or null when there are none. */ export function getFilterSourceRefError( refs: SourceRef[], - sources: Parameters[1], + sources: (Parameters[1][number] & { + connection: { toString(): string }; + })[], ): string | null { const invalid = getInvalidSourceIds(refs, sources); + if (invalid.length > 0) { + return `The following filter source IDs do not exist or are not a kind of source the filter supports: ${invalid.join(', ')}`; + } + + const connectionById = new Map( + sources.map(s => [s._id.toString(), s.connection.toString()]), + ); + const mismatched = refs + .filter( + ({ id, connection }) => + connection !== undefined && connectionById.get(id) !== connection, + ) + .map(({ id }) => id); + if (mismatched.length === 0) return null; + return `The following filter source IDs are not on the filter's connection: ${[...new Set(mismatched)].join(', ')}`; +} + +/** + * An error naming SQL_QUERY filters whose source or connection is a variable + * while it has a source, or null. A source is only offered on a fixed + * connection, so neither field may change what the other points at. + */ +export function getSqlQueryFilterSourceVariableError( + filters: FilterWithOptionalId[] = [], +): string | null { + const invalid = filters + .filter(isSqlQueryFilter) + .filter( + ({ source, connection }) => + source != null && + (parseVariableReference(source) != null || + parseVariableReference(connection) != null), + ) + .map(({ name }) => name); if (invalid.length === 0) return null; - return `The following filter source IDs do not exist or are not a kind of source the filter supports: ${invalid.join(', ')}`; + return `SQL_QUERY filters with a source need a source ID and a connection ID, not variables. The following filters use a variable: ${invalid.join(', ')}`; } diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index 87df22e508..5bdf6526b0 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -207,11 +207,14 @@ const externalPromqlLabelFilterShape = PromqlLabelDashboardFilterSchema.omit({ isVariableEnabled: z.literal(true).default(true), }); -// `connectionId` rather than `connection`, matching raw SQL tiles. +// `connectionId`/`sourceId` rather than `connection`/`source`, matching raw SQL +// tiles. const externalSqlQueryFilterShape = SqlQueryDashboardFilterSchema.omit({ connection: true, + source: true, }).extend({ connectionId: objectIdOrVariableReferenceSchema, + sourceId: objectIdSchema.optional(), isBroadcastEnabled: z.literal(false).default(false), isVariableEnabled: z.literal(true).default(true), }); diff --git a/packages/app/src/DBDashboardImportPage.tsx b/packages/app/src/DBDashboardImportPage.tsx index 5f6542a469..de05a15914 100644 --- a/packages/app/src/DBDashboardImportPage.tsx +++ b/packages/app/src/DBDashboardImportPage.tsx @@ -31,6 +31,7 @@ import { QUERY_EXPRESSION_FILTER_SOURCE_KINDS, type SavedFilterValueIssue, type SavedQueryIssue, + SQL_QUERY_FILTER_SOURCE_KINDS, validateDashboardFilterQueries, validateSavedFilterValues, validateSavedQuery, @@ -52,6 +53,7 @@ import { type QueryExpressionDashboardFilter, SavedChartConfig, SourceKind, + type SqlQueryDashboardFilter, TSource, } from '@hyperdx/common-utils/dist/types'; import { @@ -465,8 +467,17 @@ const mappableSourceNames = (names: string[] | undefined) => /** Whether the filter names a single source that import must map. */ const isSourceBackedTemplateFilter = ( filter: DashboardFilter, -): filter is QueryExpressionDashboardFilter | PromqlLabelDashboardFilter => - (isQueryExpressionFilter(filter) || isPrometheusLabelFilter(filter)) && +): filter is + | QueryExpressionDashboardFilter + | PromqlLabelDashboardFilter + | SqlQueryDashboardFilter => + (isQueryExpressionFilter(filter) || + isPrometheusLabelFilter(filter) || + // The source is optional for a SQL query filter, and only offered on a + // connection ID + (isSqlQueryFilter(filter) && + !!filter.source && + !isVariableRef(filter.connection))) && !isVariableRef(filter.source); /** @@ -513,7 +524,9 @@ const filterSourceKinds = ( ? [SourceKind.Promql] : isQueryExpressionFilter(filter) ? QUERY_EXPRESSION_FILTER_SOURCE_KINDS - : undefined; + : isSqlQueryFilter(filter) + ? SQL_QUERY_FILTER_SOURCE_KINDS + : undefined; /** * One mapping select's value. Mantine's Select writes `null` when the user @@ -531,7 +544,7 @@ const MappingValueSchema = z */ export function buildMappingFormSchema( input: DashboardTemplate, - sources?: readonly Pick[], + sources?: readonly Pick[], ) { return z .object({ @@ -676,6 +689,22 @@ export function buildMappingFormSchema( ? 'Select a PromQL source for this filter' : 'Select a non-PromQL source for this filter', }); + return; + } + + // The filter's query runs on its connection, so its source must be there too. + const mappedConnectionId = data.filterConnectionMappings?.[idx]; + if ( + isSqlQueryFilter(filter) && + mappedSource && + mappedConnectionId && + mappedSource.connection !== mappedConnectionId + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['filterSourceMappings', idx], + message: "Select a source on this filter's connection", + }); } }); }); @@ -1231,11 +1260,16 @@ export function Mapping({ input }: { input: DashboardTemplate }) { }; } if (connectionBackedTemplateFilter(filter)) { + // A SQL query filter's optional source + const source = isSourceBackedTemplateFilter(filter) + ? findSource(data.filterSourceMappings?.[idx]) + : undefined; return { ...filter, // The form schema requires this mapping connection: findConnection(data.filterConnectionMappings?.[idx])! .id, + ...(source ? { source: source.id } : {}), }; } const source = findSource(data.filterSourceMappings?.[idx]); @@ -1409,6 +1443,10 @@ export function Mapping({ input }: { input: DashboardTemplate }) { {/** Map filter sources */} {input.filters?.map((filter, i) => { + // A SQL query filter's source must be on its mapped connection. + const sourceConnectionId = isSqlQueryFilter(filter) + ? filterConnectionMappings?.[i] + : undefined; const appliesToSourceIds = mappableSourceNames( appliesToTemplateFilter(filter)?.appliesToSourceIds, ); @@ -1446,8 +1484,13 @@ export function Mapping({ input }: { input: DashboardTemplate }) { control={control} name={`filterSourceMappings.${i}`} data={sources - ?.filter(source => - filterSourceKinds(filter)?.includes(source.kind), + ?.filter( + source => + filterSourceKinds(filter)?.includes( + source.kind, + ) && + (!sourceConnectionId || + source.connection === sourceConnectionId), ) .map(source => ({ value: source.id, diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index 44fefde825..6c712a8b1b 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -3425,6 +3425,7 @@ function DBDashboardPage({ incompatibleReferenceVariablesByFilterId={ incompatibleReferenceVariablesByFilterId } + getDashboardFilters={getFilterQueriesForSource} /> )} {/* Selection indicator */} @@ -3604,6 +3605,7 @@ function DBDashboardPage({ showRequiredFilterOptions variables={variables} dateRange={searchedTimeRange} + getDashboardFilters={getFilterQueriesForSource} /> )} diff --git a/packages/app/src/DashboardFilters.tsx b/packages/app/src/DashboardFilters.tsx index 0bace5b5f8..8205f0b403 100644 --- a/packages/app/src/DashboardFilters.tsx +++ b/packages/app/src/DashboardFilters.tsx @@ -17,6 +17,7 @@ import { import { ReferenceVariableValues } from '@hyperdx/common-utils/dist/referenceVariables'; import { AdhocFilterCondition, + BroadcastFilter, ChartVariable, DashboardFilter, } from '@hyperdx/common-utils/dist/types'; @@ -193,6 +194,8 @@ interface DashboardFilterProps { unresolvedReferenceVariablesByFilterId?: ReadonlyMap; /** The subset of those selecting a source the filter can't use. */ incompatibleReferenceVariablesByFilterId?: ReadonlyMap; + /** The filter selections that broadcast to a source, for `$__filters` */ + getDashboardFilters?: (sourceId: string) => BroadcastFilter[]; } const DashboardFilters = ({ @@ -207,6 +210,7 @@ const DashboardFilters = ({ referenceVariableValues, unresolvedReferenceVariablesByFilterId, incompatibleReferenceVariablesByFilterId, + getDashboardFilters, }: DashboardFilterProps) => { // "Link" mode (opt-in, off by default): each dropdown's values are narrowed by // the others' selections. Off by default because contingent value lookups @@ -232,6 +236,7 @@ const DashboardFilters = ({ filters: queryableFilters, dateRange, variables, + getDashboardFilters, // Only narrow by sibling selections when linked. selectionByFilterId: linked ? selectionByFilterId : EMPTY_SELECTIONS, }); diff --git a/packages/app/src/__tests__/DBDashboardImportPage.test.tsx b/packages/app/src/__tests__/DBDashboardImportPage.test.tsx index 81502709f0..34e77da879 100644 --- a/packages/app/src/__tests__/DBDashboardImportPage.test.tsx +++ b/packages/app/src/__tests__/DBDashboardImportPage.test.tsx @@ -15,14 +15,44 @@ import { // Sources/connections the import maps the template's source *names* onto. The // auto-mapping effect in `Mapping` matches by name (case-insensitive), so a // template tile with `source: 'Metrics'` resolves to `src-metrics`, etc. The -// mock objects only need the fields the import path touches (id/name/kind). +// mock objects only need the fields the import path touches +// (id/name/kind/connection). const mockSources = [ - { id: 'src-metrics', name: 'Metrics', kind: SourceKind.Metric }, - { id: 'src-traces', name: 'Traces', kind: SourceKind.Trace }, - { id: 'src-logs', name: 'Logs', kind: SourceKind.Log }, - { id: 'src-promql', name: 'Prom', kind: SourceKind.Promql }, + { + id: 'src-metrics', + name: 'Metrics', + kind: SourceKind.Metric, + connection: 'conn-default', + }, + { + id: 'src-traces', + name: 'Traces', + kind: SourceKind.Trace, + connection: 'conn-default', + }, + { + id: 'src-logs', + name: 'Logs', + kind: SourceKind.Log, + connection: 'conn-default', + }, + { + id: 'src-promql', + name: 'Prom', + kind: SourceKind.Promql, + connection: 'conn-default', + }, + { + id: 'src-other-logs', + name: 'Other logs', + kind: SourceKind.Log, + connection: 'conn-other', + }, +]; +const mockConnections = [ + { id: 'conn-default', name: 'Default' }, + { id: 'conn-other', name: 'Other' }, ]; -const mockConnections = [{ id: 'conn-default', name: 'Default' }]; const mockMutateAsync = jest.fn().mockResolvedValue({ id: 'dash-new' }); jest.mock('next/router', () => ({ @@ -269,7 +299,7 @@ const unresolvedConnectionTemplate = DashboardTemplateSchema.parse({ ], }); -const sqlQueryFilterTemplate = (connection: string) => +const sqlQueryFilterTemplate = (connection: string, source?: string) => DashboardTemplateSchema.parse({ version: '0.1.0', name: 'SQL Query Filter', @@ -280,6 +310,7 @@ const sqlQueryFilterTemplate = (connection: string) => type: 'SQL_QUERY', name: 'Environment', connection, + source, sqlTemplate: 'SELECT DISTINCT env FROM envs', isBroadcastEnabled: false, isVariableEnabled: true, @@ -429,6 +460,102 @@ describe('Dashboard import - SQL query filter connection', () => { type: 'SQL_QUERY', connection: 'conn-default', }); + expect(payload.filters[0].source).toBeUndefined(); + }); + + it('auto-maps an optional source by name and imports it as an id', async () => { + renderWithMantine( + , + ); + + const finish = await screen.findByRole('button', { + name: /finish import/i, + }); + expect(screen.getByDisplayValue('Logs')).toBeInTheDocument(); + await act(async () => { + fireEvent.click(finish); + }); + + await waitFor(() => expect(mockMutateAsync).toHaveBeenCalledTimes(1)); + const payload = mockMutateAsync.mock.calls[0][0]; + expect(DashboardWithoutIdSchema.safeParse(payload).success).toBe(true); + expect(payload.filters[0]).toMatchObject({ + type: 'SQL_QUERY', + connection: 'conn-default', + source: 'src-logs', + }); + }); + + it('maps neither the connection variable nor a source paired with it', () => { + const result = buildMappingFormSchema( + sqlQueryFilterTemplate('$conn', 'Logs'), + mockSources, + ).safeParse({ + dashboardName: 'SQL Query Filter', + tags: [], + tileSourceMappings: ['src-metrics'], + connectionMappings: [''], + filterSourceMappings: [''], + filterConnectionMappings: [''], + }); + + expect(result.success).toBe(true); + }); + + it('rejects a PromQL source for a filter with a source', () => { + const result = buildMappingFormSchema( + sqlQueryFilterTemplate('default', 'Logs'), + mockSources, + ).safeParse({ + dashboardName: 'SQL Query Filter', + tags: [], + tileSourceMappings: ['src-metrics'], + connectionMappings: [''], + filterSourceMappings: ['src-promql'], + filterConnectionMappings: ['conn-default'], + }); + + expect( + result.error?.issues.map(issue => [issue.path.join('.'), issue.message]), + ).toEqual([ + ['filterSourceMappings.0', 'Select a non-PromQL source for this filter'], + ]); + }); + + it("rejects a source on another connection than the filter's", () => { + const result = buildMappingFormSchema( + sqlQueryFilterTemplate('default', 'Logs'), + mockSources, + ).safeParse({ + dashboardName: 'SQL Query Filter', + tags: [], + tileSourceMappings: ['src-metrics'], + connectionMappings: [''], + filterSourceMappings: ['src-other-logs'], + filterConnectionMappings: ['conn-default'], + }); + + expect( + result.error?.issues.map(issue => [issue.path.join('.'), issue.message]), + ).toEqual([ + ['filterSourceMappings.0', "Select a source on this filter's connection"], + ]); + }); + + it("only lists sources on the filter's mapped connection", async () => { + renderWithMantine( + , + ); + + await screen.findByRole('button', { name: /finish import/i }); + fireEvent.click(screen.getByDisplayValue('Logs')); + + expect( + await screen.findByRole('option', { name: 'Logs' }), + ).toBeInTheDocument(); + expect( + screen.queryByRole('option', { name: 'Other logs' }), + ).not.toBeInTheDocument(); }); it('blocks the import and flags the filter row when its connection is unmapped', async () => { diff --git a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx index 895c1685be..030e229250 100644 --- a/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/RawSqlChartEditor.tsx @@ -50,6 +50,7 @@ import useResizable from '@/hooks/useResizable'; import { useSources } from '@/source'; import { getAllMetricTables, usePrevious } from '@/utils'; import { DEFAULT_TILE_ALERT } from '@/utils/alerts'; +import { resolveConnectionSourceSync } from '@/utils/connectionSourceSync'; import { SQL_PLACEHOLDERS } from './constants'; import { RawSqlChartInstructions } from './RawSqlChartInstructions'; @@ -58,68 +59,6 @@ import { buildRawSqlCompletions } from './utils'; import resizeStyles from '@/../styles/ResizablePanel.module.scss'; -type ConnectionSourceSyncResult = - | { field: 'connection'; value: string } - | { field: 'source'; value: '' } - | null; - -/** - * Decides how the `connection` and `source` form fields should be kept in sync - * in raw SQL mode. Extracted as a pure function so the behavior can be unit - * tested independently of the component's effect. - */ -export function resolveConnectionSourceSync({ - source, - connection, - prevSource, - prevConnection, - sources, -}: { - source: string | undefined; - connection: string | undefined; - prevSource: string | undefined; - prevConnection: string | undefined; - sources: { id: string; connection: string }[] | undefined; -}): ConnectionSourceSyncResult { - if (!sources) return null; - // A data source variable reference isn't bound to a connection here: the - // filter bar decides what it selects, so it neither moves the connection - // nor is cleared by a connection change. - const isSourceReference = parseVariableReference(source) != null; - - // When the source changes, sync the connection to match. - if (source !== prevSource) { - if (isSourceReference) return null; - const sourceConnection = sources.find(s => s.id === source)?.connection; - if (sourceConnection && sourceConnection !== connection) { - return { field: 'connection', value: sourceConnection }; - } - return null; - } - - // Set a default connection when none is selected. - if (!connection) { - const defaultConnection = sources[0]?.connection; - if (defaultConnection) { - return { field: 'connection', value: defaultConnection }; - } - return null; - } - - // When the connection changes, clear the source only if the currently - // selected source doesn't belong to the new connection. This avoids clearing - // a source that was just carried over (e.g. from builder mode), where the - // connection change above was itself triggered by the source. - if (connection !== prevConnection && prevConnection !== undefined) { - const sourceConnection = sources.find(s => s.id === source)?.connection; - if (source && !isSourceReference && sourceConnection !== connection) { - return { field: 'source', value: '' }; - } - } - - return null; -} - export default function RawSqlChartEditor({ control, setValue, diff --git a/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx b/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx index a6f0424efa..848a3c326d 100644 --- a/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx +++ b/packages/app/src/components/DashboardFiltersModal/DashboardFilterEditForm.tsx @@ -6,6 +6,7 @@ import { isReferenceVariableFilterType, } from '@hyperdx/common-utils/dist/filters'; import { + BroadcastFilter, ChartVariable, DashboardFilter, DashboardFilterType, @@ -88,6 +89,8 @@ interface DashboardFilterEditFormProps { variables?: ChartVariable[]; /** The dashboard's date range, which filter previews run over */ dateRange?: [Date, Date]; + /** The dashboard filter selections that broadcast to the given source */ + getDashboardFilters?: (sourceId: string) => BroadcastFilter[]; onSave: (filter: DashboardFilter) => void; onClose: () => void; onCancel: () => void; @@ -107,6 +110,7 @@ export const DashboardFilterEditForm = ({ showRequiredFilterOptions, variables, dateRange, + getDashboardFilters, onSave, onClose, onCancel, @@ -294,8 +298,10 @@ export const DashboardFilterEditForm = ({ ) : formFilterType === 'PROMETHEUS_LABEL' ? ( diff --git a/packages/app/src/components/DashboardFiltersModal/SqlQueryFilterEditForm.tsx b/packages/app/src/components/DashboardFiltersModal/SqlQueryFilterEditForm.tsx index bfdc907ffd..a09a0281fb 100644 --- a/packages/app/src/components/DashboardFiltersModal/SqlQueryFilterEditForm.tsx +++ b/packages/app/src/components/DashboardFiltersModal/SqlQueryFilterEditForm.tsx @@ -1,6 +1,11 @@ -import { useMemo, useState } from 'react'; -import { useFormState, useWatch } from 'react-hook-form'; -import { validateSqlQueryFilterTemplate } from '@hyperdx/common-utils/dist/filters'; +import { useEffect, useMemo, useState } from 'react'; +import { UseFormSetValue, useFormState, useWatch } from 'react-hook-form'; +import { + getSqlQueryFilterSourceTables, + SQL_QUERY_FILTER_SOURCE_KINDS, + SqlQueryFilterSourceTables, + validateSqlQueryFilterTemplate, +} from '@hyperdx/common-utils/dist/filters'; import { INTERVAL_MACROS, MACRO_SUGGESTIONS, @@ -8,6 +13,7 @@ import { } from '@hyperdx/common-utils/dist/macros'; import { parseVariableReference } from '@hyperdx/common-utils/dist/referenceVariables'; import { + BroadcastFilter, DashboardFilter, SqlQueryDashboardFilter, } from '@hyperdx/common-utils/dist/types'; @@ -17,6 +23,7 @@ import { useQueryClient } from '@tanstack/react-query'; import { ConnectionSelectControlled } from '@/components/ConnectionSelect'; import EmptyState from '@/components/EmptyState'; +import { SourceSelectControlled } from '@/components/SourceSelect'; import { useResolvedReference } from '@/components/sourceSelectUtils'; import { SQLEditorControlled } from '@/components/SQLEditor/SQLEditor'; import { @@ -29,11 +36,13 @@ import { SQL_QUERY_FILTER_VALUES_QUERY_KEY, useSqlQueryFilterValues, } from '@/hooks/useSqlQueryFilterValues'; -import { truncateText } from '@/utils'; +import { useSources } from '@/source'; +import { truncateText, usePrevious } from '@/utils'; +import { resolveConnectionSourceSync } from '@/utils/connectionSourceSync'; import { TOOLTIP_PORTAL_TARGET } from './constants'; import { CustomInputWrapper } from './CustomInputWrapper'; -import { FilterFormControl } from './filterFormState'; +import { FilterFormControl, FilterFormValues } from './filterFormState'; import { VariableNameInput } from './VariableNameInput'; /** Values the preview lists before summarizing the rest as a count. */ @@ -41,10 +50,10 @@ const PREVIEW_VALUES_SHOWN = 20; /** Characters of each value the preview shows before truncating it. */ const PREVIEW_VALUE_MAX_LENGTH = 20; -/** - * Macros a SQL query filter can't expand: it has no source, and no time buckets. - */ -const UNSUPPORTED_MACROS = new Set([ +/** Macros a SQL query filter can't expand: it has no time buckets. */ +const UNSUPPORTED_MACROS = new Set(INTERVAL_MACROS); +/** Without a source, the source macros can't expand either. */ +const UNSUPPORTED_MACROS_WITHOUT_SOURCE = new Set([ ...SOURCE_DEPENDENT_MACROS, ...INTERVAL_MACROS, ]); @@ -56,38 +65,94 @@ ORDER BY env`; interface SqlQueryFilterEditFormProps { control: FilterFormControl; + setValue: UseFormSetValue; /** Filters other than the one being edited, used to keep variable names unique. */ otherFilters: DashboardFilter[]; /** The range the preview runs over. Defaults to the last hour. */ dateRange?: [Date, Date]; + /** The dashboard filter selections that broadcast to the given source */ + getDashboardFilters?: (sourceId: string) => BroadcastFilter[]; } /** Form describing a dashboard filter whose values come from a SQL query */ export const SqlQueryFilterEditForm = ({ control, + setValue, otherFilters, dateRange, + getDashboardFilters, }: SqlQueryFilterEditFormProps) => { const { errors } = useFormState({ control }); const variables = useChartVariables(); - const completions = useMemo( - () => [ + const { data: sources } = useSources(); + + const [connection, source, sqlTemplate] = useWatch({ + control, + name: ['connection', 'source', 'sqlTemplate'], + }); + const sourceObject = source ? sources?.find(s => s.id === source) : undefined; + // A source is only offered on a connection ID, since a connection variable + // can later select a connection the source isn't on. + const isConnectionVariable = parseVariableReference(connection) != null; + + const prevSource = usePrevious(source); + const prevConnection = usePrevious(connection); + useEffect(() => { + if (isConnectionVariable) { + if (source) setValue('source', ''); + return; + } + // The source field is shared with the other filter types, so switching + // type can carry over one this filter can't use. + if ( + source && + sources && + (!sourceObject || + !SQL_QUERY_FILTER_SOURCE_KINDS.includes(sourceObject.kind)) + ) { + setValue('source', ''); + return; + } + const update = resolveConnectionSourceSync({ + source, + connection, + prevSource, + prevConnection, + sources, + }); + if (update?.field === 'connection') { + setValue('connection', update.value); + } else if (update?.field === 'source') { + setValue('source', update.value); + } + }, [ + connection, + isConnectionVariable, + prevConnection, + prevSource, + setValue, + source, + sourceObject, + sources, + ]); + + const completions = useMemo(() => { + const unsupported = source + ? UNSUPPORTED_MACROS + : UNSUPPORTED_MACROS_WITHOUT_SOURCE; + return [ ...MACRO_SUGGESTIONS.filter( // `sourceTable(gauge)` etc. are per-metric variants of `sourceTable`. - macro => !UNSUPPORTED_MACROS.has(macro.name.split('(')[0]), + macro => !unsupported.has(macro.name.split('(')[0]), ).map(toMacroCompletion), ...buildSqlVariableCompletions(variables), - ], - [variables], - ); + ]; + }, [variables, source]); - const [connection, sqlTemplate] = useWatch({ - control, - name: ['connection', 'sqlTemplate'], - }); // What the preview last ran, so editing doesn't re-query on every keystroke. const [previewed, setPreviewed] = useState<{ connection: string; + source?: string; sqlTemplate: string; }>(); // A connection variable previews as the connection it currently selects. @@ -98,56 +163,100 @@ export const SqlQueryFilterEditForm = ({ const queryClient = useQueryClient(); const runPreview = () => { if (!canPreview) return; + const previewSource = source || undefined; // The same query keeps its cache key, so re-run it explicitly. if ( previewed?.connection === resolvedConnection && + previewed.source === previewSource && previewed.sqlTemplate === sqlTemplate ) { void queryClient.invalidateQueries({ queryKey: [SQL_QUERY_FILTER_VALUES_QUERY_KEY, resolvedConnection], }); } - setPreviewed({ connection: resolvedConnection, sqlTemplate }); + setPreviewed({ + connection: resolvedConnection, + source: previewSource, + sqlTemplate, + }); + }; + + let sourceTables: SqlQueryFilterSourceTables | 'unknown' | undefined; + if (sourceObject) { + sourceTables = getSqlQueryFilterSourceTables(sourceObject); + } else if (source) { + // Set, but the sources list hasn't loaded yet. + sourceTables = 'unknown'; + } + + const validateSqlTemplate = (value: unknown) => { + if (typeof value !== 'string' || !value.trim()) { + return 'This field is required'; + } + // `variables` leaves out this filter's own, so a reference to it is + // rejected as unknown. + const error = validateSqlQueryFilterTemplate(value, { + variables, + sourceTables, + }); + return error ?? true; }; return ( <> - -
- -
-
+ + +
+ +
+
+ + + {/* The source select's wrapper is inline-flex; a grid cell + stretches it to the full width. */} +
+ +
+
+
{ - if (typeof value !== 'string' || !value.trim()) { - return 'This field is required'; - } - // `variables` leaves out this filter's own, so a reference to - // it is rejected as unknown. - return ( - validateSqlQueryFilterTemplate(value, variables ?? []) ?? true - ); - }, - }} + rules={{ validate: validateSqlTemplate }} height="150px" enableLineWrapping placeholder={PLACEHOLDER_SQL} @@ -177,6 +286,7 @@ export const SqlQueryFilterEditForm = ({ {...previewed} dateRange={dateRange} variables={variables} + getDashboardFilters={getDashboardFilters} /> )} @@ -189,14 +299,18 @@ export const SqlQueryFilterEditForm = ({ const SqlQueryFilterPreview = ({ connection, + source, sqlTemplate, dateRange, variables, + getDashboardFilters, }: { connection: string; + source?: string; sqlTemplate: string; dateRange?: [Date, Date]; variables: ReturnType; + getDashboardFilters?: (sourceId: string) => BroadcastFilter[]; }) => { const lastHour = useMemo( (): [Date, Date] => [new Date(NOW - 60 * 60 * 1000), new Date(NOW)], @@ -209,18 +323,20 @@ const SqlQueryFilterPreview = ({ type: 'SQL_QUERY', name: 'Preview', connection, + source, sqlTemplate, isBroadcastEnabled: false, isVariableEnabled: true, }, ], - [connection, sqlTemplate], + [connection, source, sqlTemplate], ); const { data, filterErrorMessages, isLoading, isFetching, isError } = useSqlQueryFilterValues({ filters, dateRange: dateRange ?? lastHour, variables, + getDashboardFilters, }); const values = data.get('preview')?.values ?? []; diff --git a/packages/app/src/components/DashboardFiltersModal/__tests__/SqlQueryFilterEditForm.test.tsx b/packages/app/src/components/DashboardFiltersModal/__tests__/SqlQueryFilterEditForm.test.tsx index b5aea12e9d..1fd9f54cb5 100644 --- a/packages/app/src/components/DashboardFiltersModal/__tests__/SqlQueryFilterEditForm.test.tsx +++ b/packages/app/src/components/DashboardFiltersModal/__tests__/SqlQueryFilterEditForm.test.tsx @@ -1,6 +1,7 @@ import { RegisterOptions, useForm } from 'react-hook-form'; +import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; -import { screen } from '@testing-library/react'; +import { fireEvent, screen, waitFor } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { @@ -14,6 +15,39 @@ import { SqlVariablesProvider } from '@/components/SQLEditor/variableCompletions const mockUseSqlQueryFilterValues = jest.fn(); const mockCompletionLabels: string[] = []; +let mockSources: TSource[] = []; + +const logSource = (id: string, connection: string): TSource => ({ + id, + name: id, + kind: SourceKind.Log, + connection, + from: { databaseName: 'default', tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: '', +}); + +jest.mock('@/source', () => ({ + ...jest.requireActual('@/source'), + useSources: () => ({ data: mockSources }), +})); +jest.mock('@/components/SourceSelect', () => ({ + SourceSelectControlled: ({ + control, + name, + disabled, + }: { + control: FilterFormControl; + name: 'source'; + disabled?: boolean; + }) => ( + + ), +})); jest.mock('@/hooks/useSqlQueryFilterValues', () => ({ SQL_QUERY_FILTER_VALUES_QUERY_KEY: 'sql-values', @@ -76,18 +110,24 @@ const NO_REFERENCE_VALUES = new Map(); let queryClient: QueryClient; function Harness({ + defaultValues, referenceValues = NO_REFERENCE_VALUES, }: { + defaultValues?: Partial; referenceValues?: Map; }) { - const { control, handleSubmit } = useForm({ - defaultValues: { ...toFormValues(), type: 'SQL_QUERY' }, + const { control, handleSubmit, setValue } = useForm({ + defaultValues: { ...toFormValues(), type: 'SQL_QUERY', ...defaultValues }, }); return ( - + @@ -115,6 +155,7 @@ const renderForm = async ( describe('SqlQueryFilterEditForm', () => { beforeEach(() => { queryClient = new QueryClient(); + mockSources = []; mockUseSqlQueryFilterValues.mockReset(); mockUseSqlQueryFilterValues.mockReturnValue( valuesResult({ values: ['prod', 'dev'] }), @@ -154,10 +195,115 @@ describe('SqlQueryFilterEditForm', () => { await user.click(screen.getByRole('button', { name: 'Save' })); expect( - await screen.findByText(/unknown variable|cannot be used/), + await screen.findByText( + /unknown variable|cannot be used|no source is selected/, + ), ).toBeInTheDocument(); }); + describe('with a source', () => { + const renderWithSource = async (sql: string) => { + mockSources = [logSource('logs', 'conn1'), logSource('other', 'conn2')]; + renderWithMantine( + , + ); + const user = userEvent.setup(); + await user.type(screen.getByTestId('sql-input'), sql); + return user; + }; + + it('suggests and accepts the source macros', async () => { + const user = await renderWithSource( + 'SELECT ServiceName FROM $__sourceTable WHERE $__filters', + ); + + expect(mockCompletionLabels).toContain('$__sourceTable'); + await user.click(screen.getByRole('button', { name: 'Save' })); + + expect( + screen.queryByText(/no source is selected|cannot be used/), + ).not.toBeInTheDocument(); + }); + + it('previews with the source', async () => { + const user = await renderWithSource( + 'SELECT ServiceName FROM $__sourceTable', + ); + + await user.click(screen.getByTestId('filter-sql-preview-button')); + + expect(mockUseSqlQueryFilterValues).toHaveBeenLastCalledWith( + expect.objectContaining({ + filters: [expect.objectContaining({ source: 'logs' })], + }), + ); + }); + + it('clears the source when the connection changes to one it is not on', async () => { + await renderWithSource('SELECT 1'); + + fireEvent.change(screen.getByTestId('connection-input'), { + target: { value: 'conn2' }, + }); + + await waitFor(() => + expect(screen.getByTestId('source-input')).toHaveValue(''), + ); + }); + + it('clears and disables the source when the connection becomes a variable', async () => { + await renderWithSource('SELECT 1'); + + fireEvent.change(screen.getByTestId('connection-input'), { + target: { value: '$conn' }, + }); + + await waitFor(() => + expect(screen.getByTestId('source-input')).toHaveValue(''), + ); + expect(screen.getByTestId('source-input')).toBeDisabled(); + expect(screen.getByTestId('connection-input')).toHaveValue('$conn'); + }); + + it('runs a new preview, not a re-run, when only the source changed', async () => { + const invalidate = jest.spyOn(queryClient, 'invalidateQueries'); + const user = await renderWithSource('SELECT 1'); + + await user.click(screen.getByTestId('filter-sql-preview-button')); + fireEvent.change(screen.getByTestId('source-input'), { + target: { value: '' }, + }); + await user.click(screen.getByTestId('filter-sql-preview-button')); + + expect(invalidate).not.toHaveBeenCalled(); + expect(mockUseSqlQueryFilterValues).toHaveBeenLastCalledWith( + expect.objectContaining({ + filters: [expect.objectContaining({ source: undefined })], + }), + ); + }); + + it('clears a source of a kind the filter cannot use', async () => { + mockSources = [ + { + id: 'prom', + name: 'prom', + kind: SourceKind.Promql, + connection: 'conn1', + from: { databaseName: 'default', tableName: 'ts' }, + timestampValueExpression: 'Timestamp', + }, + ]; + renderWithMantine( + , + ); + + await waitFor(() => + expect(screen.getByTestId('source-input')).toHaveValue(''), + ); + }); + }); + it('shows the loader, not the last values, while a re-run fetches', async () => { mockUseSqlQueryFilterValues.mockReturnValue( valuesResult({ values: ['prod', 'dev'], isFetching: true }), diff --git a/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts b/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts index 9d8ee49e4f..4eb8618969 100644 --- a/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts +++ b/packages/app/src/components/DashboardFiltersModal/__tests__/filterFormState.test.ts @@ -393,9 +393,9 @@ describe('toSavedFilter', () => { name: 'Region', connection: 'conn1', sqlTemplate: ' SELECT region FROM regions ', + source: 'logs', // Left behind by the other editors before the type was switched. expression: 'Env', - source: 'logs', where: 'x = 1', options: ['prod'], }), @@ -406,6 +406,7 @@ describe('toSavedFilter', () => { type: 'SQL_QUERY', name: 'Region', connection: 'conn1', + source: 'logs', sqlTemplate: 'SELECT region FROM regions', isBroadcastEnabled: false, isVariableEnabled: true, @@ -413,10 +414,26 @@ describe('toSavedFilter', () => { }); expect(toFormValues(saved)).toMatchObject({ connection: 'conn1', + source: 'logs', sqlTemplate: 'SELECT region FROM regions', }); }); + it('saves a SQL query filter without a source when none is picked', () => { + const saved = toSavedFilter( + formValues({ + type: 'SQL_QUERY', + name: 'Region', + connection: 'conn1', + source: '', + sqlTemplate: 'SELECT region FROM regions', + }), + ); + + // The schema requires a non-empty source when one is set. + expect(saved).toMatchObject({ source: undefined }); + }); + it('rejects a SQL query filter without a connection', () => { expect(() => toSavedFilter( diff --git a/packages/app/src/components/DashboardFiltersModal/filterFormState.ts b/packages/app/src/components/DashboardFiltersModal/filterFormState.ts index a5b49c1208..804df515ca 100644 --- a/packages/app/src/components/DashboardFiltersModal/filterFormState.ts +++ b/packages/app/src/components/DashboardFiltersModal/filterFormState.ts @@ -86,7 +86,12 @@ export const toFormValues = ( // QUERY_EXPRESSION fields expression: queried?.expression ?? '', - source: queried?.source ?? promqlLabel?.source ?? presetSourceId ?? '', + source: + queried?.source ?? + promqlLabel?.source ?? + sqlQuery?.source ?? + presetSourceId ?? + '', sourceMetricType: queried?.sourceMetricType, where: queried?.where ?? adhoc?.where ?? '', whereLanguage: @@ -159,6 +164,7 @@ export const toSavedFilter = (values: FilterFormValues): DashboardFilter => { if (values.type === 'SQL_QUERY') { return DashboardFilterSchema.parse({ ...rest, + source: values.source || undefined, sqlTemplate: values.sqlTemplate.trim(), isBroadcastEnabled: false, isVariableEnabled: true, diff --git a/packages/app/src/components/DashboardFiltersModal/index.tsx b/packages/app/src/components/DashboardFiltersModal/index.tsx index 15ec455f0e..96c27f9f9c 100644 --- a/packages/app/src/components/DashboardFiltersModal/index.tsx +++ b/packages/app/src/components/DashboardFiltersModal/index.tsx @@ -1,5 +1,6 @@ import { useCallback, useEffect, useState } from 'react'; import { + BroadcastFilter, ChartVariable, DashboardFilter, TSource, @@ -22,6 +23,8 @@ interface DashboardFiltersEditModalProps { variables?: ChartVariable[]; /** The dashboard's date range, which filter previews run over */ dateRange?: [Date, Date]; + /** The filter selections that broadcast to a source, for filter previews */ + getDashboardFilters?: (sourceId: string) => BroadcastFilter[]; onClose: () => void; onSaveFilter: (filter: DashboardFilter) => void; onRemoveFilter: (id: string) => void; @@ -36,6 +39,7 @@ const DashboardFiltersModal = ({ showRequiredFilterOptions, variables, dateRange, + getDashboardFilters, onClose, onSaveFilter, onRemoveFilter, @@ -78,6 +82,7 @@ const DashboardFiltersModal = ({ showRequiredFilterOptions={showRequiredFilterOptions} variables={variables} dateRange={dateRange} + getDashboardFilters={getDashboardFilters} onSave={handleSaveFilter} onCancel={stopEditing} onClose={onClose} diff --git a/packages/app/src/hooks/__tests__/useSqlQueryFilterValues.test.tsx b/packages/app/src/hooks/__tests__/useSqlQueryFilterValues.test.tsx index 19a15d190d..ddaa8be9d8 100644 --- a/packages/app/src/hooks/__tests__/useSqlQueryFilterValues.test.tsx +++ b/packages/app/src/hooks/__tests__/useSqlQueryFilterValues.test.tsx @@ -1,8 +1,11 @@ import React from 'react'; import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; import { + BroadcastFilter, ChartVariable, + SourceKind, SqlQueryDashboardFilter, + TSource, } from '@hyperdx/common-utils/dist/types'; import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import { renderHook, waitFor } from '@testing-library/react'; @@ -13,10 +16,35 @@ import { } from '@/hooks/useSqlQueryFilterValues'; const mockQuery = jest.fn(); +const mockRenderSourceFiltersToSql = jest.fn(); +let mockSources: { data?: TSource[]; isLoading: boolean } = { + data: [], + isLoading: false, +}; jest.mock('@/clickhouse', () => ({ getClickhouseClient: () => ({ query: mockQuery }), })); +jest.mock('@/source', () => ({ + useSources: () => mockSources, +})); +jest.mock('@/hooks/useMetadata', () => ({ + useMetadataWithSettings: () => ({}), +})); +jest.mock('@hyperdx/common-utils/dist/core/renderChartConfig', () => ({ + renderSourceFiltersToSql: (...args: unknown[]) => + mockRenderSourceFiltersToSql(...args), +})); + +const logs: TSource = { + id: 'logs', + name: 'Logs', + kind: SourceKind.Log, + connection: 'conn1', + from: { databaseName: 'default', tableName: 'otel_logs' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: '', +}; const respondWith = (rows: { value: string | null }[]) => mockQuery.mockResolvedValue({ json: async () => ({ data: rows }) }); @@ -49,6 +77,7 @@ describe('useSqlQueryFilterValues', () => { const renderFilters = ( filters: SqlQueryDashboardFilter[], variables?: ChartVariable[], + getDashboardFilters?: (sourceId: string) => BroadcastFilter[], ) => renderHook( () => @@ -56,11 +85,14 @@ describe('useSqlQueryFilterValues', () => { filters, dateRange: DATE_RANGE, variables, + getDashboardFilters, }), { wrapper }, ); beforeEach(() => { + mockSources = { data: [logs], isLoading: false }; + mockRenderSourceFiltersToSql.mockReset(); mockQuery.mockReset(); respondWith([{ value: 'prod' }, { value: 'dev' }]); const queryClient = new QueryClient({ @@ -134,7 +166,7 @@ describe('useSqlQueryFilterValues', () => { expect(result.current.erroredFilterIds).toEqual(new Set(['sql1'])); expect(result.current.filterErrorMessages.get('sql1')).toMatch( - /have no source/, + /no source is selected/, ); expect(result.current.data.get('sql1')).toEqual({ values: [], @@ -143,6 +175,77 @@ describe('useSqlQueryFilterValues', () => { expect(mockQuery).not.toHaveBeenCalled(); }); + describe('with a source', () => { + it("expands $__sourceTable to the source's table", async () => { + const { result } = renderFilters( + [ + filter({ + source: 'logs', + sqlTemplate: 'SELECT 1 FROM $__sourceTable', + }), + ], + [], + ); + + await waitFor(() => expect(result.current.isFetching).toBe(false)); + + expect(mockQuery.mock.calls[0][0].query).toMatch(/default.*otel_logs/); + expect(mockRenderSourceFiltersToSql).not.toHaveBeenCalled(); + }); + + it("expands $__filters to the dashboard's selections for the source", async () => { + mockRenderSourceFiltersToSql.mockResolvedValue("(ServiceName = 'api')"); + const selections: BroadcastFilter[] = [ + { type: 'sql', condition: "ServiceName IN ('api')" }, + { type: 'promql', condition: 'job="api"' }, + ]; + const getDashboardFilters = jest.fn(() => selections); + + const { result } = renderFilters( + [ + filter({ + source: 'logs', + sqlTemplate: 'SELECT 1 FROM $__sourceTable WHERE $__filters', + }), + ], + [], + getDashboardFilters, + ); + + await waitFor(() => expect(result.current.isFetching).toBe(false)); + + expect(getDashboardFilters).toHaveBeenCalledWith('logs'); + expect(mockRenderSourceFiltersToSql).toHaveBeenCalledWith( + expect.objectContaining({ source: logs, filters: [selections[0]] }), + ); + expect(mockQuery.mock.calls[0][0].query).toContain( + "(ServiceName = 'api')", + ); + }); + + it('waits for the sources to load', () => { + mockSources = { data: undefined, isLoading: true }; + + const { result } = renderFilters([filter({ source: 'logs' })], []); + + expect(result.current.isLoading).toBe(true); + expect(result.current.data.get('sql1')).toEqual({ + values: [], + isLoading: true, + }); + expect(mockQuery).not.toHaveBeenCalled(); + }); + + it('reports a source that no longer exists', () => { + const { result } = renderFilters([filter({ source: 'deleted' })], []); + + expect(result.current.filterErrorMessages.get('sql1')).toBe( + 'Source deleted not found', + ); + expect(mockQuery).not.toHaveBeenCalled(); + }); + }); + it('reports a failed query with its error message', async () => { mockQuery.mockRejectedValue(new Error('Unknown table default.envs')); diff --git a/packages/app/src/hooks/useDashboardFilterValues.tsx b/packages/app/src/hooks/useDashboardFilterValues.tsx index d55f19e80f..db383efa78 100644 --- a/packages/app/src/hooks/useDashboardFilterValues.tsx +++ b/packages/app/src/hooks/useDashboardFilterValues.tsx @@ -7,6 +7,7 @@ import { isStaticListFilter, } from '@hyperdx/common-utils/dist/filters'; import { + BroadcastFilter, ChartVariable, DashboardFilter, } from '@hyperdx/common-utils/dist/types'; @@ -34,6 +35,7 @@ export function useDashboardFilterValues({ dateRange, selectionByFilterId, variables, + getDashboardFilters, }: { filters: DashboardFilter[]; dateRange: [Date, Date]; @@ -41,6 +43,8 @@ export function useDashboardFilterValues({ selectionByFilterId?: ReadonlyMap; /** The dashboard's variables and their current selections, if any */ variables?: ChartVariable[]; + /** The filter selections that broadcast to a source, for `$__filters` */ + getDashboardFilters?: (sourceId: string) => BroadcastFilter[]; }): DashboardFilterValuesResult { const [queriedFilters, staticFilters, promqlLabelFilters, sqlQueryFilters] = useMemo( @@ -74,6 +78,7 @@ export function useDashboardFilterValues({ filters: sqlQueryFilters, dateRange, variables, + getDashboardFilters, }); const data = useMemo( diff --git a/packages/app/src/hooks/useSqlQueryFilterValues.tsx b/packages/app/src/hooks/useSqlQueryFilterValues.tsx index da6f25d115..de51ac452b 100644 --- a/packages/app/src/hooks/useSqlQueryFilterValues.tsx +++ b/packages/app/src/hooks/useSqlQueryFilterValues.tsx @@ -3,10 +3,19 @@ import { ClickHouseQueryError, ResponseJSON, } from '@hyperdx/common-utils/dist/clickhouse'; -import { renderSqlQueryFilterValuesQuery } from '@hyperdx/common-utils/dist/filters'; +import { renderSourceFiltersToSql } from '@hyperdx/common-utils/dist/core/renderChartConfig'; import { + getSqlFilters, + getSqlQueryFilterSourceTables, + renderSqlQueryFilterValuesQuery, +} from '@hyperdx/common-utils/dist/filters'; +import { hasMacro } from '@hyperdx/common-utils/dist/macros'; +import { + BroadcastFilter, ChartVariable, + Filter, SqlQueryDashboardFilter, + TSource, } from '@hyperdx/common-utils/dist/types'; import { keepPreviousData, @@ -16,6 +25,10 @@ import { import { getClickhouseClient } from '@/clickhouse'; import { useMetadataQueryAttribution } from '@/queryAttribution'; +import { useSources } from '@/source'; +import { mapKeyBy } from '@/utils'; + +import { useMetadataWithSettings } from './useMetadata'; /** First element of every values query key, followed by the connection. */ export const SQL_QUERY_FILTER_VALUES_QUERY_KEY = @@ -27,8 +40,12 @@ const MAX_EXECUTION_SECONDS = 30; type ValuesCall = { filterId: string; connection: string; + sqlTemplate: string; sql: string; params: Record; + source?: TSource; + /** The broadcast conditions `$__filters` expands to, when the SQL uses it. */ + broadcastFilters: Filter[]; }; /** Rows from the values query, without nulls or repeats, in query order. */ @@ -43,34 +60,67 @@ export function useSqlQueryFilterValues({ filters, dateRange, variables, + getDashboardFilters, }: { filters: SqlQueryDashboardFilter[]; dateRange: [Date, Date]; /** The dashboard's variables and their current selections, if any */ variables?: ChartVariable[]; + /** The dashboard filter selections that broadcast to the given source */ + getDashboardFilters?: (sourceId: string) => BroadcastFilter[]; }) { + const { data: sources, isLoading: isLoadingSources } = useSources(); + const sourcesById = useMemo(() => mapKeyBy(sources ?? [], 'id'), [sources]); + const metadata = useMetadataWithSettings(); + // Expanded here so react-query keys on the SQL that actually runs, which - // changes with other filters' selections. + // changes with other filters' selections. `$__filters` is expanded in the + // query function instead, since that needs the source's metadata. const { calls, templateErrors } = useMemo(() => { const resolved: ValuesCall[] = []; const errors = new Map(); for (const filter of filters) { + const source = filter.source ? sourcesById.get(filter.source) : undefined; + if (filter.source && !source) { + // Every source looks missing until the list arrives. + if (!isLoadingSources) { + errors.set(filter.id, `Source ${filter.source} not found`); + } + continue; + } try { + const rendered = renderSqlQueryFilterValuesQuery({ + sqlTemplate: filter.sqlTemplate, + variables, + dateRange, + sourceTables: source + ? getSqlQueryFilterSourceTables(source) + : undefined, + }); resolved.push({ filterId: filter.id, connection: filter.connection, - ...renderSqlQueryFilterValuesQuery({ - sqlTemplate: filter.sqlTemplate, - variables, - dateRange, - }), + sqlTemplate: filter.sqlTemplate, + ...rendered, + source, + broadcastFilters: + source && hasMacro(filter.sqlTemplate, 'filters') + ? getSqlFilters(getDashboardFilters?.(source.id) ?? []) + : [], }); } catch (e) { errors.set(filter.id, e instanceof Error ? e.message : String(e)); } } return { calls: resolved, templateErrors: errors }; - }, [filters, variables, dateRange]); + }, [ + filters, + variables, + dateRange, + sourcesById, + isLoadingSources, + getDashboardFilters, + ]); const attribution = useMetadataQueryAttribution(); @@ -81,6 +131,8 @@ export function useSqlQueryFilterValues({ call.connection, call.sql, call.params, + call.source?.id, + call.broadcastFilters, ], // Keep the last values while a new range or selection loads, rather than // blanking the dropdown. @@ -90,9 +142,24 @@ export function useSqlQueryFilterValues({ !(error instanceof ClickHouseQueryError) && failureCount < 1, refetchOnWindowFocus: false, queryFn: async ({ signal }) => { + let { sql } = call; + if (call.source && call.broadcastFilters.length > 0) { + const filtersSQL = await renderSourceFiltersToSql({ + source: call.source, + filters: call.broadcastFilters, + metadata, + }); + sql = renderSqlQueryFilterValuesQuery({ + sqlTemplate: call.sqlTemplate, + variables, + dateRange, + sourceTables: getSqlQueryFilterSourceTables(call.source), + filtersSQL, + }).sql; + } const result = (await getClickhouseClient({ attribution }) .query<'JSON'>({ - query: call.sql, + query: sql, query_params: call.params, format: 'JSON', abort_signal: signal, @@ -135,13 +202,22 @@ export function useSqlQueryFilterValues({ filterErrorMessages.set(filterId, message); } + // A filter waiting on its source has no call yet, but is still loading. + const isWaitingForSources = + isLoadingSources && filters.some(filter => filter.source); + for (const filter of filters) { + if (isWaitingForSources && !data.has(filter.id)) { + data.set(filter.id, { values: [], isLoading: true }); + } + } + return { data, erroredFilterIds, filterErrorMessages, - isLoading: results.some(r => r.isLoading), - isFetching: results.some(r => r.isFetching), + isLoading: isWaitingForSources || results.some(r => r.isLoading), + isFetching: isWaitingForSources || results.some(r => r.isFetching), isError: erroredFilterIds.size > 0, }; - }, [results, calls, templateErrors]); + }, [results, calls, templateErrors, filters, isLoadingSources]); } diff --git a/packages/app/src/components/ChartEditor/__tests__/resolveConnectionSourceSync.test.ts b/packages/app/src/utils/__tests__/connectionSourceSync.test.ts similarity index 97% rename from packages/app/src/components/ChartEditor/__tests__/resolveConnectionSourceSync.test.ts rename to packages/app/src/utils/__tests__/connectionSourceSync.test.ts index 57efcda348..99f22bcb18 100644 --- a/packages/app/src/components/ChartEditor/__tests__/resolveConnectionSourceSync.test.ts +++ b/packages/app/src/utils/__tests__/connectionSourceSync.test.ts @@ -1,4 +1,4 @@ -import { resolveConnectionSourceSync } from '@/components/ChartEditor/RawSqlChartEditor'; +import { resolveConnectionSourceSync } from '@/utils/connectionSourceSync'; const sources = [ { id: 'src-a', connection: 'conn-1' }, diff --git a/packages/app/src/utils/connectionSourceSync.ts b/packages/app/src/utils/connectionSourceSync.ts new file mode 100644 index 0000000000..98878c305b --- /dev/null +++ b/packages/app/src/utils/connectionSourceSync.ts @@ -0,0 +1,63 @@ +import { parseVariableReference } from '@hyperdx/common-utils/dist/referenceVariables'; + +type ConnectionSourceSyncResult = + | { field: 'connection'; value: string } + | { field: 'source'; value: '' } + | null; + +/** + * Decides how a form's `connection` and `source` fields should be kept in + * sync. Extracted as a pure function so the behavior can be unit tested + * independently of the components' effects. + */ +export function resolveConnectionSourceSync({ + source, + connection, + prevSource, + prevConnection, + sources, +}: { + source: string | undefined; + connection: string | undefined; + prevSource: string | undefined; + prevConnection: string | undefined; + sources: { id: string; connection: string }[] | undefined; +}): ConnectionSourceSyncResult { + if (!sources) return null; + // A data source variable reference isn't bound to a connection here: the + // filter bar decides what it selects, so it neither moves the connection + // nor is cleared by a connection change. + const isSourceReference = parseVariableReference(source) != null; + + // When the source changes, sync the connection to match. + if (source !== prevSource) { + if (isSourceReference) return null; + const sourceConnection = sources.find(s => s.id === source)?.connection; + if (sourceConnection && sourceConnection !== connection) { + return { field: 'connection', value: sourceConnection }; + } + return null; + } + + // Set a default connection when none is selected. + if (!connection) { + const defaultConnection = sources[0]?.connection; + if (defaultConnection) { + return { field: 'connection', value: defaultConnection }; + } + return null; + } + + // When the connection changes, clear the source only if the currently + // selected source doesn't belong to the new connection. This avoids clearing + // a source that was just carried over (e.g. from builder mode), where the + // connection change above was itself triggered by the source. + if (connection !== prevConnection && prevConnection !== undefined) { + const sourceConnection = sources.find(s => s.id === source)?.connection; + if (source && !isSourceReference && sourceConnection !== connection) { + return { field: 'source', value: '' }; + } + } + + return null; +} diff --git a/packages/common-utils/src/__tests__/filters.test.ts b/packages/common-utils/src/__tests__/filters.test.ts index fd60ed8fa0..3fa24d7a9c 100644 --- a/packages/common-utils/src/__tests__/filters.test.ts +++ b/packages/common-utils/src/__tests__/filters.test.ts @@ -745,11 +745,15 @@ describe('filters', () => { }); describe('SQL query filters', () => { - const sqlFilter = (sqlTemplate: string): SqlQueryDashboardFilter => ({ + const sqlFilter = ( + sqlTemplate: string, + source?: string, + ): SqlQueryDashboardFilter => ({ id: 'sql', type: 'SQL_QUERY', name: 'Environment', connection: 'conn-1', + source, sqlTemplate, isBroadcastEnabled: false, isVariableEnabled: true, @@ -790,12 +794,30 @@ describe('filters', () => { filterName: 'Environment', language: 'sql', where: sql, - detail: `SQL query filters have no source, so ${macro} cannot be used`, + detail: `SQL uses ${macro} but no source is selected`, }, ]); }, ); + it.each([ + 'SELECT env FROM $__sourceTable WHERE $__filters', + 'SELECT env FROM $__sourceTable(gauge) UNION ALL SELECT env FROM $__sourceTable(sum)', + ])('accepts source macros when the filter has a source: %s', sql => { + expect( + validateDashboardFilterQueries([sqlFilter(sql, 'source-1')]), + ).toEqual([]); + }); + + // No source has both a table and metric tables. + it('flags a mix of bare and metric $__sourceTable calls', () => { + const sql = + 'SELECT env FROM $__sourceTable UNION ALL SELECT env FROM $__sourceTable(gauge)'; + expect( + validateDashboardFilterQueries([sqlFilter(sql, 'source-1')]), + ).toEqual([expect.objectContaining({ filterId: 'sql', where: sql })]); + }); + it.each([ ['an undeclared variable macro', 'WHERE $__filter(env, $missing)'], ['an unterminated macro', 'WHERE $__timeFilter(ts'], @@ -1292,14 +1314,71 @@ describe('filters', () => { expect(sql).not.toContain('$'); }); - it('throws when the template uses a source macro', () => { + it('throws when the template uses a source macro without a source', () => { expect(() => renderSqlQueryFilterValuesQuery({ sqlTemplate: 'SELECT env FROM $__sourceTable', variables: [], dateRange, }), - ).toThrow(/have no source/); + ).toThrow(/no source is selected/); + }); + + it("expands source macros against the filter's source", () => { + const { sql } = renderSqlQueryFilterValuesQuery({ + sqlTemplate: 'SELECT env FROM $__sourceTable WHERE $__filters', + variables: [], + sourceTables: { from: { databaseName: 'db', tableName: 'logs' } }, + filtersSQL: "(ServiceName = 'api')", + dateRange, + }); + expect(sql).toContain( + "SELECT env FROM `db`.`logs` WHERE (ServiceName = 'api')", + ); + }); + + it('expands $__filters to 1=1 when no filters apply', () => { + const { sql } = renderSqlQueryFilterValuesQuery({ + sqlTemplate: 'SELECT env FROM $__sourceTable WHERE $__filters', + variables: [], + sourceTables: { from: { databaseName: 'db', tableName: 'logs' } }, + dateRange, + }); + expect(sql).toContain('WHERE (1=1'); + }); + + it.each([ + ['a bare call on a metric source', 'SELECT 1 FROM $__sourceTable', true], + [ + 'a metric call on a non-metric source', + 'SELECT 1 FROM $__sourceTable(gauge)', + false, + ], + [ + 'a metric type the source has no table for', + 'SELECT 1 FROM $__sourceTable(sum)', + true, + ], + ])('throws on %s', (_label, sqlTemplate, isMetricSource) => { + expect(() => + renderSqlQueryFilterValuesQuery({ + sqlTemplate, + variables: [], + sourceTables: { + from: { databaseName: 'db', tableName: '' }, + metricTables: isMetricSource + ? { + gauge: 'metrics_gauge', + histogram: '', + sum: '', + summary: '', + 'exponential histogram': '', + } + : undefined, + }, + dateRange, + }), + ).toThrow(/\$__sourceTable/); }); it.each([ diff --git a/packages/common-utils/src/__tests__/utils.test.ts b/packages/common-utils/src/__tests__/utils.test.ts index 8485e745c7..739c5627b6 100644 --- a/packages/common-utils/src/__tests__/utils.test.ts +++ b/packages/common-utils/src/__tests__/utils.test.ts @@ -1643,6 +1643,65 @@ describe('utils', () => { ]); }); + it('should remap a SQL query filter source id to a name, dropping one that no longer exists', () => { + const sqlFilter = { + id: 'filter-sql', + type: 'SQL_QUERY' as const, + name: 'Service', + connection: 'conn1', + source: 'source1', + sqlTemplate: 'SELECT DISTINCT ServiceName FROM $__sourceTable', + isBroadcastEnabled: false as const, + isVariableEnabled: true as const, + }; + const dashboard: z.infer = { + id: 'dashboard1', + name: 'SQL Filter Dashboard', + tags: [], + tiles: [], + filters: [sqlFilter, { ...sqlFilter, id: 'gone', source: 'deleted' }], + }; + const sources: TSource[] = [ + { + id: 'source1', + name: 'Logs', + connection: 'conn1', + kind: SourceKind.Log, + from: { databaseName: 'db1', tableName: 'logs_table' }, + timestampValueExpression: 'Timestamp', + defaultTableSelectExpression: '', + }, + ]; + const connections: Connection[] = [ + { + id: 'conn1', + name: 'Production DB', + host: 'http://localhost:8123', + username: 'default', + }, + ]; + + const template = convertToDashboardTemplate( + dashboard, + sources, + connections, + ); + + expect(template.filters?.[0]).toEqual({ + ...sqlFilter, + connection: 'Production DB', + source: 'Logs', + }); + // An empty name would fail the template schema's min(1) on import. + expect(template.filters?.[1]).toEqual({ + ...sqlFilter, + id: 'gone', + connection: 'Production DB', + source: undefined, + }); + expect(() => DashboardTemplateSchema.parse(template)).not.toThrow(); + }); + it('should convert a dashboard without filters to a dashboard template', () => { const dashboard: z.infer = { id: 'dashboard1', diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index a38d8e6567..11bbbf2f4e 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -51,6 +51,9 @@ import { CteChartConfig, DateRange, DisplayType, + Filter, + isLogSource, + isTraceSource, MetricsDataType, PromqlChartConfig, QuerySettings, @@ -61,6 +64,7 @@ import { SortSpecificationList, SqlAstFilter, SQLInterval, + TSource, } from '@/types'; import { substituteChartConfigVariables } from '@/variables'; @@ -2600,7 +2604,16 @@ async function translateMetricChartConfig( /** Renders the config's filters into a SQL condition string */ async function renderFiltersToSql( - chartConfig: RawSqlChartConfig, + chartConfig: Pick< + RawSqlChartConfig, + | 'filters' + | 'source' + | 'from' + | 'connection' + | 'implicitColumnExpression' + | 'bodyExpression' + | 'useTextIndexForImplicitColumn' + >, metadata: Metadata, ): Promise { if ( @@ -2650,6 +2663,38 @@ async function renderFiltersToSql( return conditions.length > 0 ? `(${conditions.join(' AND ')})` : undefined; } +/** + * The condition `$__filters` expands to for SQL that queries `source`, as for + * a raw SQL tile with that source. + */ +export function renderSourceFiltersToSql({ + source, + filters, + metadata, +}: { + source: TSource; + filters: Filter[]; + metadata: Metadata; +}): Promise { + const isLogOrTrace = isLogSource(source) || isTraceSource(source); + return renderFiltersToSql( + { + filters, + source: source.id, + from: source.from, + connection: source.connection, + implicitColumnExpression: isLogOrTrace + ? source.implicitColumnExpression + : undefined, + bodyExpression: isLogSource(source) ? source.bodyExpression : undefined, + useTextIndexForImplicitColumn: isLogOrTrace + ? source.useTextIndexForImplicitColumn + : undefined, + }, + metadata, + ); +} + export async function renderRawSqlChartConfig( chartConfig: RawSqlChartConfig & Partial, metadata: Metadata, diff --git a/packages/common-utils/src/core/utils.ts b/packages/common-utils/src/core/utils.ts index c0b6ea41df..6ce5b3caba 100644 --- a/packages/common-utils/src/core/utils.ts +++ b/packages/common-utils/src/core/utils.ts @@ -952,6 +952,11 @@ export function convertToDashboardTemplate( if (isStaticListFilter(filter) || isConnectionFilter(filter)) return filter; if (isSqlQueryFilter(filter)) { filter.connection = getConnectionName(filter.connection); + // The source is optional, so drop it rather than export an empty name + // the template schema would reject. + if (filter.source) { + filter.source = getSourceName(filter.source) || undefined; + } return filter; } if (isDataSourceFilter(filter)) { diff --git a/packages/common-utils/src/filters.ts b/packages/common-utils/src/filters.ts index 0d55cd315d..ed05ebade2 100644 --- a/packages/common-utils/src/filters.ts +++ b/packages/common-utils/src/filters.ts @@ -3,6 +3,7 @@ import * as SQLParser from 'node-sql-parser'; import { escapeSqlString, replaceJsonExpressions } from '@/core/utils'; import { getSourceDependentMacrosUsed, + getSourceTableMacroArgCounts, hasMacro, INTERVAL_MACROS, replaceMacros, @@ -23,10 +24,13 @@ import { DashboardFilter, DashboardFilterValue, Filter, + MetricsDataType, PROMQL_ADHOC_FILTER_OPERATORS, PromqlConditionFilter, + RawSqlChartConfig, SourceKind, SQL_ADHOC_FILTER_OPERATORS, + TSource, } from '@/types'; import { escapePromqlStringValue, @@ -812,10 +816,11 @@ export function validateDashboardFilterQueries( for (const filter of filters) { if (isSqlQueryFilter(filter)) { - const detail = validateSqlQueryFilterTemplate( - filter.sqlTemplate, + // Source IDs here may not be mapped to this team's sources yet. + const detail = validateSqlQueryFilterTemplate(filter.sqlTemplate, { variables, - ); + sourceTables: filter.source ? 'unknown' : undefined, + }); if (detail) { issues.push({ filterId: filter.id, @@ -857,16 +862,70 @@ export function validateDashboardFilterQueries( return issues; } +/** The tables a SQL query filter's source resolves `$__sourceTable` to. */ +export type SqlQueryFilterSourceTables = Pick< + RawSqlChartConfig, + 'from' | 'metricTables' +>; + +/** The `$__sourceTable` tables of a source, as a raw SQL tile resolves them. */ +export function getSqlQueryFilterSourceTables( + source: TSource, +): SqlQueryFilterSourceTables { + return { + from: source.from, + metricTables: + source.kind === SourceKind.Metric ? source.metricTables : undefined, + }; +} + +/** + * Stand-in tables for a source that is set but not loaded. Every metric table + * is configured when the SQL calls `$__sourceTable()`, so only a + * mix of bare and metric calls, which no source satisfies, is rejected. + */ +function getUnknownSourceTables( + sqlTemplate: string, +): SqlQueryFilterSourceTables { + const hasMetricCall = getSourceTableMacroArgCounts(sqlTemplate).some( + count => count > 0, + ); + return { + from: { databaseName: 'source', tableName: 'table' }, + metricTables: hasMetricCall + ? { + [MetricsDataType.Gauge]: 'table', + [MetricsDataType.Histogram]: 'table', + [MetricsDataType.Sum]: 'table', + [MetricsDataType.Summary]: 'table', + [MetricsDataType.ExponentialHistogram]: 'table', + } + : undefined, + }; +} + +type SqlQueryFilterTemplateContext = { + /** The dashboard's variables and their current selections. */ + variables: ChartVariable[] | undefined; + /** + * The filter's source tables. `'unknown'` when the filter has a source whose + * tables aren't loaded; undefined when it has no source. + */ + sourceTables?: SqlQueryFilterSourceTables | 'unknown'; + /** The broadcast filter conditions `$__filters` expands to. */ + filtersSQL?: string; +}; + /** * Why a SQL query filter's template can't expand, or undefined when it can. * The SQL itself is not parsed: ClickHouse syntax is only checked when it runs. */ export function validateSqlQueryFilterTemplate( sqlTemplate: string, - variables: ChartVariable[], + context: SqlQueryFilterTemplateContext, ): string | undefined { try { - expandSqlQueryFilterTemplate(sqlTemplate, variables); + expandSqlQueryFilterTemplate(sqlTemplate, context); return undefined; } catch (e) { return e instanceof Error ? e.message : String(e); @@ -876,15 +935,17 @@ export function validateSqlQueryFilterTemplate( /** Expand a SQL query filter's macros and variables. Throws when it can't. */ function expandSqlQueryFilterTemplate( sqlTemplate: string, - variables: ChartVariable[] | undefined, + { variables, sourceTables, filtersSQL }: SqlQueryFilterTemplateContext, ): string { - const sourceMacros = getSourceDependentMacrosUsed(sqlTemplate); - if (sourceMacros.length > 0) { - throw new Error( - `SQL query filters have no source, so ${sourceMacros - .map(macro => `$__${macro}`) - .join(' and ')} cannot be used`, - ); + if (!sourceTables) { + const sourceMacros = getSourceDependentMacrosUsed(sqlTemplate); + if (sourceMacros.length > 0) { + throw new Error( + `SQL uses ${sourceMacros + .map(macro => `$__${macro}`) + .join(' and ')} but no source is selected`, + ); + } } const intervalMacros = INTERVAL_MACROS.filter(macro => hasMacro(sqlTemplate, macro), @@ -896,7 +957,11 @@ function expandSqlQueryFilterTemplate( .join(' and ')} cannot be used`, ); } - return replaceMacros({ sqlTemplate, variables }); + const tables = + sourceTables === 'unknown' + ? getUnknownSourceTables(sqlTemplate) + : sourceTables; + return replaceMacros({ sqlTemplate, variables, ...tables }, filtersSQL); } /** Most dropdown values a SQL query filter reads from its query. */ @@ -912,14 +977,13 @@ export const SQL_QUERY_FILTER_VALUES_LIMIT = 10_000; */ export function renderSqlQueryFilterValuesQuery({ sqlTemplate, - variables, dateRange, -}: { + ...context +}: SqlQueryFilterTemplateContext & { sqlTemplate: string; - variables: ChartVariable[] | undefined; dateRange: [Date, Date]; }): { sql: string; params: Record } { - const sql = expandSqlQueryFilterTemplate(sqlTemplate, variables) + const sql = expandSqlQueryFilterTemplate(sqlTemplate, context) .trim() .replace(/;+$/, ''); // The SQL goes on its own lines so a trailing `--` comment can't swallow the @@ -1014,6 +1078,14 @@ export const QUERY_EXPRESSION_FILTER_SOURCE_KINDS: SourceKind[] = [ SourceKind.Metric, ]; +/** The source kinds a SQL_QUERY filter can use, the same as raw SQL tiles. */ +export const SQL_QUERY_FILTER_SOURCE_KINDS: SourceKind[] = [ + SourceKind.Log, + SourceKind.Trace, + SourceKind.Session, + SourceKind.Metric, +]; + /** * The source kinds an ADHOC filter of the given source type can read keys * from. Metric sources need a metric type to pick a table, so they are left out. diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index c733b1bc78..1a14fc6453 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -2448,6 +2448,11 @@ export const SqlQueryDashboardFilterSchema = dashboardFilterBaseSchema.extend({ type: z.literal(DashboardFilterType.enum.SQL_QUERY), /** ID of the ClickHouse connection the query runs on. */ connection: z.string().min(1), + /** + * ID of the source `$__sourceTable` and `$__filters` resolve against, on the + * same connection. Optional, as for raw SQL tiles. + */ + source: z.string().min(1).optional(), sqlTemplate: z.string().min(1), // Variable-only: there is no column to broadcast isBroadcastEnabled: z.literal(false), From 7dedd9bf3fa9c0073deb39aa3645c90e7b6dc3eb Mon Sep 17 00:00:00 2001 From: Karl Power Date: Wed, 18 Nov 2026 15:48:24 +0100 Subject: [PATCH 2/4] refactor: alias SQL_QUERY_FILTER_SOURCE_KINDS to QUERY_EXPRESSION_FILTER_SOURCE_KINDS --- packages/common-utils/src/filters.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/common-utils/src/filters.ts b/packages/common-utils/src/filters.ts index ed05ebade2..fcb4dfe97d 100644 --- a/packages/common-utils/src/filters.ts +++ b/packages/common-utils/src/filters.ts @@ -1078,13 +1078,12 @@ export const QUERY_EXPRESSION_FILTER_SOURCE_KINDS: SourceKind[] = [ SourceKind.Metric, ]; -/** The source kinds a SQL_QUERY filter can use, the same as raw SQL tiles. */ -export const SQL_QUERY_FILTER_SOURCE_KINDS: SourceKind[] = [ - SourceKind.Log, - SourceKind.Trace, - SourceKind.Session, - SourceKind.Metric, -]; +/** + * The source kinds a SQL_QUERY filter can use, the same as raw SQL tiles. + * Kept as an alias so the two filter types cannot drift apart. + */ +export const SQL_QUERY_FILTER_SOURCE_KINDS: SourceKind[] = + QUERY_EXPRESSION_FILTER_SOURCE_KINDS; /** * The source kinds an ADHOC filter of the given source type can read keys From b6817f9e35dcc8561e628df175738608dbb4e1c8 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Wed, 18 Nov 2026 11:11:17 -0500 Subject: [PATCH 3/4] feat: Allow SQL filter to reference source var (#3714) --- .../sql-query-dashboard-filter-source.md | 2 +- packages/api/openapi.json | 2 +- .../api/src/mcp/prompts/dashboards/content.ts | 2 +- .../api/src/mcp/tools/dashboards/schemas.ts | 8 +- .../api/__tests__/dashboard.int.test.ts | 52 +++++++++--- packages/api/src/routers/api/dashboards.ts | 5 +- .../__tests__/dashboards.int.test.ts | 59 +++++++++++-- .../src/routers/external-api/v2/dashboards.ts | 6 +- .../external-api/v2/utils/dashboards.ts | 4 - .../src/utils/__tests__/filterSources.test.ts | 29 ++----- packages/api/src/utils/filterSources.ts | 27 ++---- packages/api/src/utils/zod.ts | 2 +- packages/app/src/DBDashboardImportPage.tsx | 10 +-- .../__tests__/DBDashboardImportPage.test.tsx | 30 +++++-- .../SqlQueryFilterEditForm.tsx | 77 +++++++++-------- .../__tests__/SqlQueryFilterEditForm.test.tsx | 82 +++++++++++++++++-- .../useSqlQueryFilterValues.test.tsx | 12 +++ .../app/src/hooks/useSqlQueryFilterValues.tsx | 8 ++ .../src/__tests__/referenceVariables.test.ts | 29 +++++++ .../common-utils/src/referenceVariables.ts | 11 ++- packages/common-utils/src/types.ts | 3 +- 21 files changed, 322 insertions(+), 138 deletions(-) diff --git a/.changeset/sql-query-dashboard-filter-source.md b/.changeset/sql-query-dashboard-filter-source.md index 5eef88e796..a981db1fad 100644 --- a/.changeset/sql-query-dashboard-filter-source.md +++ b/.changeset/sql-query-dashboard-filter-source.md @@ -6,4 +6,4 @@ feat: let SQL query dashboard filters use an optional source -A SQL query filter can now pick a source on its connection. With one, its SQL can use `$__sourceTable` and `$__filters`, which applies the dashboard's filter selections for that source, the same way raw SQL tiles do. The external API and MCP read and write it as `sourceId`. +A SQL query filter can now pick a source on its connection, or a data source variable, and can do so on a connection variable too. With one, its SQL can use `$__sourceTable` and `$__filters`, which applies the dashboard's filter selections for that source, the same way raw SQL tiles do. The external API and MCP read and write it as `sourceId`. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 9a0f6ae4b4..ba618f6178 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -4553,7 +4553,7 @@ }, "sourceId": { "type": "string", - "description": "Optional Log, Trace, Session, or Metric source on the same\nconnection. Lets `sqlTemplate` use `$__sourceTable` and\n`$__filters`. Requires a connection ID in `connectionId`, not\na connection variable.\n", + "description": "Optional Log, Trace, Session, or Metric source on the same\nconnection, or `$name` of a DATA_SOURCE filter on this\ndashboard. Lets `sqlTemplate` use `$__sourceTable` and\n`$__filters`.\n", "example": "65f5e4a3b9e77c001a345678" }, "sqlTemplate": { diff --git a/packages/api/src/mcp/prompts/dashboards/content.ts b/packages/api/src/mcp/prompts/dashboards/content.ts index eabc46825e..6c41c79813 100644 --- a/packages/api/src/mcp/prompts/dashboards/content.ts +++ b/packages/api/src/mcp/prompts/dashboards/content.ts @@ -1216,7 +1216,7 @@ SQL query filter shape: { type: "SQL_QUERY", name, connectionId, sourceId?, sqlTemplate, variableName? } connectionId ClickHouse connection the query runs on (from clickstack_list_sources), or a CONNECTION filter's "$variableName". - sourceId Optional log, trace, session, or metric source on that connection. Requires a connection ID, not a connection variable. Needed for $__sourceTable and $__filters; $__filters expands to the dashboard's broadcast filter selections for that source. + sourceId Optional log, trace, session, or metric source on that connection, or a DATA_SOURCE filter's "$variableName". Needed for $__sourceTable and $__filters; $__filters expands to the dashboard's broadcast filter selections for that source. sqlTemplate SELECT whose FIRST column becomes the dropdown values, cast to string with nulls and duplicates dropped, in row order (add ORDER BY). May use the time macros and other filters' variables, plus $__sourceTable and $__filters when sourceId is set. Use it when the values live in a lookup table, dictionary, or join that no single source column covers; otherwise prefer QUERY_EXPRESSION. Like a static filter it is variable-only, so tiles pass the column explicitly: $__filter(, $). diff --git a/packages/api/src/mcp/tools/dashboards/schemas.ts b/packages/api/src/mcp/tools/dashboards/schemas.ts index 86bc522de5..9e062647b7 100644 --- a/packages/api/src/mcp/tools/dashboards/schemas.ts +++ b/packages/api/src/mcp/tools/dashboards/schemas.ts @@ -1798,13 +1798,13 @@ const mcpSqlQueryFilterSchema = z 'ClickHouse connection the query runs on: a connection ID from ' + 'clickstack_list_sources, or `$name` of a CONNECTION filter on this dashboard.', ), - sourceId: objectIdSchema + sourceId: objectIdOrVariableReferenceSchema .optional() .describe( 'Optional log, trace, session, or metric source on the same connection. ' + - 'Lets sqlTemplate use $__sourceTable and $__filters. Requires a ' + - 'connection ID in connectionId, not a connection variable. Get IDs ' + - 'from clickstack_list_sources.', + 'Lets sqlTemplate use $__sourceTable and $__filters. Get IDs from ' + + 'clickstack_list_sources. ' + + SOURCE_VARIABLE_REFERENCE_NOTE, ), sqlTemplate: SqlQueryDashboardFilterSchema.shape.sqlTemplate.describe( 'ClickHouse SELECT whose FIRST column becomes the dropdown values (cast to ' + diff --git a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts index c53edc4aa1..3f516bf17d 100644 --- a/packages/api/src/routers/api/__tests__/dashboard.int.test.ts +++ b/packages/api/src/routers/api/__tests__/dashboard.int.test.ts @@ -2274,27 +2274,51 @@ describe('dashboard router', () => { expect(response.body.message).toContain(source._id.toString()); }); - it('rejects a source with a connection variable', async () => { + it('accepts a source with a connection variable', async () => { const source = await createSqlFilterSource(SourceKind.Log); + const filter = makeSqlFilter({ + connection: '$conn', + source: source._id.toString(), + }); - const response = await agent + const created = await agent + .post('/dashboards') + .send({ ...MOCK_DASHBOARD, filters: [connectionFilter, filter] }) + .expect(200); + + expect(created.body.filters).toEqual([connectionFilter, filter]); + }); + + it('accepts a data source variable', async () => { + const dataSourceFilter = { + id: new Types.ObjectId().toString(), + type: 'DATA_SOURCE' as const, + name: 'Logs source', + variableName: 'logs', + allowedSourceKinds: [SourceKind.Log], + minSelections: 1, + maxSelections: 1, + isBroadcastEnabled: false, + isVariableEnabled: true, + }; + const filter = makeSqlFilter({ connection: '$conn', source: '$logs' }); + + const created = await agent .post('/dashboards') .send({ ...MOCK_DASHBOARD, - filters: [ - connectionFilter, - makeSqlFilter({ - connection: '$conn', - source: source._id.toString(), - }), - ], + filters: [connectionFilter, dataSourceFilter, filter], }) - .expect(400); + .expect(200); - expect(response.body.message).toContain('not variables'); + expect(created.body.filters).toEqual([ + connectionFilter, + dataSourceFilter, + filter, + ]); }); - it('rejects a source variable', async () => { + it('rejects a reference to an unknown data source variable', async () => { const response = await agent .post('/dashboards') .send({ @@ -2303,7 +2327,9 @@ describe('dashboard router', () => { }) .expect(400); - expect(response.body.message).toContain('not variables'); + expect(JSON.stringify(response.body)).toMatch( + /does not name a data source variable/, + ); }); }); diff --git a/packages/api/src/routers/api/dashboards.ts b/packages/api/src/routers/api/dashboards.ts index 27bf2f7d52..4fe5c0fa68 100644 --- a/packages/api/src/routers/api/dashboards.ts +++ b/packages/api/src/routers/api/dashboards.ts @@ -40,7 +40,6 @@ import type { ObjectId } from '@/models'; import { getChangedFilterSourceRefs, getFilterSourceRefError, - getSqlQueryFilterSourceVariableError, } from '@/utils/filterSources'; import { objectIdSchema } from '@/utils/zod'; @@ -70,7 +69,7 @@ const addFilterIssues = ( /** * Rejects PROMETHEUS_LABEL, ADHOC and SQL_QUERY filters that reference a source * that is missing or of the wrong kind, and SQL_QUERY filters whose source is - * on another connection or that pair a source with a variable. Resolving these filters needs a live source of the + * on another connection. Resolving these filters needs a live source of the * right kind, unlike other filter types. References the stored filter with * the same ID already made are skipped, so a since-deleted source doesn't block * unrelated saves. Returns an error message, or null when there is nothing to @@ -81,8 +80,6 @@ async function validateFilterSources( filters: DashboardFilter[] = [], existingFilters: DashboardFilter[] = [], ): Promise { - const variableError = getSqlQueryFilterSourceVariableError(filters); - if (variableError != null) return variableError; const refs = getChangedFilterSourceRefs(filters, existingFilters).filter( // Variable references are checked by validateDashboardVariableReferences ref => parseVariableReference(ref.id) == null, diff --git a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts index 374f5c3247..5c53b3f1ee 100644 --- a/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/dashboards.int.test.ts @@ -6425,22 +6425,65 @@ describe('External API v2 Dashboards - new format', () => { expect(response.body.message).toContain(traceSource._id.toString()); }); - it('rejects a source with a connection variable', async () => { + const connectionVariableInput = { + id: new ObjectId().toString(), + type: 'CONNECTION', + name: 'Connection', + variableName: 'conn', + }; + + it('accepts a source with a connection variable', async () => { const response = await sendFilters([ - { - id: new ObjectId().toString(), - type: 'CONNECTION', - name: 'Connection', - variableName: 'conn', - }, + connectionVariableInput, sqlFilterInput({ connectionId: '$conn', sourceId: traceSource._id.toString(), }), ]); + expect(response.status).toBe(200); + expect(response.body.data.filters[1]).toMatchObject({ + connectionId: '$conn', + sourceId: traceSource._id.toString(), + }); + }); + + it('accepts a data source variable and round-trips it through GET', async () => { + const response = await sendFilters([ + connectionVariableInput, + { + id: new ObjectId().toString(), + type: 'DATA_SOURCE', + name: 'Traces source', + variableName: 'traces', + allowedSourceKinds: [SourceKind.Trace], + }, + sqlFilterInput({ connectionId: '$conn', sourceId: '$traces' }), + ]); + expect(response.status).toBe(200); + expect(response.body.data.filters[2]).toMatchObject({ + connectionId: '$conn', + sourceId: '$traces', + }); + + const getResponse = await authRequest( + 'get', + `${BASE_URL}/${response.body.data.id}`, + ).expect(200); + expect(getResponse.body.data.filters).toEqual( + response.body.data.filters, + ); + }); + + it('rejects a reference to an unknown data source variable', async () => { + const response = await sendFilters([ + sqlFilterInput({ sourceId: '$traces' }), + ]); + expect(response.status).toBe(400); - expect(response.body.message).toContain('not variables'); + expect(JSON.stringify(response.body)).toMatch( + /does not name a data source variable/, + ); }); }); diff --git a/packages/api/src/routers/external-api/v2/dashboards.ts b/packages/api/src/routers/external-api/v2/dashboards.ts index 3cf80c645e..d8baa07f01 100644 --- a/packages/api/src/routers/external-api/v2/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/dashboards.ts @@ -2913,9 +2913,9 @@ const EXTERNAL_DASHBOARD_PROJECTION = { * type: string * description: | * Optional Log, Trace, Session, or Metric source on the same - * connection. Lets `sqlTemplate` use `$__sourceTable` and - * `$__filters`. Requires a connection ID in `connectionId`, not - * a connection variable. + * connection, or `$name` of a DATA_SOURCE filter on this + * dashboard. Lets `sqlTemplate` use `$__sourceTable` and + * `$__filters`. * example: "65f5e4a3b9e77c001a345678" * sqlTemplate: * type: string diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 54f1301cab..220b01dc6a 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -71,7 +71,6 @@ import { getChangedFilterSourceRefs, getFilterSourceRefError, getInvalidSourceIds, - getSqlQueryFilterSourceVariableError, } from '@/utils/filterSources'; import logger from '@/utils/logger'; import { @@ -1916,9 +1915,6 @@ export async function validateDashboardTiles( const internalFilters = (filters ?? []).map(filter => translateExternalFilterToFilter({ id: '', ...filter }), ); - const sqlQueryFilterVariableError = - getSqlQueryFilterSourceVariableError(internalFilters); - if (sqlQueryFilterVariableError != null) return sqlQueryFilterVariableError; const filterSourceError = getFilterSourceRefError( getChangedFilterSourceRefs(internalFilters, existingFilters).filter( ref => parseVariableReference(ref.id) == null, diff --git a/packages/api/src/utils/__tests__/filterSources.test.ts b/packages/api/src/utils/__tests__/filterSources.test.ts index 739bf68575..b89b06d35b 100644 --- a/packages/api/src/utils/__tests__/filterSources.test.ts +++ b/packages/api/src/utils/__tests__/filterSources.test.ts @@ -3,7 +3,6 @@ import { DashboardFilter, SourceKind } from '@hyperdx/common-utils/dist/types'; import { getChangedFilterSourceRefs, getFilterSourceRefError, - getSqlQueryFilterSourceVariableError, } from '@/utils/filterSources'; const adhoc = ( @@ -86,6 +85,12 @@ describe('getChangedFilterSourceRefs', () => { ]); }); + it('does not pin a SQL query source to a connection variable', () => { + expect( + getChangedFilterSourceRefs([sqlQuery({ connection: '$conn' })]), + ).toEqual([{ id: 'logs', kinds: sqlQueryKinds }]); + }); + it('re-checks a SQL query source whose connection changed', () => { expect( getChangedFilterSourceRefs( @@ -156,25 +161,3 @@ describe('getFilterSourceRefError', () => { ); }); }); - -describe('getSqlQueryFilterSourceVariableError', () => { - it('accepts a source ID on a connection ID, or no source', () => { - expect( - getSqlQueryFilterSourceVariableError([ - sqlQuery(), - sqlQuery({ connection: '$conn', source: undefined }), - ]), - ).toBeNull(); - }); - - it('names filters with a source variable or a connection variable', () => { - expect( - getSqlQueryFilterSourceVariableError([ - sqlQuery({ name: 'A', source: '$logs' }), - sqlQuery({ name: 'B', connection: '$conn' }), - ]), - ).toBe( - 'SQL_QUERY filters with a source need a source ID and a connection ID, not variables. The following filters use a variable: A, B', - ); - }); -}); diff --git a/packages/api/src/utils/filterSources.ts b/packages/api/src/utils/filterSources.ts index ce613ff096..d738d60026 100644 --- a/packages/api/src/utils/filterSources.ts +++ b/packages/api/src/utils/filterSources.ts @@ -37,7 +37,11 @@ function getFilterSourceRefs(filter: FilterWithOptionalId): SourceRef[] { { id: filter.source, kinds: SQL_QUERY_FILTER_SOURCE_KINDS, - connection: filter.connection, + // A connection variable can select any connection, so only an ID + // pins the source to one. + ...(parseVariableReference(filter.connection) == null && { + connection: filter.connection, + }), }, ]; } @@ -111,24 +115,3 @@ export function getFilterSourceRefError( if (mismatched.length === 0) return null; return `The following filter source IDs are not on the filter's connection: ${[...new Set(mismatched)].join(', ')}`; } - -/** - * An error naming SQL_QUERY filters whose source or connection is a variable - * while it has a source, or null. A source is only offered on a fixed - * connection, so neither field may change what the other points at. - */ -export function getSqlQueryFilterSourceVariableError( - filters: FilterWithOptionalId[] = [], -): string | null { - const invalid = filters - .filter(isSqlQueryFilter) - .filter( - ({ source, connection }) => - source != null && - (parseVariableReference(source) != null || - parseVariableReference(connection) != null), - ) - .map(({ name }) => name); - if (invalid.length === 0) return null; - return `SQL_QUERY filters with a source need a source ID and a connection ID, not variables. The following filters use a variable: ${invalid.join(', ')}`; -} diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index 5bdf6526b0..f6513b2205 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -214,7 +214,7 @@ const externalSqlQueryFilterShape = SqlQueryDashboardFilterSchema.omit({ source: true, }).extend({ connectionId: objectIdOrVariableReferenceSchema, - sourceId: objectIdSchema.optional(), + sourceId: objectIdOrVariableReferenceSchema.optional(), isBroadcastEnabled: z.literal(false).default(false), isVariableEnabled: z.literal(true).default(true), }); diff --git a/packages/app/src/DBDashboardImportPage.tsx b/packages/app/src/DBDashboardImportPage.tsx index de05a15914..bbb350bae3 100644 --- a/packages/app/src/DBDashboardImportPage.tsx +++ b/packages/app/src/DBDashboardImportPage.tsx @@ -473,11 +473,8 @@ const isSourceBackedTemplateFilter = ( | SqlQueryDashboardFilter => (isQueryExpressionFilter(filter) || isPrometheusLabelFilter(filter) || - // The source is optional for a SQL query filter, and only offered on a - // connection ID - (isSqlQueryFilter(filter) && - !!filter.source && - !isVariableRef(filter.connection))) && + // The source is optional for a SQL query filter + (isSqlQueryFilter(filter) && !!filter.source)) && !isVariableRef(filter.source); /** @@ -1443,7 +1440,8 @@ export function Mapping({ input }: { input: DashboardTemplate }) { {/** Map filter sources */} {input.filters?.map((filter, i) => { - // A SQL query filter's source must be on its mapped connection. + // A SQL query filter's source must be on its mapped connection, + // unless that is a connection variable, which isn't mapped. const sourceConnectionId = isSqlQueryFilter(filter) ? filterConnectionMappings?.[i] : undefined; diff --git a/packages/app/src/__tests__/DBDashboardImportPage.test.tsx b/packages/app/src/__tests__/DBDashboardImportPage.test.tsx index 34e77da879..892c9542b9 100644 --- a/packages/app/src/__tests__/DBDashboardImportPage.test.tsx +++ b/packages/app/src/__tests__/DBDashboardImportPage.test.tsx @@ -486,20 +486,34 @@ describe('Dashboard import - SQL query filter connection', () => { }); }); - it('maps neither the connection variable nor a source paired with it', () => { - const result = buildMappingFormSchema( - sqlQueryFilterTemplate('$conn', 'Logs'), - mockSources, - ).safeParse({ + const parseSqlQueryFilterMappings = ( + template: ReturnType, + filterSourceMapping: string, + filterConnectionMapping: string, + ) => + buildMappingFormSchema(template, mockSources).safeParse({ dashboardName: 'SQL Query Filter', tags: [], tileSourceMappings: ['src-metrics'], connectionMappings: [''], - filterSourceMappings: [''], - filterConnectionMappings: [''], + filterSourceMappings: [filterSourceMapping], + filterConnectionMappings: [filterConnectionMapping], }); - expect(result.success).toBe(true); + it('maps a source paired with a connection variable, on any connection', () => { + const template = sqlQueryFilterTemplate('$conn', 'Logs'); + + expect(parseSqlQueryFilterMappings(template, '', '').success).toBe(false); + expect( + parseSqlQueryFilterMappings(template, 'src-other-logs', '').success, + ).toBe(true); + }); + + it('maps neither connection nor source variables', () => { + expect( + parseSqlQueryFilterMappings(sqlQueryFilterTemplate('$conn', '$logs'), '', '') + .success, + ).toBe(true); }); it('rejects a PromQL source for a filter with a source', () => { diff --git a/packages/app/src/components/DashboardFiltersModal/SqlQueryFilterEditForm.tsx b/packages/app/src/components/DashboardFiltersModal/SqlQueryFilterEditForm.tsx index a09a0281fb..fe8b377129 100644 --- a/packages/app/src/components/DashboardFiltersModal/SqlQueryFilterEditForm.tsx +++ b/packages/app/src/components/DashboardFiltersModal/SqlQueryFilterEditForm.tsx @@ -90,49 +90,56 @@ export const SqlQueryFilterEditForm = ({ control, name: ['connection', 'source', 'sqlTemplate'], }); - const sourceObject = source ? sources?.find(s => s.id === source) : undefined; - // A source is only offered on a connection ID, since a connection variable - // can later select a connection the source isn't on. - const isConnectionVariable = parseVariableReference(connection) != null; + const isSourceVariable = parseVariableReference(source) != null; + // Source and connection variables preview, and are validated, as the source + // and connection they currently select. + const resolvedSource = useResolvedReference(source); + const resolvedConnection = useResolvedReference(connection); + const sourceObject = resolvedSource + ? sources?.find(s => s.id === resolvedSource) + : undefined; + const storedSourceObject = isSourceVariable ? undefined : sourceObject; const prevSource = usePrevious(source); - const prevConnection = usePrevious(connection); + const prevConnection = usePrevious(resolvedConnection); useEffect(() => { - if (isConnectionVariable) { - if (source) setValue('source', ''); - return; - } // The source field is shared with the other filter types, so switching // type can carry over one this filter can't use. if ( source && + !isSourceVariable && sources && - (!sourceObject || - !SQL_QUERY_FILTER_SOURCE_KINDS.includes(sourceObject.kind)) + (!storedSourceObject || + !SQL_QUERY_FILTER_SOURCE_KINDS.includes(storedSourceObject.kind)) ) { setValue('source', ''); return; } const update = resolveConnectionSourceSync({ source, - connection, + connection: resolvedConnection, prevSource, prevConnection, sources, }); - if (update?.field === 'connection') { + // A connection variable stays as picked, as on raw SQL tiles + if ( + update?.field === 'connection' && + parseVariableReference(connection) == null + ) { setValue('connection', update.value); } else if (update?.field === 'source') { setValue('source', update.value); } }, [ connection, - isConnectionVariable, + isSourceVariable, prevConnection, prevSource, + resolvedConnection, setValue, source, - sourceObject, + storedSourceObject, sources, ]); @@ -155,15 +162,20 @@ export const SqlQueryFilterEditForm = ({ source?: string; sqlTemplate: string; }>(); - // A connection variable previews as the connection it currently selects. - const resolvedConnection = useResolvedReference(connection); - const unresolvedConnection = - parseVariableReference(connection) != null && !resolvedConnection; - const canPreview = !!resolvedConnection && !!sqlTemplate?.trim(); + const unresolvedVariables = [ + !resolvedConnection && parseVariableReference(connection) != null + ? connection + : undefined, + isSourceVariable && !resolvedSource ? source : undefined, + ].filter((value): value is string => !!value); + const canPreview = + !!resolvedConnection && + !!sqlTemplate?.trim() && + unresolvedVariables.length === 0; const queryClient = useQueryClient(); const runPreview = () => { if (!canPreview) return; - const previewSource = source || undefined; + const previewSource = resolvedSource || undefined; // The same query keeps its cache key, so re-run it explicitly. if ( previewed?.connection === resolvedConnection && @@ -182,10 +194,11 @@ export const SqlQueryFilterEditForm = ({ }; let sourceTables: SqlQueryFilterSourceTables | 'unknown' | undefined; - if (sourceObject) { - sourceTables = getSqlQueryFilterSourceTables(sourceObject); + if (storedSourceObject) { + sourceTables = getSqlQueryFilterSourceTables(storedSourceObject); } else if (source) { - // Set, but the sources list hasn't loaded yet. + // A variable can later select a source with other tables, and a source ID + // may be set before the sources list loads. sourceTables = 'unknown'; } @@ -222,7 +235,7 @@ export const SqlQueryFilterEditForm = ({ {/* The source select's wrapper is inline-flex; a grid cell stretches it to the full width. */} @@ -235,12 +248,11 @@ export const SqlQueryFilterEditForm = ({ control={control} name="source" allowedSourceKinds={SQL_QUERY_FILTER_SOURCE_KINDS} - connectionId={connection} + allowSourceVariables + connectionId={resolvedConnection} + variableConnection={connection} clearable - disabled={isConnectionVariable} - placeholder={ - isConnectionVariable ? 'Not with a connection variable' : 'None' - } + placeholder="None" comboboxProps={{ withinPortal: true }} /> @@ -265,9 +277,10 @@ export const SqlQueryFilterEditForm = ({ parentRef={TOOLTIP_PORTAL_TARGET} /> - {unresolvedConnection && ( + {unresolvedVariables.length > 0 && ( - Select a value for {connection} to preview + Select a value for {unresolvedVariables.join(' and ')} to + preview )}