Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/alert-history-last-evaluation.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
'@hyperdx/api': patch
---

perf(api): a due grouped alert that a value of 0 does not breach reads only its
last evaluation's history rows, one per group, instead of every row in the
lookback window
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ import { SavedSearch } from '@/models/savedSearch';
import { Source } from '@/models/source';
import * as checkAlertTasks from '@/tasks/checkAlerts';
import { getScheduledWindowStart } from '@/tasks/checkAlerts';
import {
clearAlertEvaluators,
registerAlertEvaluator,
} from '@/tasks/checkAlerts/evaluators';
import {
AlertProvider,
AlertTaskType,
Expand Down Expand Up @@ -745,6 +749,7 @@ describe('DefaultAlertProvider', () => {
describe('history fetch gating (HDX-4791)', () => {
afterEach(() => {
jest.restoreAllMocks();
clearAlertEvaluators();
});

// Alerts created by the helpers use interval '5m' with no schedule
Expand Down Expand Up @@ -982,6 +987,128 @@ describe('DefaultAlertProvider', () => {
);
});

it('reads only the last evaluation for a due grouped alert that a value of 0 does not breach', async () => {
const { alert } = await createValidSavedSearchAlert(
await createConnectionAndSource(),
{ groupBy: 'ServiceName' },
);
const windowMs = 5 * 60_000;
const lastEvaluation = new Date(
currentWindowStart().getTime() - windowMs,
);
// Older evaluations: 'old' last reported OK, so it has no row in the
// last evaluation and no state worth keeping.
for (let i = 2; i <= 4; i++) {
await AlertHistory.create({
alert: alert._id,
createdAt: new Date(lastEvaluation.getTime() - i * windowMs),
state: AlertState.OK,
group: 'old',
});
}
await AlertHistory.create({
alert: alert._id,
createdAt: lastEvaluation,
state: AlertState.ALERT,
group: 'a',
fired: true,
});
await AlertHistory.create({
alert: alert._id,
createdAt: lastEvaluation,
state: AlertState.OK,
group: 'b',
});

const findSpy = jest.spyOn(AlertHistory, 'find');
const result = await provider.getAlertTasks();

// One find, on the last evaluation's createdAt: no lookback scan.
expect(findSpy).toHaveBeenCalledTimes(1);
expect(findSpy.mock.calls[0][0]).toMatchObject({
createdAt: lastEvaluation,
});
const { previousMap } = result[0].alerts[0];
expect([...previousMap.keys()].sort()).toEqual([
`${alert.id}||a`,
`${alert.id}||b`,
]);
const groupA = previousMap.get(`${alert.id}||a`);
expect(groupA?.state).toBe(AlertState.ALERT);
expect(groupA?.fired).toBe(true);
});

it.each([
[
'a value of 0 breaches',
{ thresholdType: AlertThresholdType.BELOW, threshold: 1 },
undefined,
],
[
'a registered evaluator handles',
{ thresholdType: AlertThresholdType.ABOVE, threshold: 1 },
() => true,
],
[
'throws in the evaluator lookup',
{ thresholdType: AlertThresholdType.ABOVE, threshold: 1 },
() => {
throw new Error('evaluator lookup failed');
},
],
[
'has a threshold that cannot be checked',
{ thresholdType: AlertThresholdType.BETWEEN, thresholdMax: null },
undefined,
],
])(
'keeps the lookback scan for a grouped alert that %s',
async (_, threshold, appliesTo?: () => boolean) => {
const { alert } = await createValidSavedSearchAlert(
await createConnectionAndSource(),
{ groupBy: 'ServiceName' },
);
await Alert.updateOne({ _id: alert._id }, { $set: threshold });
if (appliesTo) {
registerAlertEvaluator({
name: 'test',
appliesTo,
evaluate: async () => {},
});
}
const windowMs = 5 * 60_000;
const lastEvaluation = new Date(
currentWindowStart().getTime() - windowMs,
);
// 'gone' stopped reporting while in ALERT, and nothing guarantees
// an OK row for it in the last evaluation: only the scan finds it.
await AlertHistory.create({
alert: alert._id,
createdAt: new Date(lastEvaluation.getTime() - 2 * windowMs),
state: AlertState.ALERT,
group: 'gone',
fired: true,
});
await AlertHistory.create({
alert: alert._id,
createdAt: lastEvaluation,
state: AlertState.OK,
group: 'a',
});

const result = await provider.getAlertTasks();

const { previousMap } = result[0].alerts[0];
expect([...previousMap.keys()].sort()).toEqual([
`${alert.id}||a`,
`${alert.id}||gone`,
]);
expect(previousMap.get(`${alert.id}||gone`)?.state).toBe(
AlertState.ALERT,
);
},
);

it('treats an alert as due when its window start cannot be computed', async () => {
const { alert } = await createValidSavedSearchAlert(
await createConnectionAndSource(),
Expand Down
36 changes: 31 additions & 5 deletions packages/api/src/tasks/checkAlerts/providers/default.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,14 @@ import Webhook, { IWebhook } from '@/models/webhook';
import {
alertHasGroupBy,
computeHistoryMapKey,
doesExceedThreshold,
getAlertWindowStart,
getConsecutiveWindowHistories,
getLatestAlertHistories,
getPreviousAlertHistories,
type LatestAlertHistory,
} from '@/tasks/checkAlerts';
import { findAlertEvaluator } from '@/tasks/checkAlerts/evaluators';
import {
type AlertDetails,
type AlertProvider,
Expand Down Expand Up @@ -477,8 +479,32 @@ async function loadAlerts(): Promise<AlertTask[]> {
}
}

// When a value of 0 does not breach the threshold, every group whose last

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 P2 — Shortcut uses the current threshold, but rows written under an earlier threshold can break the invariant

The invariant depends on the threshold that was active when the last evaluation ran. Editing an alert keeps its AlertHistory (controllers/alerts.ts:380 notes it is not cleared). Example: an alert is BELOW 1, so 0 breaches. Group x goes ALERT, then stops reporting. The missing-group loop writes nothing for it because 0 breaches. The user then changes the alert to ABOVE 5. On the next check only the last evaluation is read. It has no row for x, so x never gets its OK row or a resolve notification. The old lookback scan would have found and resolved it. Fix: fall back to the scan when the last evaluation may have been written under a different threshold, e.g. compare the alert's updatedAt with latest.createdAt. Add a test that edits the threshold.

Advisory. Fix if it is a small defect in code this PR changes; otherwise reply in the thread. Do not widen the PR or touch files it did not already change. How to respond

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not changing this one. updateAlertState writes the alert on every check, so updatedAt is always newer than the last evaluation and that comparison would turn the shortcut off for every alert. The case needs the threshold to change from one that 0 breaches to one it doesn't while a group is in ALERT and no longer reporting; that group then gets no resolve notification. #3702 already drops such groups once they fall outside its lookback, and handling it exactly would need each evaluation to record the threshold it used.

// row is ALERT or PENDING has a row in the alert's last evaluation: the
// threshold path of processAlert writes an OK row for each such group
// missing from the data. A previous OK row only matters for its createdAt,
// which the last evaluation also has. So these alerts read only that
// evaluation (below), one row per group, instead of scanning every row in
// the lookback. A registered evaluator does not run that loop, and an
// evaluator lookup or threshold check that throws is left for processAlert
// to report, so these keep the scan.
const needsLookbackScan = (alert: IAlert) => {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔴 P1 — needsLookbackScan checks the alert's current threshold, but the invariant it relies on was built under the old threshold or evaluator. Editing an alert can leave ALERT groups that never resolve

The shortcut assumes every ALERT/PENDING group has a row in the last evaluation. That only holds if earlier evaluations ran the missing-group loop, i.e. if 0 did not breach the threshold back then and no evaluator handled the alert. Example: an alert with BELOW 1 has group G in ALERT, and G stops reporting. The missing-group loop is skipped because 0 breaches, so G gets no new rows. The user then edits the threshold to ABOVE 100. Alert updates in controllers/alerts.ts do not clear AlertHistory. Now needsLookbackScan is false, the last-evaluation read finds no row for G, and G drops out of previousMap. So the missing-group loop at index.ts:1819-1842 never writes its OK row and no resolve notification is ever sent. Before this PR, the scan found G's ALERT row and resolved it. The same happens when an evaluator stops applying to an alert, since evaluators also skip that loop. Fix: also take the scan path when the alert changed after its last evaluation's createdAt (alert.updatedAt > latest.createdAt, or a stored hash of threshold, thresholdType and evaluator). Add a test that flips the threshold from 0-breaching to non-breaching while a silent group is in ALERT.

Nothing blocks merge automatically, but a maintainer will expect this fixed if it is a real defect in code this PR changes. If it is about surrounding code, reply and say so instead of patching. Do not widen the PR. How to respond

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the same issue as the two threads above. Checking alert.updatedAt doesn't work here: every alert check updates updatedAt, so it is always newer than the last evaluation. Changing that would also change what the UI and the API show. I'm leaving this for a maintainer to decide.

try {
return findAlertEvaluator(alert) != null || doesExceedThreshold(alert, 0);
} catch (e) {
logger.error(
{ alertId: alert.id, error: serializeError(e) },
'Failed to check whether a grouped alert needs the history scan',
);
return true;
}
};
const [previousAlerts, recentHistoryMap] = await Promise.all([
getPreviousAlertHistories(dueGroupedAlerts, now, historyQueryQueue),
getPreviousAlertHistories(
dueGroupedAlerts.filter(needsLookbackScan),
now,
historyQueryQueue,
),
getConsecutiveWindowHistories(dueAlertsWithHistory, now, historyQueryQueue),
]);

Expand All @@ -494,10 +520,10 @@ async function loadAlerts(): Promise<AlertTask[]> {
});
}

// A grouped alert whose latest row is older than its scan window gets no
// rows back. Read the rows of that last evaluation instead (one per group,
// all sharing its createdAt), so the evaluation still starts from it and
// every group keeps its previous state.
// Read the rows of the last evaluation (one per group, all sharing its
// createdAt) for the grouped alerts not scanned above, and for those whose
// latest row is older than their scan window, which get no rows back. The
// evaluation then starts from it and every group keeps its previous state.
await Promise.all(
dueGroupedAlerts.map(alert => {
const latest = latestHistories.get(alert.id);
Expand Down
Loading