Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(monitoring): fail stale running checks

mogery97dfb18926bc
diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.ts
new file mode 100644
index 000000000..4b1cc32ba
--- /dev/null
+++ b/apps/api/src/services/monitoring/runner.test.ts
@@ -0,0 +1,56 @@
+jest.mock("uuid", () => ({
+ v7: () => "test-uuid-v7",
+}));
+
+import { isMonitorCheckStale, MONITOR_CHECK_STALE_TIMEOUT_MS } from "./runner";
+
+describe("monitoring runner", () => {
+ describe("isMonitorCheckStale", () => {
+ const now = new Date("2026-05-06T12:00:00.000Z");
+
+ it("returns true when a running check is at least 24 hours old", () => {
+ expect(
+ isMonitorCheckStale(
+ {
+ started_at: new Date(
+ now.getTime() - MONITOR_CHECK_STALE_TIMEOUT_MS,
+ ).toISOString(),
+ updated_at: now.toISOString(),
+ created_at: now.toISOString(),
+ },
+ now,
+ ),
+ ).toBe(true);
+ });
+
+ it("returns false when a running check is not yet stale", () => {
+ expect(
+ isMonitorCheckStale(
+ {
+ started_at: new Date(
+ now.getTime() - MONITOR_CHECK_STALE_TIMEOUT_MS + 1,
+ ).toISOString(),
+ updated_at: now.toISOString(),
+ created_at: now.toISOString(),
+ },
+ now,
+ ),
+ ).toBe(false);
+ });
+
+ it("falls back to updated_at for malformed started_at values", () => {
+ expect(
+ isMonitorCheckStale(
+ {
+ started_at: null,
+ updated_at: new Date(
+ now.getTime() - MONITOR_CHECK_STALE_TIMEOUT_MS,
+ ).toISOString(),
+ created_at: now.toISOString(),
+ },
+ now,
+ ),
+ ).toBe(true);
+ });
+ });
+});
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 4afa1ce73..e5371dafe 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -55,6 +55,7 @@ import type { MonitorCheckJobData } from "./queue";
const logger = _logger.child({ module: "monitoring-runner" });
const poll = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
+export const MONITOR_CHECK_STALE_TIMEOUT_MS = 24 * 60 * 60 * 1000;
type PageResult = MonitorCheckPageInsert & {
emailStatus?: string;
@@ -917,6 +918,111 @@ async function isMonitorCheckComplete(
return true;
}
+export function isMonitorCheckStale(
+ check: Pick<MonitorCheckRow, "started_at" | "updated_at" | "created_at">,
+ now: Date = new Date(),
+): boolean {
+ const startedAt = check.started_at ?? check.updated_at ?? check.created_at;
+ const startedAtMs = Date.parse(startedAt);
+ if (!Number.isFinite(startedAtMs)) return false;
+ return now.getTime() - startedAtMs >= MONITOR_CHECK_STALE_TIMEOUT_MS;
+}
+
+async function failStaleMonitorCheck(params: {
+ monitor: MonitorRow;
+ check: MonitorCheckRow;
+}): Promise<boolean> {
+ if (!isMonitorCheckStale(params.check)) return false;
+
+ const error = "Monitor check exceeded the 24 hour running timeout.";
+ if (params.check.autumn_lock_id) {
+ await autumnService
+ .finalizeCreditsLock({
+ lockId: params.check.autumn_lock_id,
+ action: "release",
+ properties: {
+ source: "monitorCheck",
+ endpoint: "monitor",
+ jobId: params.check.id,
+ },
+ })
+ .catch(releaseError => {
+ logger.warn("Failed to release stale monitor check credit lock", {
+ error: releaseError,
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ lockId: params.check.autumn_lock_id,
+ });
+ });
+ }
+
+ const finalized = await updateMonitorCheck(params.check.id, {
+ status: "failed",
+ finished_at: new Date().toISOString(),
+ actual_credits: 0,
+ billing_status: params.check.autumn_lock_id ? "released" : "not_applicable",
+ error,
+ });
+
+ const notificationStatus = await sendNotifications({
+ monitor: params.monitor,
+ check: finalized,
+ pages: [],
+ }).catch(notificationError => {
+ logger.warn("Failed to send stale monitor check notifications", {
+ error: notificationError,
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ });
+ return {
+ webhook: {
+ attempted: !!params.monitor.webhook,
+ success: false,
+ error:
+ notificationError instanceof Error
+ ? notificationError.message
+ : String(notificationError),
+ },
+ email: {
+ attempted: !!params.monitor.notification?.email?.enabled,
+ success: false,
+ error:
+ notificationError instanceof Error
+ ? notificationError.message
+ : String(notificationError),
+ },
+ };
+ });
+
+ const withNotifications = await updateMonitorCheck(params.check.id, {
+ notification_status: notificationStatus,
+ }).catch(updateError => {
+ logger.warn("Failed to record stale monitor check notification status", {
+ error: updateError,
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ });
+ return finalized;
+ });
+
+ if (params.monitor.current_check_id === params.check.id) {
+ await updateMonitorScheduleAfterRun({
+ monitor: params.monitor,
+ check: withNotifications,
+ summary: toSummaryObject(withNotifications),
+ });
+ }
+
+ logger.warn("Failed stale monitor check", {
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ startedAt: params.check.started_at,
+ timeoutMs: MONITOR_CHECK_STALE_TIMEOUT_MS,
+ });
+
+ return true;
+}
+
export async function reconcileRunningMonitorChecks(
limit: number = 50,
): Promise<void> {
@@ -933,6 +1039,8 @@ export async function reconcileRunningMonitorChecks(
);
if (!monitor) continue;
+ if (await failStaleMonitorCheck({ monitor, check })) continue;
+
const targetResults = Array.isArray(check.target_results)
? ([...check.target_results] as any[])
: [];