Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(monitoring): advance scheduled checks before enqueue

mogery4c7af99c172f
diff --git a/apps/api/src/services/monitoring/scheduler.test.ts b/apps/api/src/services/monitoring/scheduler.test.ts
new file mode 100644
index 000000000..c5ec32983
--- /dev/null
+++ b/apps/api/src/services/monitoring/scheduler.test.ts
@@ -0,0 +1,134 @@
+import { addMonitorCheckJob } from "./queue";
+import { enqueueDueMonitorChecks } from "./scheduler";
+import {
+ advanceMonitorAfterSkippedCheck,
+ claimDueMonitors,
+ createMonitorCheck,
+ dispatchScheduledMonitorCheck,
+ updateMonitorCheck,
+ updateMonitorScheduleAfterRun,
+} from "./store";
+
+jest.mock("./queue", () => ({
+ addMonitorCheckJob: jest.fn(),
+}));
+
+jest.mock("./store", () => ({
+ advanceMonitorAfterSkippedCheck: jest.fn(),
+ claimDueMonitors: jest.fn(),
+ createMonitorCheck: jest.fn(),
+ dispatchScheduledMonitorCheck: jest.fn(),
+ updateMonitorCheck: jest.fn(),
+ updateMonitorScheduleAfterRun: jest.fn(),
+}));
+
+const mockAddMonitorCheckJob = addMonitorCheckJob as jest.MockedFunction<
+ typeof addMonitorCheckJob
+>;
+const mockClaimDueMonitors = claimDueMonitors as jest.MockedFunction<
+ typeof claimDueMonitors
+>;
+const mockCreateMonitorCheck = createMonitorCheck as jest.MockedFunction<
+ typeof createMonitorCheck
+>;
+const mockDispatchScheduledMonitorCheck =
+ dispatchScheduledMonitorCheck as jest.MockedFunction<
+ typeof dispatchScheduledMonitorCheck
+ >;
+const mockUpdateMonitorCheck = updateMonitorCheck as jest.MockedFunction<
+ typeof updateMonitorCheck
+>;
+const mockAdvanceMonitorAfterSkippedCheck =
+ advanceMonitorAfterSkippedCheck as jest.MockedFunction<
+ typeof advanceMonitorAfterSkippedCheck
+ >;
+const mockUpdateMonitorScheduleAfterRun =
+ updateMonitorScheduleAfterRun as jest.MockedFunction<
+ typeof updateMonitorScheduleAfterRun
+ >;
+
+describe("monitoring scheduler", () => {
+ const monitor = {
+ id: "monitor-1",
+ team_id: "team-1",
+ current_check_id: null,
+ next_run_at: "2026-05-05T18:45:00.000Z",
+ } as any;
+ const check = { id: "check-1" } as any;
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+ mockClaimDueMonitors.mockResolvedValue([monitor]);
+ mockCreateMonitorCheck.mockResolvedValue(check);
+ mockDispatchScheduledMonitorCheck.mockResolvedValue(true);
+ mockAddMonitorCheckJob.mockResolvedValue(undefined);
+ mockAdvanceMonitorAfterSkippedCheck.mockResolvedValue(undefined);
+ mockUpdateMonitorScheduleAfterRun.mockResolvedValue(undefined);
+ });
+
+ it("dispatches and advances a scheduled monitor before enqueueing its job", async () => {
+ await expect(
+ enqueueDueMonitorChecks({ workerId: "worker-1" }),
+ ).resolves.toBe(1);
+
+ expect(mockCreateMonitorCheck).toHaveBeenCalledWith({
+ monitor,
+ trigger: "scheduled",
+ scheduledFor: monitor.next_run_at,
+ });
+ expect(mockDispatchScheduledMonitorCheck).toHaveBeenCalledWith({
+ monitor,
+ checkId: check.id,
+ });
+ expect(mockAddMonitorCheckJob).toHaveBeenCalledWith({
+ monitorId: monitor.id,
+ checkId: check.id,
+ teamId: monitor.team_id,
+ });
+ expect(
+ mockDispatchScheduledMonitorCheck.mock.invocationCallOrder[0],
+ ).toBeLessThan(mockAddMonitorCheckJob.mock.invocationCallOrder[0]);
+ });
+
+ it("fails and clears a dispatched check when enqueueing fails", async () => {
+ const error = new Error("queue unavailable");
+ const failed = { id: check.id, status: "failed" } as any;
+ mockAddMonitorCheckJob.mockRejectedValue(error);
+ mockUpdateMonitorCheck.mockResolvedValue(failed);
+
+ await expect(
+ enqueueDueMonitorChecks({ workerId: "worker-1" }),
+ ).resolves.toBe(0);
+
+ expect(mockUpdateMonitorCheck).toHaveBeenCalledWith(check.id, {
+ status: "failed",
+ finished_at: expect.any(String),
+ error: error.message,
+ });
+ expect(mockUpdateMonitorScheduleAfterRun).toHaveBeenCalledWith({
+ monitor,
+ check: failed,
+ });
+ });
+
+ it("records an overlap if dispatch finds another running check", async () => {
+ const skipped = { id: check.id, status: "skipped_overlap" } as any;
+ mockDispatchScheduledMonitorCheck.mockResolvedValue(false);
+ mockUpdateMonitorCheck.mockResolvedValue(skipped);
+
+ await expect(
+ enqueueDueMonitorChecks({ workerId: "worker-1" }),
+ ).resolves.toBe(0);
+
+ expect(mockAddMonitorCheckJob).not.toHaveBeenCalled();
+ expect(mockUpdateMonitorCheck).toHaveBeenCalledWith(check.id, {
+ status: "skipped_overlap",
+ finished_at: expect.any(String),
+ error: "Previous monitor check is still running.",
+ });
+ expect(mockAdvanceMonitorAfterSkippedCheck).toHaveBeenCalledWith({
+ monitor,
+ check: skipped,
+ });
+ });
+});
diff --git a/apps/api/src/services/monitoring/scheduler.ts b/apps/api/src/services/monitoring/scheduler.ts
index 99b817b34..84a7984e7 100644
--- a/apps/api/src/services/monitoring/scheduler.ts
+++ b/apps/api/src/services/monitoring/scheduler.ts
@@ -5,7 +5,9 @@ import {
advanceMonitorAfterSkippedCheck,
claimDueMonitors,
createMonitorCheck,
+ dispatchScheduledMonitorCheck,
updateMonitorCheck,
+ updateMonitorScheduleAfterRun,
} from "./store";
const logger = _logger.child({ module: "monitoring-scheduler" });
@@ -34,6 +36,8 @@ export async function enqueueDueMonitorChecks(
let enqueued = 0;
for (const monitor of monitors) {
+ let check: Awaited<ReturnType<typeof createMonitorCheck>> | null = null;
+ let dispatched = false;
try {
if (monitor.current_check_id) {
const skipped = await createMonitorCheck({
@@ -51,11 +55,25 @@ export async function enqueueDueMonitorChecks(
continue;
}
- const check = await createMonitorCheck({
+ check = await createMonitorCheck({
monitor,
trigger: "scheduled",
scheduledFor: monitor.next_run_at,
});
+ dispatched = await dispatchScheduledMonitorCheck({
+ monitor,
+ checkId: check.id,
+ });
+ if (!dispatched) {
+ check = await updateMonitorCheck(check.id, {
+ status: "skipped_overlap",
+ finished_at: new Date().toISOString(),
+ error: "Previous monitor check is still running.",
+ });
+ await advanceMonitorAfterSkippedCheck({ monitor, check });
+ continue;
+ }
+
await enqueueMonitorCheck({
monitorId: monitor.id,
checkId: check.id,
@@ -63,6 +81,38 @@ export async function enqueueDueMonitorChecks(
});
enqueued++;
} catch (error) {
+ if (check) {
+ const failed = await updateMonitorCheck(check.id, {
+ status: "failed",
+ finished_at: new Date().toISOString(),
+ error: error instanceof Error ? error.message : String(error),
+ }).catch(updateError => {
+ logger.error("Failed to mark monitor check enqueue failure", {
+ updateError,
+ error,
+ monitorId: monitor.id,
+ checkId: check?.id,
+ teamId: monitor.team_id,
+ });
+ return null;
+ });
+
+ if (failed && dispatched) {
+ await updateMonitorScheduleAfterRun({
+ monitor,
+ check: failed,
+ }).catch(updateError => {
+ logger.error("Failed to clear failed dispatched monitor check", {
+ updateError,
+ error,
+ monitorId: monitor.id,
+ checkId: failed.id,
+ teamId: monitor.team_id,
+ });
+ });
+ }
+ }
+
logger.error("Failed to enqueue due monitor check", {
error,
monitorId: monitor.id,
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index 2d4e51dff..d5a2f2697 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -254,6 +254,37 @@ export async function markMonitorRunning(params: {
throwIfError(error, "Failed to mark monitor running");
}
+export async function dispatchScheduledMonitorCheck(params: {
+ monitor: MonitorRow;
+ checkId: string;
+}): Promise<boolean> {
+ const nextRunAt =
+ params.monitor.status === "active"
+ ? getNextMonitorRunAt(
+ params.monitor.schedule_cron,
+ new Date(),
+ params.monitor.schedule_timezone,
+ ).toISOString()
+ : null;
+
+ const { data, error } = await supabase_service
+ .from("monitors")
+ .update({
+ current_check_id: params.checkId,
+ locked_at: null,
+ locked_until: null,
+ next_run_at: nextRunAt,
+ updated_at: new Date().toISOString(),
+ })
+ .eq("id", params.monitor.id)
+ .is("current_check_id", null)
+ .select("id")
+ .maybeSingle();
+
+ throwIfError(error, "Failed to dispatch scheduled monitor check");
+ return !!data;
+}
+
export async function updateMonitorScheduleAfterRun(params: {
monitor: MonitorRow;
check: MonitorCheckRow;