Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Guard paused monitors and add finalize timeout

Developers Digestd2a2d5e64b59
diff --git a/apps/api/src/controllers/v2/monitor.ts b/apps/api/src/controllers/v2/monitor.ts
index 01dee9b74..e39236664 100644
--- a/apps/api/src/controllers/v2/monitor.ts
+++ b/apps/api/src/controllers/v2/monitor.ts
@@ -443,6 +443,12 @@ export async function runMonitorController(
if (!monitor) {
return res.status(404).json({ success: false, error: "Monitor not found" });
}
+ if (monitor.status === "paused") {
+ return res.status(409).json({
+ success: false,
+ error: "Monitor is paused. Resume it before running a check.",
+ });
+ }
if (monitor.current_check_id) {
return res.status(409).json({
success: false,
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 0b61e09a9..75c168a49 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -1038,6 +1038,37 @@ async function enqueueMonitorCrawlTarget(params: {
// Runs inline, persisting onto the same monitor_pages / monitor_check_pages
// tables the reconciler tallies.
+// Bound the inline finalize writes: under write-pool exhaustion they can wait forever,
+// stranding the check until the 10-min reaper. Throw so the catch fails it fast instead.
+const MONITOR_FINALIZE_WRITE_TIMEOUT_MS = 60_000;
+
+class MonitorFinalizeTimeoutError extends Error {
+ constructor(what: string, ms: number) {
+ super(`${what} exceeded ${ms}ms`);
+ this.name = "MonitorFinalizeTimeoutError";
+ }
+}
+
+// Reject (not resolve) on timeout so a stalled write fails fast into the catch path.
+async function withFinalizeTimeout<T>(
+ work: () => Promise<T>,
+ what: string,
+ ms: number = MONITOR_FINALIZE_WRITE_TIMEOUT_MS,
+): Promise<T> {
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ const timeout = new Promise<never>((_, reject) => {
+ timer = setTimeout(
+ () => reject(new MonitorFinalizeTimeoutError(what, ms)),
+ ms,
+ );
+ });
+ try {
+ return await Promise.race([work(), timeout]);
+ } finally {
+ if (timer) clearTimeout(timer);
+ }
+}
+
async function runMonitorSearchTarget(params: {
monitor: MonitorRow;
check: MonitorCheckRow;
@@ -1144,24 +1175,26 @@ async function runMonitorSearchTarget(params: {
};
});
- for (let i = 0; i < pages.length; i++) {
- const page = pages[i];
- await upsertMonitorPage({
- monitorId: monitor.id,
- teamId: monitor.team_id,
- targetId: target.id,
- url: page.url,
- source: "discovered",
- checkId: check.id,
- scrapeId: null,
- status: page.status,
- metadata: page.metadata as Record<string, unknown>,
- });
- }
+ await withFinalizeTimeout(async () => {
+ for (let i = 0; i < pages.length; i++) {
+ const page = pages[i];
+ await upsertMonitorPage({
+ monitorId: monitor.id,
+ teamId: monitor.team_id,
+ targetId: target.id,
+ url: page.url,
+ source: "discovered",
+ checkId: check.id,
+ scrapeId: null,
+ status: page.status,
+ metadata: page.metadata as Record<string, unknown>,
+ });
+ }
- // Clear rows from a prior (crashed/redelivered) run so the insert is a replace, not a duplicate.
- await deleteMonitorCheckPages({ checkId: check.id, targetId: target.id });
- await insertMonitorCheckPages(pages);
+ // Clear rows from a prior (crashed/redelivered) run so the insert is a replace, not a duplicate.
+ await deleteMonitorCheckPages({ checkId: check.id, targetId: target.id });
+ await insertMonitorCheckPages(pages);
+ }, "monitor search page-write tail");
for (const page of pages) {
if (page.status !== "new" && page.status !== "error") continue;
@@ -1326,12 +1359,15 @@ export async function processMonitorCheckJob(
targetRun.resultsJudged = searchResult.resultsJudged;
// Set last, after credits are stamped, so the reconciler never finalizes with credits at 0.
targetRun.searchCompleted = true;
- // Persist now, not just at end-of-loop: the search work already happened,
- // so a crash before the final flush would re-run and re-scrape everything;
- // flushing searchCompleted lets findCompletedSearchTargetRun short-circuit.
- await updateMonitorCheck(check.id, {
- target_results: targetResults,
- });
+ // Persist searchCompleted now so a crash/redelivery short-circuits via
+ // findCompletedSearchTargetRun instead of re-running and re-billing.
+ await withFinalizeTimeout(
+ () =>
+ updateMonitorCheck(check.id, {
+ target_results: targetResults,
+ }),
+ "monitor search searchCompleted flush",
+ );
}
}
diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.ts
index 20838ae37..54d00ef25 100644
--- a/apps/api/src/services/monitoring/types.ts
+++ b/apps/api/src/services/monitoring/types.ts
@@ -64,20 +64,30 @@ const searchTargetSchema = z.preprocess(
}
return value;
},
- z.strictObject({
- id: z.string().uuid().optional(),
- type: z.literal("search"),
- queries: z.array(z.string().min(1).max(256)).min(1).max(12),
- searchWindow: z
- .enum(["5m", "15m", "1h", "6h", "24h", "7d"], {
- error: "searchWindow must be one of: 5m, 15m, 1h, 6h, 24h, 7d",
- })
- .optional()
- .default("24h"),
- includeDomains: z.array(monitorDomainSchema).max(50).optional(),
- excludeDomains: z.array(monitorDomainSchema).max(50).optional(),
- maxResults: z.number().int().min(1).max(50).optional().default(10),
- }),
+ z
+ .strictObject({
+ id: z.string().uuid().optional(),
+ type: z.literal("search"),
+ queries: z.array(z.string().min(1).max(256)).min(1).max(12),
+ searchWindow: z
+ .enum(["5m", "15m", "1h", "6h", "24h", "7d"], {
+ error: "searchWindow must be one of: 5m, 15m, 1h, 6h, 24h, 7d",
+ })
+ .optional()
+ .default("24h"),
+ includeDomains: z.array(monitorDomainSchema).max(50).optional(),
+ excludeDomains: z.array(monitorDomainSchema).max(50).optional(),
+ maxResults: z.number().int().min(1).max(50).optional().default(10),
+ })
+ .superRefine((target, ctx) => {
+ if (target.includeDomains?.length && target.excludeDomains?.length) {
+ ctx.addIssue({
+ code: "custom",
+ message: "includeDomains and excludeDomains are mutually exclusive",
+ path: ["excludeDomains"],
+ });
+ }
+ }),
);
const monitorTargetSchema = z.union([
@@ -229,6 +239,8 @@ export const updateMonitorSchema = createMonitorBaseSchema
notification: monitorNotificationInner.optional(),
// Drop the create-path .default(30): otherwise an omitting PATCH resets configured retention.
retentionDays: z.number().int().positive().max(365).optional(),
+ // Drop the create-path .prefault("api") so an empty PATCH stays empty (the guard below fires).
+ origin: z.string().optional(),
})
.superRefine(rejectGoalClearedWithSearchTargets)
.refine(x => Object.keys(x).length > 0, "Update body cannot be empty");