Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api/monitoring): prevent double webhook send and backfill deliver… (#3618)

tomsideguide3d1d2ca37daf
diff --git a/apps/api/src/controllers/v2/monitor.ts b/apps/api/src/controllers/v2/monitor.ts
index bd90cae70..dae0eea83 100644
--- a/apps/api/src/controllers/v2/monitor.ts
+++ b/apps/api/src/controllers/v2/monitor.ts
@@ -35,6 +35,12 @@ import {
trackMonitorConfiguredInterest,
trackMonitorDeactivatedInterest,
} from "../../services/monitoring/interest";
+import {
+ getLatestWebhookLog,
+ getLatestWebhookLogsByJob,
+ type WebhookLogRow,
+} from "../../services/webhook/logs";
+import { WebhookEvent } from "../../services/webhook";
const logger = _logger.child({ module: "monitor-controller" });
@@ -86,6 +92,38 @@ function serializeMonitor(monitor: any) {
};
}
+function overlayWebhookLog<T extends { notificationStatus: any }>(
+ serialized: T,
+ log: WebhookLogRow | null,
+): T {
+ if (!log) return serialized;
+ const notificationStatus =
+ serialized.notificationStatus &&
+ typeof serialized.notificationStatus === "object"
+ ? serialized.notificationStatus
+ : {};
+ const existing =
+ notificationStatus.webhook && typeof notificationStatus.webhook === "object"
+ ? notificationStatus.webhook
+ : {};
+ return {
+ ...serialized,
+ notificationStatus: {
+ ...notificationStatus,
+ webhook: {
+ ...existing,
+ attempted: true,
+ success: log.success === true,
+ delivered: log.success === true,
+ queued: false,
+ statusCode: log.status_code ?? undefined,
+ error: log.error ?? undefined,
+ deliveredAt: log.created_at,
+ },
+ },
+ };
+}
+
function serializeCheck(check: any) {
return {
id: check.id,
@@ -357,9 +395,19 @@ export async function listMonitorChecksController(
status: query.status,
});
+ const webhookLogs = await getLatestWebhookLogsByJob({
+ jobIds: checks.map(c => c.id),
+ event: WebhookEvent.MONITOR_CHECK_COMPLETED,
+ });
+
res.status(200).json({
success: true,
- data: checks.map(serializeCheck),
+ data: checks.map(check =>
+ overlayWebhookLog(
+ serializeCheck(check),
+ webhookLogs.get(check.id) ?? null,
+ ),
+ ),
});
}
@@ -375,7 +423,7 @@ export async function getMonitorCheckController(
return res.status(404).json({ success: false, error: "Check not found" });
}
- const [pages, totalPagesForFilter] = await Promise.all([
+ const [pages, totalPagesForFilter, webhookLog] = await Promise.all([
listMonitorCheckPages({
teamId: req.auth.team_id,
monitorId,
@@ -388,6 +436,10 @@ export async function getMonitorCheckController(
checkId,
status: query.status,
}),
+ getLatestWebhookLog({
+ jobId: checkId,
+ event: WebhookEvent.MONITOR_CHECK_COMPLETED,
+ }),
]);
const pagesWithDiffs = await Promise.all(
@@ -442,7 +494,7 @@ export async function getMonitorCheckController(
success: true,
next,
data: {
- ...serializeCheck(check),
+ ...overlayWebhookLog(serializeCheck(check), webhookLog),
pages: pagesWithDiffs,
next,
},
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 6ad84a5e3..6122eb4c0 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -28,6 +28,7 @@ import {
import { createWebhookSender, WebhookEvent } from "../webhook";
import { sendMonitoringEmailSummary } from "../notification/monitoring_email";
import {
+ getMonitorCheck,
getMonitorForUpdate,
getMonitorPage,
countMonitorCheckPages,
@@ -64,6 +65,24 @@ export { isMonitorCheckStale, MONITOR_CHECK_STALE_TIMEOUT_MS };
const MONITOR_DEFAULT_WAIT_FOR_MS = 5000;
const MONITOR_MAX_WAIT_FOR_MS = 60000;
const MONITOR_MAX_TIMEOUT_MS = MONITOR_MAX_WAIT_FOR_MS * 2;
+const MONITOR_NOTIFY_CLAIM_TTL_SECONDS = 7 * 24 * 60 * 60;
+const TERMINAL_CHECK_STATUSES = new Set([
+ "completed",
+ "partial",
+ "failed",
+ "skipped_overlap",
+]);
+
+async function claimMonitorNotification(checkId: string): Promise<boolean> {
+ const result = await redisEvictConnection.set(
+ `monitor-check-notify:${checkId}`,
+ "1",
+ "EX",
+ MONITOR_NOTIFY_CLAIM_TTL_SECONDS,
+ "NX",
+ );
+ return result === "OK";
+}
type PageResult = MonitorCheckPageInsert & {
emailStatus?: string;
@@ -837,12 +856,24 @@ export async function processMonitorCheckJob(
throw new Error("Monitor not found");
}
+ const initialCheck = await getMonitorCheck(
+ job.teamId,
+ job.monitorId,
+ job.checkId,
+ );
+ if (!initialCheck) {
+ throw new Error("Monitor check not found");
+ }
+ if (TERMINAL_CHECK_STATUSES.has(initialCheck.status)) {
+ return;
+ }
+
await markMonitorRunning({
monitorId: monitor.id,
checkId: job.checkId,
});
- let check = await updateMonitorCheck(job.checkId, {
+ let check: MonitorCheckRow = await updateMonitorCheck(job.checkId, {
status: "running",
started_at: new Date().toISOString(),
});
@@ -909,15 +940,37 @@ export async function processMonitorCheckJob(
error: error instanceof Error ? error.message : String(error),
});
- await sendNotifications({
- monitor,
- check,
- pages: [],
- }).catch(err =>
- logger.warn("Failed to send monitor failure notifications", {
- error: err,
- }),
- );
+ if ((await claimMonitorNotification(check.id).catch(error => {
+ logger.warn("Failed to claim monitor notification; continuing without dedupe", {
+ error,
+ monitorId: monitor.id,
+ checkId: check.id,
+ });
+ return true;
+ }))) {
+ const notificationStatus = await sendNotifications({
+ monitor,
+ check,
+ pages: [],
+ }).catch(err => {
+ logger.warn("Failed to send monitor failure notifications", {
+ error: err,
+ });
+ return null;
+ });
+ if (notificationStatus) {
+ check = await updateMonitorCheck(check.id, {
+ notification_status: notificationStatus,
+ }).catch(updateError => {
+ logger.warn("Failed to record monitor failure notification status", {
+ error: updateError,
+ monitorId: monitor.id,
+ checkId: check.id,
+ });
+ return check;
+ });
+ }
+ }
await updateMonitorScheduleAfterRun({
monitor,
@@ -1066,46 +1119,49 @@ async function failStaleMonitorCheck(params: {
error,
});
- const notificationStatus = await sendNotifications({
- monitor: params.monitor,
- check: finalized,
- pages: [],
- }).catch(notificationError => {
- logger.warn("Failed to send stale monitor check notifications", {
- error: notificationError,
- monitorId: params.monitor.id,
- checkId: params.check.id,
+ let withNotifications = finalized;
+ if (await claimMonitorNotification(params.check.id)) {
+ const notificationStatus = await sendNotifications({
+ monitor: params.monitor,
+ check: finalized,
+ pages: [],
+ }).catch(notificationError => {
+ logger.warn("Failed to send stale monitor check notifications", {
+ error: notificationError,
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ });
+ return {
+ webhook: {
+ attempted: !!params.monitor.webhook,
+ success: false,
+ error:
+ notificationError instanceof Error
+ ? notificationError.message
+ : String(notificationError),
+ },
+ email: {
+ attempted: !!params.monitor.notification?.email?.enabled,
+ success: false,
+ error:
+ notificationError instanceof Error
+ ? notificationError.message
+ : String(notificationError),
+ },
+ };
});
- return {
- webhook: {
- attempted: !!params.monitor.webhook,
- success: false,
- error:
- notificationError instanceof Error
- ? notificationError.message
- : String(notificationError),
- },
- email: {
- attempted: !!params.monitor.notification?.email?.enabled,
- success: false,
- error:
- notificationError instanceof Error
- ? notificationError.message
- : String(notificationError),
- },
- };
- });
- const withNotifications = await updateMonitorCheck(params.check.id, {
- notification_status: notificationStatus,
- }).catch(updateError => {
- logger.warn("Failed to record stale monitor check notification status", {
- error: updateError,
- monitorId: params.monitor.id,
- checkId: params.check.id,
+ withNotifications = await updateMonitorCheck(params.check.id, {
+ notification_status: notificationStatus,
+ }).catch(updateError => {
+ logger.warn("Failed to record stale monitor check notification status", {
+ error: updateError,
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ });
+ return finalized;
});
- return finalized;
- });
+ }
if (params.monitor.current_check_id === params.check.id) {
await updateMonitorScheduleAfterRun({
@@ -1218,60 +1274,62 @@ export async function reconcileRunningMonitorChecks(
});
}
- let notificationStatus: { webhook?: unknown; email?: unknown } | null =
- null;
- try {
- const pages = (await listMonitorCheckPages({
- teamId: monitor.team_id,
- monitorId: monitor.id,
- checkId: check.id,
- limit: 100,
- skip: 0,
- })) as PageResult[];
-
- notificationStatus = await sendNotifications({
- monitor,
- check: finalized,
- pages,
- });
+ if (await claimMonitorNotification(check.id)) {
+ let notificationStatus: { webhook?: unknown; email?: unknown } | null =
+ null;
+ try {
+ const pages = (await listMonitorCheckPages({
+ teamId: monitor.team_id,
+ monitorId: monitor.id,
+ checkId: check.id,
+ limit: 100,
+ skip: 0,
+ })) as PageResult[];
+
+ notificationStatus = await sendNotifications({
+ monitor,
+ check: finalized,
+ pages,
+ });
- finalized = await updateMonitorCheck(check.id, {
- notification_status: notificationStatus,
- webhook_payload: notificationStatus.webhook
- ? { summary: toSummaryObject(finalized) }
- : null,
- email_payload: notificationStatus.email
- ? { summary: toSummaryObject(finalized) }
- : null,
- });
- } catch (error) {
- logger.warn("Failed to send monitor check notifications", {
- monitorId: monitor.id,
- checkId: finalized.id,
- error,
- });
- notificationStatus = {
- webhook: {
- attempted: !!monitor.webhook,
- success: false,
- error: error instanceof Error ? error.message : String(error),
- },
- email: {
- attempted: !!monitor.notification?.email?.enabled,
- success: false,
- error: error instanceof Error ? error.message : String(error),
- },
- };
- finalized = await updateMonitorCheck(check.id, {
- notification_status: notificationStatus,
- }).catch(updateError => {
- logger.warn("Failed to record monitor check notification failure", {
+ finalized = await updateMonitorCheck(check.id, {
+ notification_status: notificationStatus,
+ webhook_payload: notificationStatus.webhook
+ ? { summary: toSummaryObject(finalized) }
+ : null,
+ email_payload: notificationStatus.email
+ ? { summary: toSummaryObject(finalized) }
+ : null,
+ });
+ } catch (error) {
+ logger.warn("Failed to send monitor check notifications", {
monitorId: monitor.id,
checkId: finalized.id,
- error: updateError,
+ error,
});
- return finalized;
- });
+ notificationStatus = {
+ webhook: {
+ attempted: !!monitor.webhook,
+ success: false,
+ error: error instanceof Error ? error.message : String(error),
+ },
+ email: {
+ attempted: !!monitor.notification?.email?.enabled,
+ success: false,
+ error: error instanceof Error ? error.message : String(error),
+ },
+ };
+ finalized = await updateMonitorCheck(check.id, {
+ notification_status: notificationStatus,
+ }).catch(updateError => {
+ logger.warn("Failed to record monitor check notification failure", {
+ monitorId: monitor.id,
+ checkId: finalized.id,
+ error: updateError,
+ });
+ return finalized;
+ });
+ }
}
await updateMonitorScheduleAfterRun({
diff --git a/apps/api/src/services/webhook/logs.ts b/apps/api/src/services/webhook/logs.ts
new file mode 100644
index 000000000..af69ac1d1
--- /dev/null
+++ b/apps/api/src/services/webhook/logs.ts
@@ -0,0 +1,68 @@
+import { supabase_rr_service } from "../supabase";
+import { logger as _logger } from "../../lib/logger";
+
+export type WebhookLogRow = {
+ id: string;
+ success: boolean;
+ error: string | null;
+ status_code: number | null;
+ latency_ms: number | null;
+ url: string;
+ event: string;
+ created_at: string;
+};
+
+export async function getLatestWebhookLog(params: {
+ jobId: string;
+ event: string;
+}): Promise<WebhookLogRow | null> {
+ const { data, error } = await supabase_rr_service
+ .from("webhook_logs")
+ .select("id,success,error,status_code,latency_ms,url,event,created_at")
+ .eq("crawl_id", params.jobId)
+ .eq("event", params.event)
+ .order("created_at", { ascending: false })
+ .limit(1)
+ .maybeSingle();
+ if (error) {
+ _logger.warn("Failed to fetch latest webhook log", {
+ module: "webhook-logs",
+ jobId: params.jobId,
+ event: params.event,
+ error,
+ });
+ return null;
+ }
+ return data as WebhookLogRow | null;
+}
+
+export async function getLatestWebhookLogsByJob(params: {
+ jobIds: string[];
+ event: string;
+}): Promise<Map<string, WebhookLogRow>> {
+ const result = new Map<string, WebhookLogRow>();
+ if (params.jobIds.length === 0) return result;
+
+ const { data, error } = await supabase_rr_service
+ .from("webhook_logs")
+ .select(
+ "id,success,error,status_code,latency_ms,url,event,created_at,crawl_id",
+ )
+ .in("crawl_id", params.jobIds)
+ .eq("event", params.event)
+ .order("created_at", { ascending: false });
+
+ if (error) {
+ _logger.warn("Failed to fetch webhook logs batch", {
+ module: "webhook-logs",
+ event: params.event,
+ error,
+ });
+ return result;
+ }
+
+ for (const row of (data ?? []) as (WebhookLogRow & { crawl_id: string })[]) {
+ if (!result.has(row.crawl_id)) result.set(row.crawl_id, row);
+ }
+ return result;
+}